Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#794: Fixed install_via_pip.pl might downgrade already installed packages #358

Merged
merged 13 commits into from
Jun 18, 2024
16 changes: 14 additions & 2 deletions ext/scripts/install_scripts/install_via_pip.pl
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ =head1 SYNOPSIS
--ignore-installed Set the --ignore-installed option for pip
--use-deprecated-legacy-resolver Set the --use-deprecated=legacy-resolver option for pip
--python-binary Python-binary to use for the installation
--ancestor-pip-package-root-path Base directory of pip package files for previous build steps

=cut

Expand All @@ -36,6 +37,7 @@ =head1 SYNOPSIS
my $allow_no_version_for_urls = 0;
my $ignore_installed = 0;
my $use_deprecated_legacy_resolver = 0;
my $ancestor_pip_package_root_path = '';
GetOptions (
"help" => \$help,
"dry-run" => \$dry_run,
Expand All @@ -45,7 +47,8 @@ =head1 SYNOPSIS
"allow-no-version-for-urls" => \$allow_no_version_for_urls,
"ignore-installed" => \$ignore_installed,
"use-deprecated-legacy-resolver" => \$use_deprecated_legacy_resolver,
"python-binary=s" => \$python_binary
"python-binary=s" => \$python_binary,
"ancestor-pip-package-root-path=s" => \$ancestor_pip_package_root_path
) or package_mgmt_utils::print_usage_and_abort(__FILE__,"Error in command line arguments",2);
package_mgmt_utils::print_usage_and_abort(__FILE__,"",0) if $help;

Expand Down Expand Up @@ -102,9 +105,18 @@ sub replace_missing_version_for_urls{
@rendered_line_transformation_functions = (\&replace_missing_version_for_urls);
}

my $cmd =
my $cmd = '';

if($ancestor_pip_package_root_path eq '') {
$cmd =
package_mgmt_utils::generate_joined_and_transformed_string_from_file(
$file, $element_separator, $combining_template, \@templates, \@separators, \@rendered_line_transformation_functions);
} else {
my @all_files = package_mgmt_utils::merge_package_files($file, $ancestor_pip_package_root_path, 'python3_pip_packages');
$cmd =
package_mgmt_utils::generate_joined_and_transformed_string_from_files(
$element_separator, $combining_template, \@templates, \@separators, \@rendered_line_transformation_functions, \@all_files);
}

if($with_versions){
if (index($cmd, "==<<<<1>>>>") != -1) {
Expand Down
59 changes: 58 additions & 1 deletion ext/scripts/package_mgmt_utils.pm
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,37 @@ sub generate_joined_and_transformed_string_from_file{
return $final_string;
}


sub generate_joined_and_transformed_string_from_files {
my ($element_separator, $combining_template, $templates_ref, $separators_ref, $rendered_line_transformation_functions_ref, $files_ref) = @_;
my @transformed_lines;
foreach my $file ( @$files_ref ) {
my @transformed_lines_for_current_file = generate_transformed_lines_for_templates(
$file, $element_separator, $templates_ref, $rendered_line_transformation_functions_ref);

if (!@transformed_lines) {
@transformed_lines = @transformed_lines_for_current_file;
} else {
if ($#transformed_lines_for_current_file != $#transformed_lines) {
die "Internal error processing package file $file\n";
}
for my $i (0 .. $#transformed_lines_for_current_file) {
#Resolve reference for the resulting and new arrays, merge both and assign back the reference to the resulting array
my $transformed_lines_for_current_file_part_ref = $transformed_lines_for_current_file[$i];
my @transformed_lines_for_current_file_part = @$transformed_lines_for_current_file_part_ref;

my $transformed_lines_part_ref = $transformed_lines[$i];
my @transformed_lines_part = @$transformed_lines_part_ref;

push (@transformed_lines_part, @transformed_lines_for_current_file_part);
$transformed_lines[$i] = \@transformed_lines_part;
}
}
}
my $final_string = generate_joined_string_from_lines(\@transformed_lines, $combining_template, $separators_ref);
return $final_string;
}

sub generate_joined_string_from_lines{
my ($lines_ref, $combining_template, $separators_ref) = @_;
my @separators = @$separators_ref;
Expand Down Expand Up @@ -219,5 +250,31 @@ sub print_usage_and_abort{
exit($exitcode);
}

1;
sub find_files_matching_pattern {
my ($dir, $pattern) = @_;
my @files;

opendir(my $dh, $dir) or die "Can't open directory $dir: $!";
while (my $file = readdir($dh)) {
next if ($file =~ /^\./); # Skip hidden files
my $path = "$dir/$file";
if (-d $path) {
# Recursively traverse subdirectories
push(@files, find_files_matching_pattern($path, $pattern));
} elsif (-f $path and $file =~ /^$pattern$/) {
push(@files, $path);
}
}
closedir($dh);

return @files;
}

sub merge_package_files {
my ($base_file, $base_search_dir, $file_pattern) = @_;

my @files_in_search_dir = find_files_matching_pattern($base_search_dir, $file_pattern);
return sort ($base_file, @files_in_search_dir);
}

1;
112 changes: 101 additions & 11 deletions ext/scripts/tests/install_scripts/run_pip_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,61 +9,151 @@ SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
# shellcheck source=./ext/scripts/tests/install_scripts/assert.sh
source "$SCRIPT_DIR/assert.sh"

PATH_TO_INSTALL_SCRIPTS="$SCRIPT_DIR/../../install_scripts"
if [ -z "${PATH_TO_INSTALL_SCRIPTS-}" ]
then
PATH_TO_INSTALL_SCRIPTS="$SCRIPT_DIR/../../install_scripts"
fi

DRY_RUN_OPTION=--dry-run
if [ "${1-}" == "--no-dry-run" ]
then
DRY_RUN_OPTION=
fi

if [ -z "${RUN_PIP_TESTS_EXECUTOR-}" ]
then
echo Running pip tests without exector.
else
echo Running pip tests with executor "'$RUN_PIP_TESTS_EXECUTOR'".
fi

function run_install() {
if [ -z "${RUN_PIP_TESTS_EXECUTOR-}" ]
then
eval "$*"
else
eval "$RUN_PIP_TESTS_EXECUTOR $PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl $*"
fi
}

function run_install_must_fail() {
if [ -z "${RUN_PIP_TESTS_EXECUTOR-}" ]
then
if [ -z "${DRY_RUN_OPTION-}" ]
then
eval "$*" && return 1 || return 0;
else
eval "$*"
fi
else
if [ -z "${DRY_RUN_OPTION-}" ]
then
eval "$RUN_PIP_TESTS_EXECUTOR $PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl $*" && return 1 || return 0;
else
eval "$RUN_PIP_TESTS_EXECUTOR $PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl $*"
fi
fi
}


echo ./install_via_pip.pl with empty
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/empty_test_file --python-binary python3 "$DRY_RUN_OPTION")
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/empty_test_file --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" ""
echo

echo ./install_via_pip.pl without versions
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --python-binary python3 "$DRY_RUN_OPTION")
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
echo


echo ./install_via_pip.pl without versions and --ignore-installed
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --ignore-installed --python-binary python3 "$DRY_RUN_OPTION")
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --ignore-installed --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --ignore-installed --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
echo


echo ./install_via_pip.pl without versions and --use-deprecated-legacy-resolver
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --use-deprecated-legacy-resolver --python-binary python3 "$DRY_RUN_OPTION")
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --use-deprecated-legacy-resolver --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --use-deprecated=legacy-resolver --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
echo


echo ./install_via_pip.pl with versions, without allow-no-version
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --with-versions --python-binary python3 "$DRY_RUN_OPTION")
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --with-versions --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'"
echo


echo ./install_via_pip.pl with versions, with allow-no-version, all versions specified
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --with-versions --allow-no-version --python-binary python3 "$DRY_RUN_OPTION")
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --with-versions --allow-no-version --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'"
echo


echo ./install_via_pip.pl with versions, with allow-no-version, some versions missing
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/some_missing_versions --with-versions --allow-no-version --python-binary python3 "$DRY_RUN_OPTION")
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/some_missing_versions --with-versions --allow-no-version --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
echo


echo ./install_via_pip.pl with versions, with allow-no-version-for-urls, file with urls
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/with_urls --with-versions --allow-no-version-for-urls --python-binary python3 "$DRY_RUN_OPTION")
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/with_urls --with-versions --allow-no-version-for-urls --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.27.1' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
echo


echo ./install_via_pip.pl with versions, with allow-no-version-for-urls, file with urls and some missing versions
"$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/with_urls_some_missing_versions --with-versions --allow-no-version-for-urls --python-binary python3 "$DRY_RUN_OPTION" || echo PASSED
run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/with_urls_some_missing_versions --with-versions --allow-no-version-for-urls --python-binary python3 "$DRY_RUN_OPTION" || echo PASSED
echo


echo ./install_via_pip.pl with pip version syntax
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/pip_version_syntax --python-binary python3 "$DRY_RUN_OPTION")
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/pip_version_syntax --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests>=2.21.0' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
echo


echo ./install_via_pip.pl installing a package twice with different versions must fail
TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/same_pkg/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/same_pkg/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.4' 'azure-common==1.1.28'"
echo


echo ./install_via_pip.pl installing with ancestors but all empty
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/empty/step2 --ancestor-pip-package-root-path test_files/pip/empty/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" ""
echo


echo ./install_via_pip.pl installing with ancestors and correct dependency
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/no_version_conflict/dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/no_version_conflict/dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.4' 'azure-batch==1.0.0'"
echo


echo ./install_via_pip.pl installing with ancestors and dependency with wrong version must fail
TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.28' 'azure-batch==1.0.0'"
echo


echo ./install_via_pip.pl installing with ancestors and package which has a dependency to an older package must fail
TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/other_package_with_older_dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/other_package_with_older_dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-storage-queue==1.1.0' 'azure-batch==1.0.0'"
echo


echo ./install_via_pip.pl installing with ancestors and package which has a dependency to a newer package must fail
TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/other_package_with_newer_dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/other_package_with_newer_dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-batch==1.0.0' 'azure-storage-queue==1.1.0'"
echo


echo ./install_via_pip.pl installing with multiple ancestors
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/no_version_conflict/multiple_ancestors/step3 --ancestor-pip-package-root-path test_files/pip/no_version_conflict/multiple_ancestors/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.28' 'azure-batch==14.2.0' 'azure-storage-queue==1.1.0'"
echo

check_for_failed_tests
echo "All pip tests passed"
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-common|1.1.4
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-batch|1.0.0 #depends on azure-common1.1.4
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-common|1.1.28
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-batch|14.2.0
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-storage-queue|1.1.0
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-common|1.1.28
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-batch|1.0.0 #depends on azure-common1.1.4
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-batch|1.0.0 #depends on azure-common1.1.4
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
azure-storage-queue|1.1.0 #depends on azure-common>=1.1.28

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-storage-queue|1.1.0 #depends on azure-common>=1.1.28
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-batch|1.0.0 #depends on azure-common1.1.4
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
should be ignored
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-common|1.1.4
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
azure-common|1.1.28

2 changes: 1 addition & 1 deletion ext/scripts/tests/test_scripts_in_docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ echo Use DOCKER_TTY_OPTION="$DOCKER_TTY_OPTION"
# shellcheck disable=SC2086
docker run $DOCKER_TTY_OPTION -w /scripts/tests/install_scripts "$IMAGE_NAME" bash run_apt_tests.sh --no-dry-run
# shellcheck disable=SC2086
docker run $DOCKER_TTY_OPTION -w /scripts/tests/install_scripts "$IMAGE_NAME" bash run_pip_tests.sh --no-dry-run
RUN_PIP_TESTS_EXECUTOR="docker run $DOCKER_TTY_OPTION -w /scripts/tests/install_scripts \"$IMAGE_NAME\"" PATH_TO_INSTALL_SCRIPTS="/scripts/install_scripts" bash install_scripts/run_pip_tests.sh --no-dry-run
# shellcheck disable=SC2086
docker run $DOCKER_TTY_OPTION -w /scripts/tests/install_scripts "$IMAGE_NAME" bash run_r_remotes_tests.sh --no-dry-run
# shellcheck disable=SC2086
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps

RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions

RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --ancestor-pip-package-root-path /build_info/packages
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/lang
RUN /scripts/install_scripts/install_python3.10_pip.sh "pip == 21.3.1"

COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions --ancestor-pip-package-root-path /build_info/packages

ENV PYTHON3_PREFIX /usr
ENV PYTHON3_VERSION python3.10
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ COPY flavor_customization/packages/apt_get_packages /build_info/packages/flavor_
RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_customization/apt_get_packages --with-versions

COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary python3 --with-versions --allow-no-version
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary python3 --with-versions --allow-no-version --ancestor-pip-package-root-path /build_info/packages


##########################################################################
Expand Down