Skip to content

Commit

Permalink
Added new tests
Browse files Browse the repository at this point in the history
  • Loading branch information
tomuben committed Jun 11, 2024
1 parent 24b8bf4 commit 695b9a9
Show file tree
Hide file tree
Showing 22 changed files with 141 additions and 89 deletions.
6 changes: 3 additions & 3 deletions ext/scripts/install_scripts/install_via_pip.pl
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/perl -W
#!/usr/bin/perl -w

=pod
Expand Down Expand Up @@ -48,7 +48,7 @@ =head1 SYNOPSIS
"ignore-installed" => \$ignore_installed,
"use-deprecated-legacy-resolver" => \$use_deprecated_legacy_resolver,
"python-binary=s" => \$python_binary,
"ancestor-pip-package-root-path" => \$ancestor_pip_package_root_path
"ancestor-pip-package-root-path=s" => \$ancestor_pip_package_root_path
) or package_mgmt_utils::print_usage_and_abort(__FILE__,"Error in command line arguments",2);
package_mgmt_utils::print_usage_and_abort(__FILE__,"",0) if $help;

Expand Down Expand Up @@ -115,7 +115,7 @@ sub replace_missing_version_for_urls{
my @all_files = package_mgmt_utils::merge_package_files($file, $ancestor_pip_package_root_path, 'python3_pip_packages');
$cmd =
package_mgmt_utils::generate_joined_and_transformed_string_from_files(
@all_files, $element_separator, $combining_template, \@templates, \@separators, \@rendered_line_transformation_functions);
$element_separator, $combining_template, \@templates, \@separators, \@rendered_line_transformation_functions, \@all_files);
}

if($with_versions){
Expand Down
42 changes: 28 additions & 14 deletions ext/scripts/package_mgmt_utils.pm
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use File::Find;

package package_mgmt_utils;

#use Data::Dumper;
use Data::Dumper;

sub generate_joined_and_transformed_string_from_file{
my ($file, $element_separator, $combining_template, $templates_ref, $separators_ref, $rendered_line_transformation_functions_ref) = @_;
Expand All @@ -17,17 +17,31 @@ sub generate_joined_and_transformed_string_from_file{


sub generate_joined_and_transformed_string_from_files {
my (@files, $element_separator, $combining_template, $templates_ref, $separators_ref, $rendered_line_transformation_functions_ref) = @_;
my ($element_separator, $combining_template, $templates_ref, $separators_ref, $rendered_line_transformation_functions_ref, $files_ref) = @_;
my %transformed_lines;
for (my $i=0; $i <= $#files; $i++) {
my @files = @$files_ref;
my $file = '';
foreach $file ( @files ) {
my @transformed_lines_for_current_file = generate_transformed_lines_for_templates(
$files[$i], $element_separator, $templates_ref, $rendered_line_transformation_functions_ref);
foreach (@transformed_lines_for_current_file) {
$transformed_lines{$_} = 1;
$file, $element_separator, $templates_ref, $rendered_line_transformation_functions_ref);
my $line_ref = '';
foreach $line_ref ( @transformed_lines_for_current_file ) {
my @lines = @$line_ref;
my $line = '';
foreach $line ( @lines) {
$transformed_lines{$line} = 1;
}
}
}
my @transformed_lines_arr = keys %transformed_lines;
my $final_string = generate_joined_string_from_lines(\@transformed_lines_arr, $combining_template, $separators_ref);

my @transformed_lines_arr = sort( keys %transformed_lines );

if( !@transformed_lines_arr ){
return "";
}

my @transformed_lines_arr_as_ref = ( \@transformed_lines_arr );
my $final_string = generate_joined_string_from_lines(\@transformed_lines_arr_as_ref, $combining_template, $separators_ref);
return $final_string;
}

Expand Down Expand Up @@ -239,15 +253,15 @@ sub print_usage_and_abort{


sub merge_package_files {
my ($base_file, $base_search_dir,$file_pattern) = @_;
my @result = $base_file;
my ($base_file, $base_search_dir, $file_pattern) = @_;
my @result = ($base_file);

find(sub {
if (-f and /\.$file_pattern$/) {
push(@result, $_);
File::Find::find(sub {
if (-f and /^$file_pattern$/) {
push(@result, $File::Find::name);
}
}, $base_search_dir);
return @result
return @result;
}

1;
153 changes: 89 additions & 64 deletions ext/scripts/tests/install_scripts/run_pip_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,88 +29,113 @@ function run_install() {
fi
}

function join_by {
local d=${1-} f=${2-}
if shift 2; then
printf %s "$f" "${@/#/$d}"
fi
}

function run_multiple_install() {
cmd=$(join_by ";" "${@}")
function run_install_must_fail() {
if [ -z "${RUN_PIP_TESTS_EXECUTOR-}" ]
then
eval "bash -e -c \"${cmd[@]}\""
if [ -z "${DRY_RUN_OPTION-}" ]
then
eval $@ && return 1 || return 0;
else
eval $@
fi
else
eval "$RUN_PIP_TESTS_EXECUTOR bash -e -c \"${cmd[@]}\""
if [ -z "${DRY_RUN_OPTION-}" ]
then
eval "$RUN_PIP_TESTS_EXECUTOR $PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl $@" && return 1 || return 0;
else
eval "$RUN_PIP_TESTS_EXECUTOR $PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl $@"
fi
fi
}


echo ./install_via_pip.pl with empty
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/empty_test_file --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" ""
#echo ./install_via_pip.pl with empty
#TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/empty_test_file --python-binary python3 "$DRY_RUN_OPTION")
#assert "$TEST_OUTPUT" ""
#echo
#
#echo ./install_via_pip.pl without versions
#TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --python-binary python3 "$DRY_RUN_OPTION")
#assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
#echo
#
#
#echo ./install_via_pip.pl without versions and --ignore-installed
#TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --ignore-installed --python-binary python3 "$DRY_RUN_OPTION")
#assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --ignore-installed --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
#echo
#
#
#echo ./install_via_pip.pl without versions and --use-deprecated-legacy-resolver
#TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --use-deprecated-legacy-resolver --python-binary python3 "$DRY_RUN_OPTION")
#assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --use-deprecated=legacy-resolver --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
#echo
#
#
#echo ./install_via_pip.pl with versions, without allow-no-version
#TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --with-versions --python-binary python3 "$DRY_RUN_OPTION")
#assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'"
#echo
#
#
#echo ./install_via_pip.pl with versions, with allow-no-version, all versions specified
#TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --with-versions --allow-no-version --python-binary python3 "$DRY_RUN_OPTION")
#assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'"
#echo
#
#
#echo ./install_via_pip.pl with versions, with allow-no-version, some versions missing
#TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/some_missing_versions --with-versions --allow-no-version --python-binary python3 "$DRY_RUN_OPTION")
#assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
#echo
#
#
#echo ./install_via_pip.pl with versions, with allow-no-version-for-urls, file with urls
#TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/with_urls --with-versions --allow-no-version-for-urls --python-binary python3 "$DRY_RUN_OPTION")
#assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.27.1' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
#echo
#
#
#echo ./install_via_pip.pl with versions, with allow-no-version-for-urls, file with urls and some missing versions
#run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/with_urls_some_missing_versions --with-versions --allow-no-version-for-urls --python-binary python3 "$DRY_RUN_OPTION" || echo PASSED
#echo
#
#
#echo ./install_via_pip.pl with pip version syntax
#TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/pip_version_syntax --python-binary python3 "$DRY_RUN_OPTION")
#assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests>=2.21.0' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
#echo
#
#
echo ./install_via_pip.pl installing a package twice with different versions must fail
TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/same_pkg/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/same_pkg/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.28' 'azure-common==1.1.4'"
echo

echo ./install_via_pip.pl without versions
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
echo


echo ./install_via_pip.pl without versions and --ignore-installed
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --ignore-installed --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --ignore-installed --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
echo


echo ./install_via_pip.pl without versions and --use-deprecated-legacy-resolver
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --use-deprecated-legacy-resolver --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --use-deprecated=legacy-resolver --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
echo


echo ./install_via_pip.pl with versions, without allow-no-version
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --with-versions --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'"
echo


echo ./install_via_pip.pl with versions, with allow-no-version, all versions specified
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --with-versions --allow-no-version --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'"
echo


echo ./install_via_pip.pl with versions, with allow-no-version, some versions missing
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/some_missing_versions --with-versions --allow-no-version --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
echo ./install_via_pip.pl installing with ancestors but all empty
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/empty/step2 --ancestor-pip-package-root-path test_files/pip/empty/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" ""
echo


echo ./install_via_pip.pl with versions, with allow-no-version-for-urls, file with urls
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/with_urls --with-versions --allow-no-version-for-urls --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.27.1' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
echo ./install_via_pip.pl installing with ancestors and correct dependency
TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/no_version_conflict/dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/no_version_conflict/dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-batch==1.0.0' 'azure-common==1.1.4'"
echo


echo ./install_via_pip.pl with versions, with allow-no-version-for-urls, file with urls and some missing versions
"$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/with_urls_some_missing_versions --with-versions --allow-no-version-for-urls --python-binary python3 "$DRY_RUN_OPTION" || echo PASSED
echo ./install_via_pip.pl installing with ancestors and dependency with wrong version must fail
TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-batch==1.0.0' 'azure-common==1.1.28'"
echo


echo ./install_via_pip.pl with pip version syntax
TEST_OUTPUT=$("$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/pip_version_syntax --python-binary python3 "$DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests>=2.21.0' 'git+http://github.com/exasol/[email protected]#egg=exasol-bucketfs-utils-python'"
echo ./install_via_pip.pl installing with ancestors and package which has a dependency to an older package must fail
TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/other_package_with_older_dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/other_package_with_older_dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-batch==1.0.0' 'azure-storage-queue==1.1.0'"
echo


echo ./install_via_pip.pl installing a package twice with different versions
run_multiple_install \
"$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/step1 --python-binary python3 --with-versions $DRY_RUN_OPTION" \
"$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/step2 --python-binary python3 --with-versions $DRY_RUN_OPTION"
echo ./install_via_pip.pl installing with ancestors and package which has a dependency to a newer package must fail
TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/other_package_with_newer_dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/other_package_with_newer_dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION")
assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-batch==1.0.0' 'azure-storage-queue==1.1.0'"
echo


check_for_failed_tests
echo "All pip tests passed"
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-common|1.1.4
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
azure-batch|1.0.0 #depends on azure-common1.1.4

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-common|1.1.28
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-batch|1.0.0 #depends on azure-common1.1.4
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-batch|1.0.0 #depends on azure-common1.1.4
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
azure-storage-queue|1.1.0 #depends on azure-common>=1.1.28

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-storage-queue|1.1.0 #depends on azure-common>=1.1.28
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-batch|1.0.0 #depends on azure-common1.1.4
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
should be ignored
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
azure-common|1.1.4
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
azure-common|1.1.28

This file was deleted.

This file was deleted.

6 changes: 3 additions & 3 deletions ext/scripts/tests/test_scripts_in_docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ fi
echo Use DOCKER_TTY_OPTION="$DOCKER_TTY_OPTION"

# shellcheck disable=SC2086
docker run $DOCKER_TTY_OPTION -w /scripts/tests/install_scripts "$IMAGE_NAME" bash run_apt_tests.sh --no-dry-run
#docker run $DOCKER_TTY_OPTION -w /scripts/tests/install_scripts "$IMAGE_NAME" bash run_apt_tests.sh --no-dry-run
# shellcheck disable=SC2086
RUN_PIP_TESTS_EXECUTOR="docker run $DOCKER_TTY_OPTION -w /scripts/tests/install_scripts \"$IMAGE_NAME\"" PATH_TO_INSTALL_SCRIPTS="/scripts/install_scripts" bash install_scripts/run_pip_tests.sh --no-dry-run
# shellcheck disable=SC2086
docker run $DOCKER_TTY_OPTION -w /scripts/tests/install_scripts "$IMAGE_NAME" bash run_r_remotes_tests.sh --no-dry-run
#docker run $DOCKER_TTY_OPTION -w /scripts/tests/install_scripts "$IMAGE_NAME" bash run_r_remotes_tests.sh --no-dry-run
# shellcheck disable=SC2086
docker run $DOCKER_TTY_OPTION -w /scripts/tests/install_scripts "$IMAGE_NAME" bash run_ppa_tests.sh --no-dry-run
#docker run $DOCKER_TTY_OPTION -w /scripts/tests/install_scripts "$IMAGE_NAME" bash run_ppa_tests.sh --no-dry-run
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps

RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions

RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --ancestor-pip-package-root-path /build_info/packages
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/lang
RUN /scripts/install_scripts/install_python3.10_pip.sh "pip == 21.3.1"

COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions
RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions --ancestor-pip-package-root-path /build_info/packages

ENV PYTHON3_PREFIX /usr
ENV PYTHON3_VERSION python3.10
Expand Down
Loading

0 comments on commit 695b9a9

Please sign in to comment.