From 399313e717d776a7ebf6727695de2baed4d2c0aa Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 12 Jul 2023 16:54:48 +0300 Subject: [PATCH 1/7] * Adds error message sense for validation * Removes beta tests --- .../packaging_warning_handler.py | 22 ++++++++++++++++--- .../tests/test_packaging_warning_handler.py | 6 ++--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/packaging_automation/packaging_warning_handler.py b/packaging_automation/packaging_warning_handler.py index c9b337a7..552500b3 100644 --- a/packaging_automation/packaging_warning_handler.py +++ b/packaging_automation/packaging_warning_handler.py @@ -28,6 +28,9 @@ def validate_output(output: str, ignore_file_path: str, package_type: PackageTyp print(f"Base ignore list:{base_ignore_list}") output_lines = output.splitlines() + + error_lines = return_lines_starts__with_error(output_lines) + warning_lines, package_type_specific_warning_lines = filter_warning_lines( output_lines, package_type ) @@ -50,17 +53,30 @@ def validate_output(output: str, ignore_file_path: str, package_type: PackageTyp if ( len(base_warnings_to_be_raised) > 0 or len(package_type_specific_warnings_to_be_raised) > 0 + or len(error_lines) > 0 ): - error_message = get_error_message( + error_message_for_warnings = get_error_message_for_warnings( base_warnings_to_be_raised, package_type_specific_warnings_to_be_raised, package_type, ) - print(f"Build output check failed. Error Message: \n{error_message}") + + if len(error_lines) > 0: + error_message_for_warnings += "\n\nErrors:\n" + for error_line in error_lines: + error_message_for_warnings += error_line + "\n" + + print(f"Build output check failed. Error Message: \n{error_message_for_warnings}") sys.exit(1) else: print("Build output check completed succesfully. No warnings") +def return_lines_starts__with_error(output_lines: List[str]) -> List[str]: + error_lines = [] + for line in output_lines: + if line.startswith("error"): + error_lines.append(line) + return error_lines def filter_warning_lines( output_lines: List[str], package_type: PackageType @@ -144,7 +160,7 @@ def get_warnings_to_be_raised( return warnings_to_be_raised -def get_error_message( +def get_error_message_for_warnings( base_warnings_to_be_raised: List[str], package_specific_warnings_to_be_raised: List[str], package_type: PackageType, diff --git a/packaging_automation/tests/test_packaging_warning_handler.py b/packaging_automation/tests/test_packaging_warning_handler.py index 4c4e9bd9..df3bd254 100644 --- a/packaging_automation/tests/test_packaging_warning_handler.py +++ b/packaging_automation/tests/test_packaging_warning_handler.py @@ -10,7 +10,7 @@ PackageType, filter_warning_lines, get_warnings_to_be_raised, - get_error_message, + get_error_message_for_warnings, validate_output, ) @@ -122,7 +122,7 @@ def test_get_error_message(): debian_warnings_to_be_raised = get_warnings_to_be_raised( debian_ignore_list, debian_warning_lines ) - error_message = get_error_message( + error_message = get_error_message_for_warnings( base_warnings_to_be_raised, debian_warnings_to_be_raised, PackageType.deb ) assert ( @@ -154,7 +154,7 @@ def test_get_error_message_empty_package_specific_errors(): debian_warnings_to_be_raised = get_warnings_to_be_raised( debian_ignore_list, debian_warning_lines ) - error_message = get_error_message( + error_message = get_error_message_for_warnings( base_warnings_to_be_raised, debian_warnings_to_be_raised, PackageType.deb ) assert error_message == "Warning lines:\nWarning: Unhandled\n" From 32e73428306db7d8e049d1b01fe992235ced1afe Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 12 Jul 2023 16:58:47 +0300 Subject: [PATCH 2/7] * Removes beta tests --- .../citus-package-all-platforms-beta-test.yml | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 .github/workflows/citus-package-all-platforms-beta-test.yml diff --git a/.github/workflows/citus-package-all-platforms-beta-test.yml b/.github/workflows/citus-package-all-platforms-beta-test.yml deleted file mode 100644 index 41b0cfe9..00000000 --- a/.github/workflows/citus-package-all-platforms-beta-test.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Citus package all platforms beta tests - -env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} - MICROSOFT_EMAIL: gindibay@microsoft.com - USER_NAME: Gurkan Indibay - MAIN_BRANCH: all-citus - PACKAGE_CLOUD_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} - PACKAGING_BRANCH_NAME: all-citus-unit-tests-beta - -on: - push: - branches: - - "**" - - workflow_dispatch: - -jobs: - unit_test_execution: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - platform: - - el/7 - - el/8 - - ol/7 - - ol/8 - - debian/stretch - - debian/buster - - debian/bullseye - - ubuntu/bionic - - ubuntu/focal - env: - PLATFORM: ${{ matrix.platform }} - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Install dependencies - run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources - - - name: Define git credentails - run: git config --global user.email "${MICROSOFT_EMAIL}"&& git config --global user.name "${USER_NAME}" - - - name: Install python requirements - run: python -m pip install -r packaging_automation/requirements.txt - - - name: Citus package tests - run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s From 72b0abe63355dcc86b7b77d897ef7389db9279cf Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 13 Jul 2023 09:42:22 +0300 Subject: [PATCH 3/7] Formatted with black --- packaging_automation/packaging_warning_handler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packaging_automation/packaging_warning_handler.py b/packaging_automation/packaging_warning_handler.py index 552500b3..61e17cea 100644 --- a/packaging_automation/packaging_warning_handler.py +++ b/packaging_automation/packaging_warning_handler.py @@ -66,11 +66,14 @@ def validate_output(output: str, ignore_file_path: str, package_type: PackageTyp for error_line in error_lines: error_message_for_warnings += error_line + "\n" - print(f"Build output check failed. Error Message: \n{error_message_for_warnings}") + print( + f"Build output check failed. Error Message: \n{error_message_for_warnings}" + ) sys.exit(1) else: print("Build output check completed succesfully. No warnings") + def return_lines_starts__with_error(output_lines: List[str]) -> List[str]: error_lines = [] for line in output_lines: @@ -78,6 +81,7 @@ def return_lines_starts__with_error(output_lines: List[str]) -> List[str]: error_lines.append(line) return error_lines + def filter_warning_lines( output_lines: List[str], package_type: PackageType ) -> Tuple[List[str], List[str]]: From e23e023ba4356490db6ad112e034a76c534edf6a Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 13 Jul 2023 10:28:42 +0300 Subject: [PATCH 4/7] Adds code for debug --- packaging_automation/packaging_warning_handler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packaging_automation/packaging_warning_handler.py b/packaging_automation/packaging_warning_handler.py index 61e17cea..baafa1bf 100644 --- a/packaging_automation/packaging_warning_handler.py +++ b/packaging_automation/packaging_warning_handler.py @@ -29,7 +29,7 @@ def validate_output(output: str, ignore_file_path: str, package_type: PackageTyp output_lines = output.splitlines() - error_lines = return_lines_starts__with_error(output_lines) + error_lines = return_lines_starts_with_error(output_lines) warning_lines, package_type_specific_warning_lines = filter_warning_lines( output_lines, package_type @@ -74,11 +74,12 @@ def validate_output(output: str, ignore_file_path: str, package_type: PackageTyp print("Build output check completed succesfully. No warnings") -def return_lines_starts__with_error(output_lines: List[str]) -> List[str]: +def return_lines_starts_with_error(output_lines: List[str]) -> List[str]: error_lines = [] for line in output_lines: if line.startswith("error"): error_lines.append(line) + print(f"Errors:{error_lines}") return error_lines From df52cd0f174285ae2b999102ac3e34503cbc02ef Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 13 Jul 2023 12:10:50 +0300 Subject: [PATCH 5/7] Fixes requirements error --- .../packaging_warning_handler.py | 2 +- packaging_automation/requirements.txt | 141 ++++++++++-------- 2 files changed, 76 insertions(+), 67 deletions(-) diff --git a/packaging_automation/packaging_warning_handler.py b/packaging_automation/packaging_warning_handler.py index baafa1bf..e2c927ae 100644 --- a/packaging_automation/packaging_warning_handler.py +++ b/packaging_automation/packaging_warning_handler.py @@ -62,7 +62,7 @@ def validate_output(output: str, ignore_file_path: str, package_type: PackageTyp ) if len(error_lines) > 0: - error_message_for_warnings += "\n\nErrors:\n" + error_message_for_warnings += "\n\nError Lines:\n" for error_line in error_lines: error_message_for_warnings += error_line + "\n" diff --git a/packaging_automation/requirements.txt b/packaging_automation/requirements.txt index 63368ad4..9881dae2 100644 --- a/packaging_automation/requirements.txt +++ b/packaging_automation/requirements.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --output-file=packaging_automation/requirements.txt --resolver=backtracking packaging_automation/requirements.in +# pip-compile --output-file=tools/packaging_automation/requirements.txt tools/packaging_automation/requirements.in # -anyio==3.6.2 +anyio==3.7.1 # via httpcore -astroid==2.15.4 +astroid==2.15.6 # via # pylint # pylint-celery @@ -14,11 +14,11 @@ astroid==2.15.4 # requirements-detector bandit==1.7.5 # via prospector -black==23.3.0 - # via -r packaging_automation/requirements.in +black==23.7.0 + # via -r tools/packaging_automation/requirements.in build==0.10.0 # via pyroma -certifi==2022.12.7 +certifi==2023.5.7 # via # httpcore # httpx @@ -29,30 +29,32 @@ cffi==1.15.1 # pynacl chardet==5.1.0 # via mbstrdecoder -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests -click==8.1.3 +click==8.1.4 # via black -cryptography==40.0.2 +cryptography==41.0.2 # via pyjwt -dataproperty==0.55.0 +dataproperty==1.0.0 # via # pytablewriter # tabledata -deprecated==1.2.13 +deprecated==1.2.14 # via pygithub dill==0.3.6 # via pylint -docker==6.0.1 - # via -r packaging_automation/requirements.in -docutils==0.19 +docker==6.1.3 + # via -r tools/packaging_automation/requirements.in +docutils==0.20.1 # via pyroma dodgy==0.2.1 # via prospector -dominate==2.7.0 +dominate==2.8.0 # via pytablewriter -exceptiongroup==1.1.1 - # via pytest +exceptiongroup==1.1.2 + # via + # anyio + # pytest flake8==5.0.4 # via # flake8-polyfill @@ -61,18 +63,18 @@ flake8-polyfill==1.0.2 # via pep8-naming gitdb==4.0.10 # via gitpython -gitpython==3.1.31 +gitpython==3.1.32 # via - # -r packaging_automation/requirements.in + # -r tools/packaging_automation/requirements.in # bandit # prospector greenlet==2.0.2 # via sqlalchemy h11==0.14.0 # via httpcore -httpcore==0.17.0 +httpcore==0.17.3 # via httpx -httpx==0.24.0 +httpx==0.24.1 # via pypistats idna==3.4 # via @@ -84,14 +86,14 @@ iniconfig==2.0.0 isort==5.12.0 # via pylint jinja2==3.1.2 - # via -r packaging_automation/requirements.in + # via -r tools/packaging_automation/requirements.in lazy-object-proxy==1.9.0 # via astroid -markdown-it-py==2.2.0 +markdown-it-py==3.0.0 # via rich -markupsafe==2.1.2 +markupsafe==2.1.3 # via jinja2 -mbstrdecoder==1.1.2 +mbstrdecoder==1.1.3 # via # dataproperty # pytablewriter @@ -103,12 +105,14 @@ mccabe==0.7.0 # pylint mdurl==0.1.2 # via markdown-it-py -mypy==1.2.0 +mypy==1.4.1 # via prospector mypy-extensions==1.0.0 # via # black # mypy +nodeenv==1.8.0 + # via pyright packaging==23.1 # via # black @@ -120,30 +124,30 @@ packaging==23.1 # requirements-detector # typepy parameters-validation==1.2.0 - # via -r packaging_automation/requirements.in + # via -r tools/packaging_automation/requirements.in pathlib2==2.3.7.post1 - # via -r packaging_automation/requirements.in + # via -r tools/packaging_automation/requirements.in pathspec==0.11.1 # via black -pathvalidate==2.5.2 +pathvalidate==3.0.0 # via pytablewriter pbr==5.11.1 # via stevedore pep8-naming==0.10.0 # via prospector -platformdirs==3.5.0 +platformdirs==3.8.1 # via # black # pylint # pypistats -pluggy==1.0.0 +pluggy==1.2.0 # via pytest -prettytable==3.7.0 +prettytable==3.8.0 # via pypistats -prospector[with_everything]==1.9.0 - # via -r packaging_automation/requirements.in +prospector[with_everything]==1.10.2 + # via -r tools/packaging_automation/requirements.in psycopg2-binary==2.9.6 - # via -r packaging_automation/requirements.in + # via -r tools/packaging_automation/requirements.in pycodestyle==2.9.1 # via # flake8 @@ -151,22 +155,22 @@ pycodestyle==2.9.1 pycparser==2.21 # via cffi pycurl==7.45.2 - # via -r packaging_automation/requirements.in + # via -r tools/packaging_automation/requirements.in pydocstyle==6.3.0 # via prospector pyflakes==2.5.0 # via # flake8 # prospector -pygithub==1.58.1 - # via -r packaging_automation/requirements.in +pygithub==1.59.0 + # via -r tools/packaging_automation/requirements.in pygments==2.15.1 # via # pyroma # rich -pyjwt[crypto]==2.6.0 +pyjwt[crypto]==2.7.0 # via pygithub -pylint==2.17.3 +pylint==2.17.4 # via # prospector # pylint-celery @@ -187,46 +191,48 @@ pylint-plugin-utils==0.7 # pylint-flask pynacl==1.5.0 # via pygithub -pypistats==1.3.0 - # via -r packaging_automation/requirements.in +pypistats==1.4.0 + # via -r tools/packaging_automation/requirements.in pyproject-hooks==1.0.0 # via build +pyright==1.1.317 + # via prospector pyroma==4.2 # via prospector -pytablewriter[html]==0.64.2 +pytablewriter[html]==1.0.0 # via pypistats -pytest==7.3.1 - # via -r packaging_automation/requirements.in +pytest==7.4.0 + # via -r tools/packaging_automation/requirements.in python-dateutil==2.8.2 # via # pypistats # typepy python-dotenv==1.0.0 - # via -r packaging_automation/requirements.in + # via -r tools/packaging_automation/requirements.in python-gnupg==0.5.0 - # via -r packaging_automation/requirements.in + # via -r tools/packaging_automation/requirements.in python-slugify==8.0.1 # via pypistats python-string-utils==1.0.0 - # via -r packaging_automation/requirements.in + # via -r tools/packaging_automation/requirements.in pytz==2023.3 # via typepy pyyaml==6.0 # via - # -r packaging_automation/requirements.in + # -r tools/packaging_automation/requirements.in # bandit # prospector -requests==2.29.0 +requests==2.31.0 # via - # -r packaging_automation/requirements.in + # -r tools/packaging_automation/requirements.in # docker # pygithub # pyroma -requirements-detector==1.2.1 +requirements-detector==1.2.2 # via prospector -rich==13.3.5 +rich==13.4.2 # via bandit -semver==3.0.0 +semver==3.0.1 # via requirements-detector setoptconf-tmp==0.3.1 # via prospector @@ -243,13 +249,13 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via pydocstyle -sqlalchemy==2.0.12 - # via -r packaging_automation/requirements.in -stevedore==5.0.0 +sqlalchemy==2.0.18 + # via -r tools/packaging_automation/requirements.in +stevedore==5.1.0 # via bandit tabledata==1.3.1 # via pytablewriter -tcolorpy==0.1.2 +tcolorpy==0.1.3 # via pytablewriter termcolor==2.3.0 # via pypistats @@ -270,31 +276,34 @@ tomli==2.0.1 # pytest tomlkit==0.11.8 # via pylint -trove-classifiers==2023.4.29 +trove-classifiers==2023.7.6 # via pyroma -typepy[datetime]==1.3.0 +typepy[datetime]==1.3.1 # via # dataproperty # pytablewriter # tabledata -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via # astroid + # black # mypy + # pylint + # rich # sqlalchemy -urllib3==1.26.15 +urllib3==2.0.3 # via - # -r packaging_automation/requirements.in + # -r tools/packaging_automation/requirements.in # docker # requests vulture==2.7 # via prospector wcwidth==0.2.6 # via prettytable -websocket-client==1.5.1 +websocket-client==1.6.1 # via docker wheel==0.40.0 - # via -r packaging_automation/requirements.in + # via -r tools/packaging_automation/requirements.in wrapt==1.15.0 # via # astroid From c710148db8979f9053dd85f0201ef8ad829b4e64 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 13 Jul 2023 13:13:03 +0300 Subject: [PATCH 6/7] Adds attrs to dependencies --- packaging_automation/requirements.in | 1 + packaging_automation/requirements.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/packaging_automation/requirements.in b/packaging_automation/requirements.in index a6a09aaf..a910332e 100644 --- a/packaging_automation/requirements.in +++ b/packaging_automation/requirements.in @@ -1,3 +1,4 @@ +attrs black docker GitPython diff --git a/packaging_automation/requirements.txt b/packaging_automation/requirements.txt index 9881dae2..bf734a95 100644 --- a/packaging_automation/requirements.txt +++ b/packaging_automation/requirements.txt @@ -12,6 +12,8 @@ astroid==2.15.6 # pylint-celery # pylint-flask # requirements-detector +attrs==23.1.0 + # via -r tools/packaging_automation/requirements.in bandit==1.7.5 # via prospector black==23.7.0 From 8e2328d50fba9e0bd0b48b6b076dac564e79987f Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 13 Jul 2023 14:25:50 +0300 Subject: [PATCH 7/7] Removes unnecessary print --- packaging_automation/packaging_warning_handler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packaging_automation/packaging_warning_handler.py b/packaging_automation/packaging_warning_handler.py index e2c927ae..6dc213f3 100644 --- a/packaging_automation/packaging_warning_handler.py +++ b/packaging_automation/packaging_warning_handler.py @@ -79,7 +79,6 @@ def return_lines_starts_with_error(output_lines: List[str]) -> List[str]: for line in output_lines: if line.startswith("error"): error_lines.append(line) - print(f"Errors:{error_lines}") return error_lines