From 423f832e7da99e8c7f1ac7ef277a6895b1932cf5 Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Wed, 27 Mar 2024 11:33:41 -0700 Subject: [PATCH 1/4] Remove commented code. --- .github/workflows/CICD.yaml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/CICD.yaml b/.github/workflows/CICD.yaml index 655e9ea5..eab255fa 100644 --- a/.github/workflows/CICD.yaml +++ b/.github/workflows/CICD.yaml @@ -61,17 +61,6 @@ jobs: echo "cache_key_dependencies=$cache_key_dependencies" >> $GITHUB_OUTPUT echo "cache_key_run=$cache_key_run" >> $GITHUB_OUTPUT - #- name: Restore dependency cache - # if: ${{ success() && (github.event_name != 'workflow_dispatch' || inputs.use_dependency_cache == 'true') }} - # TODO figure out a way to get 3rd party dependencies cached without caching BL_Python ... maybe. - # uses: actions/cache/restore@v4 - # id: restore-dependency-cache - # with: - # path: | - # .github-venv - # node_modules - # key: ${{ steps.generate-cache-keys.outputs.cache_key_dependencies }} - - name: Set up Python ${{ env.PYTHON_VERSION }} #if: ${{ success() && (inputs.use_dependency_cache == 'false' || !steps.restore-dependency-cache.outputs.cache-hit) }} uses: actions/setup-python@v5 @@ -90,17 +79,6 @@ jobs: echo 'prefix=${{ github.workspace }}/node_modules' >> ~/.npmrc npm install -g pyright@`pyright --version | awk '{print $2}'` - # TODO see note above about 3rd party dependencies. - #- name: Save dependency cache - # if: ${{ success() && !steps.restore-dependency-cache.outputs.cache-hit }} - # uses: actions/cache/save@v4 - # id: save-dependency-cache - # with: - # path: | - # .github-venv - # node_modules - # key: ${{ steps.generate-cache-keys.outputs.cache_key_dependencies }} - - name: Save run cache uses: actions/cache/save@v4 id: save-run-cache From b5f682627e0d25d71042064b11cb8dfb5b644d4c Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Wed, 27 Mar 2024 12:57:56 -0700 Subject: [PATCH 2/4] Test whether using a consistent and specific Python version fixes venv issues during CICD. --- .github/workflows/CICD.yaml | 22 ++++++++++++---------- Makefile | 21 +-------------------- 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/.github/workflows/CICD.yaml b/.github/workflows/CICD.yaml index eab255fa..197cb336 100644 --- a/.github/workflows/CICD.yaml +++ b/.github/workflows/CICD.yaml @@ -41,6 +41,7 @@ jobs: outputs: cache-key-dependencies: ${{ steps.generate-cache-keys.outputs.cache_key }} cache-key-run: ${{ steps.generate-cache-keys.outputs.cache_key_run }} + python-version: ${{ steps.install-python.outputs.python-version }} env: PYTHON_VERSION: "3.10" @@ -61,12 +62,13 @@ jobs: echo "cache_key_dependencies=$cache_key_dependencies" >> $GITHUB_OUTPUT echo "cache_key_run=$cache_key_run" >> $GITHUB_OUTPUT - - name: Set up Python ${{ env.PYTHON_VERSION }} + - name: Set up Python ${{ steps.generate-cache-keys.outputs.python-version }} #if: ${{ success() && (inputs.use_dependency_cache == 'false' || !steps.restore-dependency-cache.outputs.cache-hit) }} uses: actions/setup-python@v5 - id: install_python + id: install-python with: python-version: ${{ env.PYTHON_VERSION }} + #run: echo "python-version=${{ steps.install-python.outputs.python-version }}" >> "$GITHUB_OUTPUT" - name: Install dependencies #if: ${{ success() && (inputs.use_dependency_cache == 'false' || !steps.restore-dependency-cache.outputs.cache-hit) }} @@ -93,14 +95,14 @@ jobs: - Checkout env: - PYTHON_VERSION: "3.10" + PYTHON_VERSION: ${{ needs.Checkout.outputs.python-version }} if: ${{( success() && !cancelled() ) }} steps: - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v5 - id: install_python + id: install-python with: python-version: ${{ env.PYTHON_VERSION }} @@ -128,14 +130,14 @@ jobs: - Checkout env: - PYTHON_VERSION: "3.10" + PYTHON_VERSION: ${{ needs.Checkout.outputs.python-version }} if: ${{( success() && !cancelled() ) }} steps: - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v5 - id: install_python + id: install-python with: python-version: ${{ env.PYTHON_VERSION }} @@ -176,7 +178,7 @@ jobs: - Checkout env: - PYTHON_VERSION: "3.10" + PYTHON_VERSION: ${{ needs.Checkout.outputs.python-version }} if: ${{( success() && !cancelled() ) }} @@ -186,7 +188,7 @@ jobs: steps: - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v5 - id: install_python + id: install-python with: python-version: ${{ env.PYTHON_VERSION }} @@ -228,14 +230,14 @@ jobs: - Checkout env: - PYTHON_VERSION: "3.10" + PYTHON_VERSION: ${{ needs.Checkout.outputs.python-version }} if: ${{( success() && !cancelled() ) }} steps: - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v5 - id: install_python + id: install-python with: python-version: ${{ env.PYTHON_VERSION }} diff --git a/Makefile b/Makefile index 34375bed..1b1a54e1 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,6 @@ endif ACTIVATE_VENV := . $(VENV)/bin/activate -PREFIX_VENV := PATH=$(CURDIR)/$(VENV)/bin:$$PATH REPORT_VENV_USAGE := echo '\nActivate your venv with `. $(VENV)/bin/activate`' PACKAGE_INSTALL_DIR := $(VENV)/lib/python*/site-packages/BL_Python @@ -82,11 +81,9 @@ _dev_build : _dev_configure else $(ACTIVATE_VENV) - $(PREFIX_VENV) \ pip install -e .[dev-dependencies] # By default, psycopg2 is not installed # but it should be for development - $(PREFIX_VENV) \ pip install -e src/database[postgres-binary] rm -rf $(PACKAGE_INSTALL_DIR) @@ -103,11 +100,9 @@ _cicd_build : _cicd_configure else $(ACTIVATE_VENV) - $(PREFIX_VENV) \ pip install .[dev-dependencies] # By default, psycopg2 is not installed # but it should be for CI/CD - $(PREFIX_VENV) \ pip install src/database[postgres-binary] fi @@ -120,10 +115,8 @@ $(PACKAGES) : BL_Python.%: src/%/pyproject.toml $(VENV) $(CONFIGURE_TARGET) $(PY $(ACTIVATE_VENV) if [ "$@" = "BL_Python.database" ]; then - $(PREFIX_VENV) \ pip install -e $(dir $<)[postgres-binary] else - $(PREFIX_VENV) \ pip install -e $(dir $<) fi @@ -137,19 +130,17 @@ SETUP_DEPENDENCIES=$(call dep_to_venv_path,toml/__init__.py) $(call dep_to_venv_ $(call dep_to_venv_path,toml/__init__.py): $(VENV) $(ACTIVATE_VENV) - $(PREFIX_VENV) pip install toml + pip install toml $(call dep_to_venv_path,typing_extensions.py): $(VENV) $(ACTIVATE_VENV) - $(PREFIX_VENV) \ pip install typing_extensions $(PACKAGE_PATHS) : $(VENV) $(SETUP_DEPENDENCIES) $(PYPROJECT_FILES) : $(VENV) $(SETUP_DEPENDENCIES) $(ACTIVATE_VENV) - $(PREFIX_VENV) \ REWRITE_DEPENDENCIES=$(REWRITE_DEPENDENCIES) \ GITHUB_REF=$(GITHUB_REF) \ GITHUB_WORKSPACE=$(GITHUB_WORKSPACE) \ @@ -164,20 +155,17 @@ $(VENV) : $(ACTIVATE_VENV) - $(PREFIX_VENV) \ pip install -U pip format-isort : $(VENV) $(BUILD_TARGET) $(ACTIVATE_VENV) - $(PREFIX_VENV) \ isort src format-ruff : $(VENV) $(BUILD_TARGET) $(ACTIVATE_VENV) - $(PREFIX_VENV) \ ruff format --preview --respect-gitignore format : $(VENV) $(BUILD_TARGET) format-isort format-ruff @@ -186,20 +174,17 @@ format : $(VENV) $(BUILD_TARGET) format-isort format-ruff test-isort : $(VENV) $(BUILD_TARGET) $(ACTIVATE_VENV) - $(PREFIX_VENV) \ isort --check-only src test-ruff : $(VENV) $(BUILD_TARGET) $(ACTIVATE_VENV) - $(PREFIX_VENV) \ ruff format --preview --respect-gitignore --check test-pyright : $(VENV) $(BUILD_TARGET) $(ACTIVATE_VENV) ifeq "$(PYRIGHT_MODE)" "pip" - $(PREFIX_VENV) \ pyright else ifeq "$(PYRIGHT_MODE)" "npm" @@ -215,7 +200,6 @@ test-pyright : $(VENV) $(BUILD_TARGET) test-bandit : $(VENV) $(BUILD_TARGET) $(ACTIVATE_VENV) - $(PREFIX_VENV) \ bandit -c pyproject.toml \ --format sarif \ --output $(BANDIT_REPORT) \ @@ -228,10 +212,8 @@ test-bandit : $(VENV) $(BUILD_TARGET) test-pytest : $(VENV) $(BUILD_TARGET) $(ACTIVATE_VENV) - $(PREFIX_VENV) \ pytest $(PYTEST_FLAGS) - $(PREFIX_VENV) \ coverage html -d coverage test : CMD_PREFIX=@ @@ -243,7 +225,6 @@ publish-all : REWRITE_DEPENDENCIES=false publish-all : reset $(VENV) $(ACTIVATE_VENV) - $(PREFIX_VENV) \ ./publish_all.sh $(PYPI_REPO) From fa9b7b313ca0bddc6c0ccd96b9a999948530c06c Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Wed, 27 Mar 2024 13:13:08 -0700 Subject: [PATCH 3/4] Remove alteration of Python symlink in venv. --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index 1b1a54e1..da715a7b 100644 --- a/Makefile +++ b/Makefile @@ -150,9 +150,6 @@ $(PYPROJECT_FILES) : $(VENV) $(SETUP_DEPENDENCIES) $(VENV) : test -d $(VENV) || env python$(PYTHON_VERSION) -m venv $(VENV) -# fix Python symlink that is wrong on GitHub Actions for some reason - ln -sf $(which python$(PYTHON_VERSION)) $(VENV)/bin/python$(PYTHON_VERSION) - $(ACTIVATE_VENV) pip install -U pip From b7705526ac17447b3844e395c0039773004f9e3d Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Wed, 27 Mar 2024 13:14:33 -0700 Subject: [PATCH 4/4] Update .github/workflows/CICD.yaml Co-authored-by: Nick Wiltsie --- .github/workflows/CICD.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/CICD.yaml b/.github/workflows/CICD.yaml index 197cb336..fd3addd4 100644 --- a/.github/workflows/CICD.yaml +++ b/.github/workflows/CICD.yaml @@ -68,7 +68,6 @@ jobs: id: install-python with: python-version: ${{ env.PYTHON_VERSION }} - #run: echo "python-version=${{ steps.install-python.outputs.python-version }}" >> "$GITHUB_OUTPUT" - name: Install dependencies #if: ${{ success() && (inputs.use_dependency_cache == 'false' || !steps.restore-dependency-cache.outputs.cache-hit) }}