From e5d81518a2331dadc1e72355b2be96e1900551a9 Mon Sep 17 00:00:00 2001 From: daquinteroflex Date: Wed, 5 Nov 2025 20:07:37 +0100 Subject: [PATCH 1/2] introduce new deploy step --- .../workflows/tidy3d-python-client-deploy.yml | 316 ++++++++++++++++++ .../tidy3d-python-client-release.yml | 127 ++++--- 2 files changed, 400 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/tidy3d-python-client-deploy.yml diff --git a/.github/workflows/tidy3d-python-client-deploy.yml b/.github/workflows/tidy3d-python-client-deploy.yml new file mode 100644 index 0000000000..8b4848abf8 --- /dev/null +++ b/.github/workflows/tidy3d-python-client-deploy.yml @@ -0,0 +1,316 @@ +name: "public/tidy3d/python-client-deploy" + +on: + workflow_dispatch: + inputs: + release_tag: + description: 'Release tag to deploy (e.g., v2.10.0, v2.10.0rc1)' + required: true + type: string + + deploy_testpypi: + description: 'Deploy to TestPyPI (recommended first step)' + type: boolean + default: true + + deploy_pypi: + description: 'Deploy to production PyPI' + type: boolean + default: false + + deploy_aws: + description: 'Deploy to AWS CodeArtifact' + type: boolean + default: false + + workflow_call: + inputs: + release_tag: + description: 'Release tag to deploy' + required: true + type: string + deploy_testpypi: + type: boolean + default: false + deploy_pypi: + type: boolean + default: false + deploy_aws: + type: boolean + default: false + +permissions: + contents: read + id-token: write + +env: + AWS_REGION: "us-east-1" + +jobs: + validate-inputs: + name: validate-deployment-inputs + runs-on: ubuntu-latest + outputs: + release_tag: ${{ env.RELEASE_TAG }} + deploy_testpypi: ${{ env.DEPLOY_TESTPYPI }} + deploy_pypi: ${{ env.DEPLOY_PYPI }} + deploy_aws: ${{ env.DEPLOY_AWS }} + env: + RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || inputs.release_tag }} + DEPLOY_TESTPYPI: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_testpypi || inputs.deploy_testpypi }} + DEPLOY_PYPI: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_pypi || inputs.deploy_pypi }} + DEPLOY_AWS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_aws || inputs.deploy_aws }} + steps: + - name: validate-inputs + run: | + set -e + echo "=== Deployment Configuration ===" + echo "Release tag: $RELEASE_TAG" + echo "Deploy to TestPyPI: $DEPLOY_TESTPYPI" + echo "Deploy to PyPI: $DEPLOY_PYPI" + echo "Deploy to AWS CodeArtifact: $DEPLOY_AWS" + echo "" + + # Validate at least one target is selected + if [[ "$DEPLOY_TESTPYPI" != "true" && "$DEPLOY_PYPI" != "true" && "$DEPLOY_AWS" != "true" ]]; then + echo "? Error: At least one deployment target must be selected" + exit 1 + fi + + # Validate tag format + TAG_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]+)?$' + if [[ ! "$RELEASE_TAG" =~ $TAG_REGEX ]]; then + echo "?? Warning: Tag format doesn't match standard pattern v{major}.{minor}.{patch}[rc{num}]" + echo " Tag: $RELEASE_TAG" + echo " Continuing anyway..." + fi + + echo "? Validation passed" + + build-package: + name: build-distribution-package + needs: validate-inputs + runs-on: ubuntu-latest + steps: + - name: checkout-tag + uses: actions/checkout@v4 + with: + ref: ${{ needs.validate-inputs.outputs.release_tag }} + persist-credentials: false + + - name: setup-python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: install-poetry + run: | + python -m pip install --upgrade pip + python -m pip install poetry + + - name: build-package + run: | + echo "Building package from tag ${{ needs.validate-inputs.outputs.release_tag }}..." + poetry build + echo "" + echo "Build artifacts:" + ls -lh dist/ + echo "" + echo "? Package built successfully" + + - name: upload-artifacts + uses: actions/upload-artifact@v4 + with: + name: dist-${{ needs.validate-inputs.outputs.release_tag }} + path: dist/ + retention-days: 7 + + deploy-testpypi: + name: deploy-to-testpypi + needs: [validate-inputs, build-package] + if: needs.validate-inputs.outputs.deploy_testpypi == 'true' + runs-on: ubuntu-latest + steps: + - name: download-artifacts + uses: actions/download-artifact@v4 + with: + name: dist-${{ needs.validate-inputs.outputs.release_tag }} + path: dist/ + + - name: setup-python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: install-twine + run: | + python -m pip install --upgrade pip + python -m pip install twine + + - name: publish-to-testpypi + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} + run: | + echo "?? Publishing to TestPyPI..." + python -m twine upload \ + --repository-url https://test.pypi.org/legacy/ \ + --verbose \ + dist/* + echo "" + echo "? Successfully published to TestPyPI" + echo " View at: https://test.pypi.org/project/tidy3d/" + + deploy-pypi: + name: deploy-to-pypi + needs: [validate-inputs, build-package, deploy-testpypi] + # Run after TestPyPI succeeds (or is skipped if not selected) + if: | + always() && + needs.validate-inputs.outputs.deploy_pypi == 'true' && + needs.build-package.result == 'success' && + (needs.deploy-testpypi.result == 'success' || needs.deploy-testpypi.result == 'skipped') + runs-on: ubuntu-latest + steps: + - name: download-artifacts + uses: actions/download-artifact@v4 + with: + name: dist-${{ needs.validate-inputs.outputs.release_tag }} + path: dist/ + + - name: setup-python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: install-twine + run: | + python -m pip install --upgrade pip + python -m pip install twine + + - name: publish-to-pypi + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: | + echo "?? Publishing to production PyPI..." + python -m twine upload \ + --repository pypi \ + --verbose \ + dist/* + echo "" + echo "? Successfully published to PyPI" + echo " View at: https://pypi.org/project/tidy3d/" + + deploy-aws-codeartifact: + name: deploy-to-aws-codeartifact + needs: [validate-inputs, build-package, deploy-testpypi] + # Run in parallel with PyPI after TestPyPI succeeds + if: | + always() && + needs.validate-inputs.outputs.deploy_aws == 'true' && + needs.build-package.result == 'success' && + (needs.deploy-testpypi.result == 'success' || needs.deploy-testpypi.result == 'skipped') + runs-on: ubuntu-latest + steps: + - name: download-artifacts + uses: actions/download-artifact@v4 + with: + name: dist-${{ needs.validate-inputs.outputs.release_tag }} + path: dist/ + + - name: setup-python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: install-twine + run: | + python -m pip install --upgrade pip + python -m pip install twine + + - name: configure-aws-credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.AWS_REGION }} + aws-access-key-id: ${{ secrets.AWS_CODEARTIFACT_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.AWS_CODEARTIFACT_ACCESS_SECRET }} + + - name: configure-codeartifact-package + run: | + echo "?? Configuring CodeArtifact package settings..." + aws codeartifact put-package-origin-configuration \ + --domain flexcompute \ + --repository pypi-releases \ + --format pypi \ + --package tidy3d \ + --restrictions '{"publish":"ALLOW", "upstream":"BLOCK"}' + + - name: get-codeartifact-token + run: | + echo "?? Getting CodeArtifact authorization token..." + CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \ + --domain flexcompute \ + --domain-owner 625554095313 \ + --region us-east-1 \ + --query authorizationToken \ + --output text) + echo "CODEARTIFACT_AUTH_TOKEN=${CODEARTIFACT_AUTH_TOKEN}" >> $GITHUB_ENV + + - name: publish-to-codeartifact + env: + TWINE_PASSWORD: ${{ env.CODEARTIFACT_AUTH_TOKEN }} + run: | + echo "?? Publishing to AWS CodeArtifact..." + python -m twine upload \ + --repository-url https://flexcompute-625554095313.d.codeartifact.us-east-1.amazonaws.com/pypi/pypi-releases/ \ + --verbose \ + -u aws \ + -p "$TWINE_PASSWORD" \ + dist/* + echo "" + echo "? Successfully published to AWS CodeArtifact" + + deployment-summary: + name: deployment-summary + needs: [validate-inputs, build-package, deploy-testpypi, deploy-pypi, deploy-aws-codeartifact] + if: always() + runs-on: ubuntu-latest + steps: + - name: generate-summary + run: | + echo "=== Deployment Summary ===" + echo "Release Tag: ${{ needs.validate-inputs.outputs.release_tag }}" + echo "" + echo "Build Package: ${{ needs.build-package.result }}" + echo "TestPyPI: ${{ needs.deploy-testpypi.result }}" + echo "PyPI: ${{ needs.deploy-pypi.result }}" + echo "AWS CodeArtifact: ${{ needs.deploy-aws-codeartifact.result }}" + echo "" + + # Check for failures + if [[ "${{ needs.build-package.result }}" == "failure" ]]; then + echo "? Build failed" + exit 1 + fi + + # Check if any selected deployment failed + failed=false + if [[ "${{ needs.validate-inputs.outputs.deploy_testpypi }}" == "true" && "${{ needs.deploy-testpypi.result }}" == "failure" ]]; then + echo "? TestPyPI deployment failed" + failed=true + fi + if [[ "${{ needs.validate-inputs.outputs.deploy_pypi }}" == "true" && "${{ needs.deploy-pypi.result }}" == "failure" ]]; then + echo "? PyPI deployment failed" + failed=true + fi + if [[ "${{ needs.validate-inputs.outputs.deploy_aws }}" == "true" && "${{ needs.deploy-aws-codeartifact.result }}" == "failure" ]]; then + echo "? AWS CodeArtifact deployment failed" + failed=true + fi + + if [[ "$failed" == "true" ]]; then + exit 1 + fi + + echo "? All selected deployments completed successfully" diff --git a/.github/workflows/tidy3d-python-client-release.yml b/.github/workflows/tidy3d-python-client-release.yml index 77e126c4ed..6e8680e861 100644 --- a/.github/workflows/tidy3d-python-client-release.yml +++ b/.github/workflows/tidy3d-python-client-release.yml @@ -45,6 +45,21 @@ on: description: 'Run submodule tests' type: boolean default: true + + deploy_testpypi: + description: 'Deploy to TestPyPI (recommended for testing)' + type: boolean + default: false + + deploy_pypi: + description: 'Deploy to production PyPI' + type: boolean + default: false + + deploy_aws: + description: 'Deploy to AWS CodeArtifact' + type: boolean + default: false workflow_call: inputs: @@ -79,6 +94,21 @@ on: description: 'Run submodule tests' type: boolean default: true + + deploy_testpypi: + description: 'Deploy to TestPyPI' + type: boolean + default: false + + deploy_pypi: + description: 'Deploy to production PyPI' + type: boolean + default: false + + deploy_aws: + description: 'Deploy to AWS CodeArtifact' + type: boolean + default: false outputs: workflow_success: @@ -104,7 +134,9 @@ jobs: run_cli_tests: ${{ steps.determine-workflow-steps.outputs.run_cli_tests }} run_submodule_tests: ${{ steps.determine-workflow-steps.outputs.run_submodule_tests }} deploy_github_release: ${{ steps.determine-workflow-steps.outputs.deploy_github_release }} + deploy_testpypi: ${{ steps.determine-workflow-steps.outputs.deploy_testpypi }} deploy_pypi: ${{ steps.determine-workflow-steps.outputs.deploy_pypi }} + deploy_aws: ${{ steps.determine-workflow-steps.outputs.deploy_aws }} sync_readthedocs: ${{ steps.determine-workflow-steps.outputs.sync_readthedocs }} sync_branches: ${{ steps.determine-workflow-steps.outputs.sync_branches }} env: @@ -114,6 +146,9 @@ jobs: CLIENT_TESTS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.client_tests || inputs.client_tests }} CLI_TESTS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.cli_tests || inputs.cli_tests }} SUBMODULE_TESTS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.submodule_tests || inputs.submodule_tests }} + DEPLOY_TESTPYPI: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_testpypi || inputs.deploy_testpypi }} + DEPLOY_PYPI: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_pypi || inputs.deploy_pypi }} + DEPLOY_AWS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_aws || inputs.deploy_aws }} steps: - name: validate-tag-format run: | @@ -153,6 +188,9 @@ jobs: echo "CLIENT_TESTS: $CLIENT_TESTS" echo "CLI_TESTS: $CLI_TESTS" echo "SUBMODULE_TESTS: $SUBMODULE_TESTS" + echo "DEPLOY_TESTPYPI: $DEPLOY_TESTPYPI" + echo "DEPLOY_PYPI: $DEPLOY_PYPI" + echo "DEPLOY_AWS: $DEPLOY_AWS" echo "" # ============================================ @@ -272,7 +310,9 @@ jobs: # PART 4: DEPLOYMENT CONTROL # ============================================ deploy_github_release=false + deploy_testpypi=false deploy_pypi=false + deploy_aws=false sync_readthedocs=false sync_branches=false @@ -281,16 +321,39 @@ jobs: deploy_github_release=true sync_readthedocs=true - # Only publish to PyPI and sync branches on final releases - if [[ "$RELEASE_TYPE" == "final" ]]; then - deploy_pypi=true + # Deployment target logic: + # 1. If any deployment checkbox is explicitly set, use those + # 2. Otherwise, use automatic defaults based on release_type + + if [[ "$DEPLOY_TESTPYPI" == "true" || "$DEPLOY_PYPI" == "true" || "$DEPLOY_AWS" == "true" ]]; then + # Manual override: use checkbox selections + echo "Using manual deployment target selections" + deploy_testpypi=$DEPLOY_TESTPYPI + deploy_pypi=$DEPLOY_PYPI + deploy_aws=$DEPLOY_AWS + else + # Automatic defaults based on release_type + echo "Using automatic deployment defaults for release_type: $RELEASE_TYPE" + if [[ "$RELEASE_TYPE" == "final" ]]; then + # Final releases: deploy to production PyPI + deploy_pypi=true + elif [[ "$RELEASE_TYPE" == "draft" ]]; then + # Draft releases: no deployment by default + echo "Draft release - no automatic deployments" + fi + fi + + # Sync branches on final releases if deploying to PyPI + if [[ "$RELEASE_TYPE" == "final" && "$deploy_pypi" == "true" ]]; then sync_branches=true fi fi echo "=== Deployment Control ===" echo "deploy_github_release: $deploy_github_release" + echo "deploy_testpypi: $deploy_testpypi" echo "deploy_pypi: $deploy_pypi" + echo "deploy_aws: $deploy_aws" echo "sync_readthedocs: $sync_readthedocs" echo "sync_branches: $sync_branches" echo "" @@ -307,7 +370,9 @@ jobs: echo "run_cli_tests=$run_cli_tests" >> $GITHUB_OUTPUT echo "run_submodule_tests=$run_submodule_tests" >> $GITHUB_OUTPUT echo "deploy_github_release=$deploy_github_release" >> $GITHUB_OUTPUT + echo "deploy_testpypi=$deploy_testpypi" >> $GITHUB_OUTPUT echo "deploy_pypi=$deploy_pypi" >> $GITHUB_OUTPUT + echo "deploy_aws=$deploy_aws" >> $GITHUB_OUTPUT echo "sync_readthedocs=$sync_readthedocs" >> $GITHUB_OUTPUT echo "sync_branches=$sync_branches" >> $GITHUB_OUTPUT @@ -441,47 +506,23 @@ jobs: # env: # GITHUB_TOKEN: ${{ secrets.GH_PAT }} - pypi-release: - name: publish-to-pypi + deploy-packages: + name: deploy-to-package-repositories needs: [determine-workflow-scope, compile-tests-results] if: | always() && (needs.compile-tests-results.outputs.proceed_deploy == 'true' || needs.compile-tests-results.result == 'skipped') && - needs.determine-workflow-scope.outputs.deploy_pypi == 'true' - runs-on: ubuntu-latest - env: - RELEASE_TAG: ${{ needs.determine-workflow-scope.outputs.release_tag }} - steps: - - run: | - echo "steps" - - # - name: checkout-tag - # uses: actions/checkout@v4 - # with: - # ref: ${{ env.RELEASE_TAG }} - # persist-credentials: false - - # - name: setup-python - # uses: actions/setup-python@v5 - # with: - # python-version: '3.10' - - # - name: install-build-tools - # run: | - # python -m pip install --upgrade pip - # python -m pip install setuptools wheel twine build - - # - name: build-package - # run: | - # echo "Building package from tag $RELEASE_TAG..." - # python -m build - # ls -lh dist/ - - # - name: publish-to-pypi - # env: - # TWINE_USERNAME: __token__ - # TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - # run: | # zizmor: ignore[use-trusted-publishing] - # echo "Publishing to PyPI..." - # python -m twine upload --repository pypi dist/* - # echo "Published to PyPI" + needs.determine-workflow-scope.outputs.run_deploy == 'true' && + (needs.determine-workflow-scope.outputs.deploy_testpypi == 'true' || + needs.determine-workflow-scope.outputs.deploy_pypi == 'true' || + needs.determine-workflow-scope.outputs.deploy_aws == 'true') + uses: ./.github/workflows/tidy3d-python-client-deploy.yml + permissions: + contents: read + id-token: write + with: + release_tag: ${{ needs.determine-workflow-scope.outputs.release_tag }} + deploy_testpypi: ${{ needs.determine-workflow-scope.outputs.deploy_testpypi == 'true' }} + deploy_pypi: ${{ needs.determine-workflow-scope.outputs.deploy_pypi == 'true' }} + deploy_aws: ${{ needs.determine-workflow-scope.outputs.deploy_aws == 'true' }} + secrets: inherit From 915ba27220f5557fec3bd5dbc4fcb5d3eae03934 Mon Sep 17 00:00:00 2001 From: daquinteroflex Date: Fri, 7 Nov 2025 16:07:03 +0100 Subject: [PATCH 2/2] a --- .../workflows/tidy3d-python-client-deploy.yml | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tidy3d-python-client-deploy.yml b/.github/workflows/tidy3d-python-client-deploy.yml index 8b4848abf8..82d4f8750b 100644 --- a/.github/workflows/tidy3d-python-client-deploy.yml +++ b/.github/workflows/tidy3d-python-client-deploy.yml @@ -73,19 +73,19 @@ jobs: # Validate at least one target is selected if [[ "$DEPLOY_TESTPYPI" != "true" && "$DEPLOY_PYPI" != "true" && "$DEPLOY_AWS" != "true" ]]; then - echo "? Error: At least one deployment target must be selected" + echo "Error: At least one deployment target must be selected" exit 1 fi # Validate tag format TAG_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]+)?$' if [[ ! "$RELEASE_TAG" =~ $TAG_REGEX ]]; then - echo "?? Warning: Tag format doesn't match standard pattern v{major}.{minor}.{patch}[rc{num}]" + echo " Warning: Tag format doesn't match standard pattern v{major}.{minor}.{patch}[rc{num}]" echo " Tag: $RELEASE_TAG" echo " Continuing anyway..." fi - echo "? Validation passed" + echo "Validation passed" build-package: name: build-distribution-package @@ -116,7 +116,7 @@ jobs: echo "Build artifacts:" ls -lh dist/ echo "" - echo "? Package built successfully" + echo "Package built successfully" - name: upload-artifacts uses: actions/upload-artifact@v4 @@ -152,14 +152,14 @@ jobs: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} run: | - echo "?? Publishing to TestPyPI..." + echo "Publishing to TestPyPI..." python -m twine upload \ --repository-url https://test.pypi.org/legacy/ \ --verbose \ dist/* echo "" - echo "? Successfully published to TestPyPI" - echo " View at: https://test.pypi.org/project/tidy3d/" + echo "Successfully published to TestPyPI" + echo "View at: https://test.pypi.org/project/tidy3d/" deploy-pypi: name: deploy-to-pypi @@ -193,13 +193,13 @@ jobs: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | - echo "?? Publishing to production PyPI..." + echo "Publishing to production PyPI..." python -m twine upload \ --repository pypi \ --verbose \ dist/* echo "" - echo "? Successfully published to PyPI" + echo "Successfully published to PyPI" echo " View at: https://pypi.org/project/tidy3d/" deploy-aws-codeartifact: @@ -238,7 +238,7 @@ jobs: - name: configure-codeartifact-package run: | - echo "?? Configuring CodeArtifact package settings..." + echo "Configuring CodeArtifact package settings..." aws codeartifact put-package-origin-configuration \ --domain flexcompute \ --repository pypi-releases \ @@ -248,7 +248,7 @@ jobs: - name: get-codeartifact-token run: | - echo "?? Getting CodeArtifact authorization token..." + echo "Getting CodeArtifact authorization token..." CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \ --domain flexcompute \ --domain-owner 625554095313 \ @@ -261,7 +261,7 @@ jobs: env: TWINE_PASSWORD: ${{ env.CODEARTIFACT_AUTH_TOKEN }} run: | - echo "?? Publishing to AWS CodeArtifact..." + echo "Publishing to AWS CodeArtifact..." python -m twine upload \ --repository-url https://flexcompute-625554095313.d.codeartifact.us-east-1.amazonaws.com/pypi/pypi-releases/ \ --verbose \ @@ -269,7 +269,7 @@ jobs: -p "$TWINE_PASSWORD" \ dist/* echo "" - echo "? Successfully published to AWS CodeArtifact" + echo "Successfully published to AWS CodeArtifact" deployment-summary: name: deployment-summary @@ -290,22 +290,22 @@ jobs: # Check for failures if [[ "${{ needs.build-package.result }}" == "failure" ]]; then - echo "? Build failed" + echo "Build failed" exit 1 fi # Check if any selected deployment failed failed=false if [[ "${{ needs.validate-inputs.outputs.deploy_testpypi }}" == "true" && "${{ needs.deploy-testpypi.result }}" == "failure" ]]; then - echo "? TestPyPI deployment failed" + echo "TestPyPI deployment failed" failed=true fi if [[ "${{ needs.validate-inputs.outputs.deploy_pypi }}" == "true" && "${{ needs.deploy-pypi.result }}" == "failure" ]]; then - echo "? PyPI deployment failed" + echo "PyPI deployment failed" failed=true fi if [[ "${{ needs.validate-inputs.outputs.deploy_aws }}" == "true" && "${{ needs.deploy-aws-codeartifact.result }}" == "failure" ]]; then - echo "? AWS CodeArtifact deployment failed" + echo "AWS CodeArtifact deployment failed" failed=true fi @@ -313,4 +313,4 @@ jobs: exit 1 fi - echo "? All selected deployments completed successfully" + echo "All selected deployments completed successfully"