diff --git a/.github/workflows/build-dask-image.yml b/.github/workflows/build-dask-image.yml index 215b8ca56b..099d804427 100644 --- a/.github/workflows/build-dask-image.yml +++ b/.github/workflows/build-dask-image.yml @@ -3,7 +3,7 @@ name: Build and push dask image on: push: branches: - - 'main' + - main paths: - '.github/workflows/build-dask-image.yml' - 'images/dask/**' @@ -16,29 +16,52 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} +env: + PYTHON_VERSION: '3.11' + POETRY_VERSION: '2.0.1' + jobs: - build_push: - name: Package docker image + docker: + name: Push docker image runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + id-token: write steps: - - uses: actions/checkout@v2 - - uses: docker/login-action@v1 + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Repository + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/setup-python@v4 + + - name: Setup Python + uses: actions/setup-python@v5 with: - python-version: '3.10' - - run: curl -sSL https://install.python-poetry.org | python - - # from https://forcepush.tech/python-package-ci-cd-with-git-hub-actions - - name: Get image tag from version.txt - run: | - cd images/dask - PROJECT_VERSION=$(cat version.txt) - echo "PROJECT_VERSION=$PROJECT_VERSION" >> $GITHUB_ENV - - uses: docker/build-push-action@v2 + python-version: ${{ env.PYTHON_VERSION }} + + - name: Setup Poetry + uses: abatilo/actions-poetry@v3 + with: + poetry-version: ${{ env.POETRY_VERSION }} + + - name: Get image tag from poetry + working-directory: images/dask + run: echo "PROJECT_VERSION=$(poetry version --short)" >> $GITHUB_ENV + + - name: Build and Push + uses: docker/build-push-action@v6 with: context: images/dask - push: ${{ github.ref == 'refs/heads/main' }} - tags: ghcr.io/${{github.repository}}/dask:${{env.PROJECT_VERSION}} + push: true + tags: ghcr.io/${{ github.repository }}/jupyter-singleuser:${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && env.PROJECT_VERSION || 'development' }} + cache-from: type=registry,ref=ghcr.io/${{ github.repository }}/jupyter-singleuser:${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && env.PROJECT_VERSION || 'development' }} + cache-to: type=inline diff --git a/.github/workflows/build-gtfs-rt-archiver-v3-image.yml b/.github/workflows/build-gtfs-rt-archiver-v3-image.yml index 35aae1306f..f0ea554716 100644 --- a/.github/workflows/build-gtfs-rt-archiver-v3-image.yml +++ b/.github/workflows/build-gtfs-rt-archiver-v3-image.yml @@ -3,7 +3,7 @@ name: Build and push gtfs-rt-archiver-v3 image on: push: branches: - - 'main' + - main paths: - '.github/workflows/build-gtfs-rt-archiver-v3-image.yml' - 'services/gtfs-rt-archiver-v3/**' @@ -16,47 +16,95 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} +env: + PYTHON_VERSION: '3.11' + POETRY_VERSION: '2.0.1' + jobs: - check: - name: Run static checkers + lint: + name: Lint runs-on: ubuntu-latest - env: - CALITP_BUCKET__GTFS_RT_PARSED: gs://some-test-bucket - CALITP_BUCKET__GTFS_RT_VALIDATION: gs://some-test-bucket - GTFS_RT_VALIDATOR_VERSION: v0.0.0 steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v4 + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Graphviz + uses: ts-graphviz/setup-graphviz@v2 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Cache poetry + uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry + key: poetry-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-poetry-${{ env.POETRY_VERSION }} + + - name: Setup Poetry + uses: abatilo/actions-poetry@v3 + with: + poetry-version: ${{ env.POETRY_VERSION }} + + - name: Cache python packages + uses: actions/cache@v3 with: - python-version: '3.9' - - run: curl -sSL https://install.python-poetry.org | python - - - run: cd services/gtfs-rt-archiver-v3 && poetry install && poetry run mypy . + path: ~/.local + key: python-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-lock-${{ hashFiles('poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }} - build_push: - name: Package docker image + - name: Install dependencies + working-directory: services/gtfs-rt-archiver-v3 + run: poetry install + + - name: Run mypy + working-directory: services/gtfs-rt-archiver-v3 + run: poetry run mypy . + + docker: + name: Push docker image runs-on: ubuntu-latest - needs: [check] + permissions: + contents: read + packages: write + attestations: write + id-token: write steps: - - uses: actions/checkout@v2 - - uses: docker/login-action@v1 + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Repository + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/setup-python@v4 + + - name: Setup Python + uses: actions/setup-python@v5 with: - python-version: '3.10' - - run: curl -sSL https://install.python-poetry.org | python - - # from https://forcepush.tech/python-package-ci-cd-with-git-hub-actions - - name: Get image tag from pyproject.toml and commit SHA - run: | - cd services/gtfs-rt-archiver-v3 - PROJECT_VERSION=$(poetry version --short) - echo "PROJECT_VERSION=$PROJECT_VERSION" >> $GITHUB_ENV + python-version: ${{ env.PYTHON_VERSION }} + + - name: Setup Poetry + uses: abatilo/actions-poetry@v3 + with: + poetry-version: ${{ env.POETRY_VERSION }} + + - name: Get image tag from poetry + working-directory: services/gtfs-rt-archiver-v3 + run: echo "PROJECT_VERSION=$(poetry version --short)" >> $GITHUB_ENV + + - name: Get short GitHub SHA + run: echo "GITHUB_SHA_SHORT=${GITHUB_SHA:0:6}" >> $GITHUB_ENV - echo "GITHUB_SHA_SHORT=${GITHUB_SHA:0:6}" >> $GITHUB_ENV - - uses: docker/build-push-action@v2 + - name: Build and Push + uses: docker/build-push-action@v6 with: context: services/gtfs-rt-archiver-v3 - push: ${{ github.ref == 'refs/heads/main' }} - tags: ghcr.io/${{github.repository}}/gtfs-rt-archiver-v3:${{env.PROJECT_VERSION}}-${{env.GITHUB_SHA_SHORT}} + push: true + tags: ghcr.io/${{ github.repository }}/gtfs-rt-parser-v2:${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest' || 'development' }} + cache-from: type=registry,ref=ghcr.io/${{ github.repository }}/gtfs-rt-parser-v2:${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && env.PROJECT_VERSION || 'development' }}${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && '-' || '' }}${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && env.GITHUB_SHA_SHORT || '' }} + cache-to: type=inline diff --git a/.github/workflows/build-gtfs-rt-parser-v2-image.yml b/.github/workflows/build-gtfs-rt-parser-v2-image.yml index 0c98a6007f..a995837b8b 100644 --- a/.github/workflows/build-gtfs-rt-parser-v2-image.yml +++ b/.github/workflows/build-gtfs-rt-parser-v2-image.yml @@ -3,7 +3,7 @@ name: Build and push gtfs-rt-parser-v2 image on: push: branches: - - 'main' + - main paths: - '.github/workflows/build-gtfs-rt-parser-v2-image.yml' - 'jobs/gtfs-rt-parser-v2/**' @@ -17,23 +17,26 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: - PYTHON_VERSION: '3.9' + PYTHON_VERSION: '3.11' POETRY_VERSION: '2.0.1' jobs: - check: - name: Run lint and Test + lint: + name: Lint runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 + - name: Setup Graphviz + uses: ts-graphviz/setup-graphviz@v2 + - name: Setup Python uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - - name: Cache Poetry + - name: Cache poetry uses: actions/cache@v3 with: path: ~/.cache/pypoetry @@ -44,28 +47,63 @@ jobs: with: poetry-version: ${{ env.POETRY_VERSION }} - - name: Cache Python packages + - name: Cache python packages uses: actions/cache@v3 with: path: ~/.local - key: python-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-lock-${{ hashFiles('warehouse/poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }} + key: python-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-lock-${{ hashFiles('poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }} - name: Install dependencies working-directory: jobs/gtfs-rt-parser-v2 run: poetry install - - name: Run linter + - name: Run mypy working-directory: jobs/gtfs-rt-parser-v2 run: poetry run mypy . + test: + name: Test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Graphviz + uses: ts-graphviz/setup-graphviz@v2 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Cache poetry + uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry + key: poetry-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-poetry-${{ env.POETRY_VERSION }} + + - name: Setup Poetry + uses: abatilo/actions-poetry@v3 + with: + poetry-version: ${{ env.POETRY_VERSION }} + + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.local + key: python-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-lock-${{ hashFiles('poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }} + + - name: Install dependencies + working-directory: jobs/gtfs-rt-parser-v2 + run: poetry install + - name: Run pytest working-directory: jobs/gtfs-rt-parser-v2 run: poetry run pytest - build_push: - name: Package docker image + docker: + name: Push docker image runs-on: ubuntu-latest - needs: [check] permissions: contents: read packages: write @@ -75,6 +113,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Repository uses: docker/login-action@v3 with: diff --git a/.github/workflows/build-gtfs-schedule-validator-image.yml b/.github/workflows/build-gtfs-schedule-validator-image.yml index 9cdaefcc2b..64261be0ca 100644 --- a/.github/workflows/build-gtfs-schedule-validator-image.yml +++ b/.github/workflows/build-gtfs-schedule-validator-image.yml @@ -3,7 +3,7 @@ name: Build and push gtfs-schedule-validator image on: push: branches: - - 'main' + - main paths: - '.github/workflows/build-gtfs-schedule-validator-image.yml' - 'jobs/gtfs-schedule-validator/**' @@ -16,45 +16,118 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} +env: + PYTHON_VERSION: '3.11' + POETRY_VERSION: '2.0.1' + jobs: - check: - name: Run static checkers + lint: + name: Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v4 + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Graphviz + uses: ts-graphviz/setup-graphviz@v2 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Cache poetry + uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry + key: poetry-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-poetry-${{ env.POETRY_VERSION }} + + - name: Setup Poetry + uses: abatilo/actions-poetry@v3 + with: + poetry-version: ${{ env.POETRY_VERSION }} + + - name: Cache python packages + uses: actions/cache@v3 with: - python-version: '3.9' - - run: curl -sSL https://install.python-poetry.org | python - - - run: cd jobs/gtfs-schedule-validator && poetry install && poetry run mypy . + path: ~/.local + key: python-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-lock-${{ hashFiles('poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }} + + - name: Install dependencies + working-directory: jobs/gtfs-schedule-validator + run: poetry install + + - name: Run mypy + working-directory: jobs/gtfs-schedule-validator + run: poetry run mypy . test: - name: Run pytest + name: Test runs-on: ubuntu-latest - env: - CALITP_BUCKET__GTFS_SCHEDULE_VALIDATION_HOURLY: gs://some-test-bucket steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v4 + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Graphviz + uses: ts-graphviz/setup-graphviz@v2 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Cache poetry + uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry + key: poetry-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-poetry-${{ env.POETRY_VERSION }} + + - name: Setup Poetry + uses: abatilo/actions-poetry@v3 + with: + poetry-version: ${{ env.POETRY_VERSION }} + + - name: Cache python packages + uses: actions/cache@v3 with: - python-version: '3.9' - - run: curl -sSL https://install.python-poetry.org | python - - - run: cd jobs/gtfs-schedule-validator && poetry install && poetry run pytest + path: ~/.local + key: python-cache-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-lock-${{ hashFiles('poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }} + + - name: Install dependencies + working-directory: jobs/gtfs-schedule-validator + run: poetry install + + - name: Run pytest + working-directory: jobs/gtfs-schedule-validator + run: poetry run pytest - build_push: - name: Package docker image + docker: + name: Push docker image runs-on: ubuntu-latest - needs: [check, test] + permissions: + contents: read + packages: write + attestations: write + id-token: write steps: - - uses: actions/checkout@v2 - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Repository + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/build-push-action@v2 + + - name: Build and Push + uses: docker/build-push-action@v6 with: context: jobs/gtfs-schedule-validator - push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - tags: ghcr.io/${{github.repository}}/gtfs-schedule-validator:latest + push: true + tags: ghcr.io/${{ github.repository }}/gtfs-schedule-validator:${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest' || 'development' }} + cache-from: type=registry,ref=ghcr.io/${{ github.repository }}/gtfs-schedule-validator:${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest' || 'development' }} + cache-to: type=inline diff --git a/.github/workflows/build-jupyter-singleuser-image.yml b/.github/workflows/build-jupyter-singleuser-image.yml index fd968ec200..a387a6ae56 100644 --- a/.github/workflows/build-jupyter-singleuser-image.yml +++ b/.github/workflows/build-jupyter-singleuser-image.yml @@ -3,7 +3,7 @@ name: Build and push jupyter-singleuser image on: push: branches: - - 'main' + - main paths: - '.github/workflows/build-jupyter-singleuser-image.yml' - 'images/jupyter-singleuser/**' @@ -16,29 +16,52 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} +env: + PYTHON_VERSION: '3.11' + POETRY_VERSION: '2.0.1' + jobs: - build_push: - name: Package docker image + docker: + name: Push docker image runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + id-token: write steps: - - uses: actions/checkout@v2 - - uses: docker/login-action@v1 + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Repository + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/setup-python@v4 + + - name: Setup Python + uses: actions/setup-python@v5 with: - python-version: '3.10' - - run: curl -sSL https://install.python-poetry.org | python - - # from https://forcepush.tech/python-package-ci-cd-with-git-hub-actions - - name: Get image tag from pyproject.toml - run: | - cd images/jupyter-singleuser - PROJECT_VERSION=$(poetry version --short) - echo "PROJECT_VERSION=$PROJECT_VERSION" >> $GITHUB_ENV - - uses: docker/build-push-action@v2 + python-version: ${{ env.PYTHON_VERSION }} + + - name: Setup Poetry + uses: abatilo/actions-poetry@v3 + with: + poetry-version: ${{ env.POETRY_VERSION }} + + - name: Get image tag from poetry + working-directory: images/jupyter-singleuser + run: echo "PROJECT_VERSION=$(poetry version --short)" >> $GITHUB_ENV + + - name: Build and Push + uses: docker/build-push-action@v6 with: context: images/jupyter-singleuser - push: ${{ github.ref == 'refs/heads/main' }} - tags: ghcr.io/${{github.repository}}/jupyter-singleuser:${{env.PROJECT_VERSION}} + push: true + tags: ghcr.io/${{ github.repository }}/jupyter-singleuser:${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && env.PROJECT_VERSION || 'development' }} + cache-from: type=registry,ref=ghcr.io/${{ github.repository }}/jupyter-singleuser:${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && env.PROJECT_VERSION || 'development' }} + cache-to: type=inline