From 4e33e57d22b9a22ae4ce523e2187d925ffa84849 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 26 Sep 2024 10:40:33 +0100 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactor=20project=20struc?= =?UTF-8?q?ture=20(#62)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jacob Woffenden --- .devcontainer/devcontainer-lock.json | 2 +- .github/dependabot.yml | 16 ++++++ .github/workflows/chart-lint.yml | 2 +- .github/workflows/dependency-review.yml | 4 +- .github/workflows/release.yml | 54 +++++++++++++++---- .github/workflows/scan-image.yml | 50 ++++++----------- .github/workflows/super-linter.yml | 4 +- .../{build-and-test.yml => test.yml} | 12 +++-- Dockerfile | 17 +++--- Makefile | 25 ++++++--- README.md | 36 ++++++------- chart/Chart.yaml | 4 +- chart/values.yaml | 2 +- src/{ => opt}/mlflow/auth.ini | 0 src/opt/mlflow/requirements.txt | 3 ++ src/usr/local/bin/entrypoint.sh | 2 - test/container-structure-test.yml | 12 +++-- test/docker-compose.yml | 6 +-- 18 files changed, 146 insertions(+), 105 deletions(-) rename .github/workflows/{build-and-test.yml => test.yml} (81%) rename src/{ => opt}/mlflow/auth.ini (100%) create mode 100644 src/opt/mlflow/requirements.txt diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json index 757a6f4..47a8b16 100644 --- a/.devcontainer/devcontainer-lock.json +++ b/.devcontainer/devcontainer-lock.json @@ -24,4 +24,4 @@ "integrity": "sha256:e81d52725655c8ffb861605feac7ad155b447d51af65f6c3a03cab32d59f1e16" } } -} +} \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 37b6438..9373cd3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,11 +6,27 @@ updates: directory: "/" schedule: interval: "daily" + commit-message: + prefix: ":dependabot: github-actions" + include: "scope" - package-ecosystem: "devcontainers" directory: "/" schedule: interval: "daily" + commit-message: + prefix: ":dependabot: devcontainers" + include: "scope" - package-ecosystem: "docker" directory: "/" schedule: interval: "daily" + commit-message: + prefix: ":dependabot: docker" + include: "scope" + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + commit-message: + prefix: ":dependabot: pip" + include: "scope" diff --git a/.github/workflows/chart-lint.yml b/.github/workflows/chart-lint.yml index e7393be..8882ca7 100644 --- a/.github/workflows/chart-lint.yml +++ b/.github/workflows/chart-lint.yml @@ -1,5 +1,5 @@ --- -name: Chart Lint +name: πŸ—ΊοΈ Chart Lint on: pull_request: diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index b54d966..b582289 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -1,5 +1,5 @@ --- -name: Dependency Review +name: πŸ” Dependency Review on: pull_request: @@ -24,7 +24,7 @@ jobs: id: checkout uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - name: Dependency review + - name: Dependency Review id: dependency_review uses: actions/dependency-review-action@5a2ce3f5b92ee19cbb1541a4984c76d921601d7c # v4.3.4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3c41dc3..0dfaa3e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,5 @@ --- -name: Release +name: πŸ”– Release on: push: @@ -13,7 +13,9 @@ jobs: name: Release Image runs-on: ubuntu-latest permissions: - contents: read + actions: read + attestations: write + contents: write id-token: write packages: write steps: @@ -26,7 +28,7 @@ jobs: uses: sigstore/cosign-installer@4959ce089c160fddf62f7b42464195ba1a56d382 # v3.6.0 - name: Log in to GitHub Container Registry - id: login_ghcr + id: ghcr_login uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: ghcr.io @@ -38,21 +40,55 @@ jobs: uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 with: push: true - tags: ghcr.io/ministryofjustice/analytical-platform-mlflow:${{ github.ref_name }} + tags: ghcr.io/${{ github.repository }}:${{ github.ref_name }} - name: Sign id: sign shell: bash run: | - cosign sign --yes ghcr.io/ministryofjustice/analytical-platform-mlflow@${{ steps.build_and_push.outputs.digest }} + cosign sign --yes ghcr.io/${{ github.repository }}@${{ steps.build_and_push.outputs.digest }} + + - name: Generate SBOM + id: generate_sbom + uses: anchore/sbom-action@61119d458adab75f756bc0b9e4bde25725f86a7a # v0.17.2 + with: + image: ghcr.io/${{ github.repository }}:${{ github.ref_name }} + format: cyclonedx-json + output-file: "sbom.cyclonedx.json" + + - name: Attest + uses: actions/attest-build-provenance@1c608d11d69870c2092266b3f9a6f3abbf17002c # v1.4.3 + id: attest + with: + subject-name: ghcr.io/${{ github.repository }} + subject-digest: ${{ steps.build_and_push.outputs.digest }} + push-to-registry: true - - name: Verify - id: verify + - name: Attest SBOM + uses: actions/attest-sbom@5026d3663739160db546203eeaffa6aa1c51a4d6 # v1.4.1 + id: attest_sbom + with: + subject-name: ghcr.io/${{ github.repository }} + subject-digest: ${{ steps.build_and_push.outputs.digest }} + sbom-path: sbom.cyclonedx.json + push-to-registry: true + + - name: cosign Verify + id: cosign_verify + shell: bash run: | cosign verify \ --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ - --certificate-identity=https://github.com/ministryofjustice/analytical-platform-mlflow/.github/workflows/release.yml@refs/tags/${{ github.ref_name }} \ - ghcr.io/ministryofjustice/analytical-platform-mlflow@${{ steps.build_and_push.outputs.digest }} + --certificate-identity=https://github.com/${{ github.workflow_ref }} \ + ghcr.io/${{ github.repository }}@${{ steps.build_and_push.outputs.digest }} + + - name: GitHub Attestation Verify + id: gh_attestation_verify + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + gh attestation verify oci://ghcr.io/${{ github.repository }}:${{ github.ref_name }} --repo ${{ github.repository }} release-chart: name: Release Chart diff --git a/.github/workflows/scan-image.yml b/.github/workflows/scan-image.yml index 873a62f..bfd77b1 100644 --- a/.github/workflows/scan-image.yml +++ b/.github/workflows/scan-image.yml @@ -1,5 +1,5 @@ --- -name: Scan Image +name: 🩻 Scan on: pull_request: @@ -9,49 +9,29 @@ on: permissions: {} jobs: - scan-image: - name: Scan Image + scan: + name: Scan runs-on: ubuntu-latest permissions: contents: read - security-events: write steps: - name: Checkout id: checkout uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - name: Build Image - id: build_image - uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 - with: - push: false - load: true - tags: mlflow + - name: Build + id: build + shell: bash + env: + IMAGE_NAME: ghcr.io/${{ github.repository }} + IMAGE_TAG: ${{ github.sha }} + run: | + make build - - name: Scan Image - id: scan_image + - name: Scan + id: scan uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0 with: - image-ref: mlflow + image-ref: ghcr.io/${{ github.repository }}:${{ github.sha }} + severity: HIGH,CRITICAL exit-code: 1 - format: sarif - output: trivy-results.sarif - severity: CRITICAL - limit-severities-for-sarif: true - - - name: Scan Image (On SARIF Scan Failure) - if: failure() && steps.scan_image.outcome == 'failure' - id: scan_image_on_failure - uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0 - with: - image-ref: mlflow - exit-code: 1 - format: table - severity: CRITICAL - - - name: Upload SARIF - if: always() - id: upload_sarif - uses: github/codeql-action/upload-sarif@294a9d92911152fe08befb9ec03e240add280cb3 # v3.26.8 - with: - sarif_file: trivy-results.sarif diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml index c773060..4fa206d 100644 --- a/.github/workflows/super-linter.yml +++ b/.github/workflows/super-linter.yml @@ -1,5 +1,5 @@ --- -name: Super-Linter +name: 🦝 Super-Linter on: pull_request: @@ -27,7 +27,7 @@ jobs: with: fetch-depth: 0 - - name: Run Super-Linter + - name: Super-Linter id: super_linter uses: super-linter/super-linter/slim@b92721f792f381cedc002ecdbb9847a15ece5bb8 # v7.1.0 env: diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/test.yml similarity index 81% rename from .github/workflows/build-and-test.yml rename to .github/workflows/test.yml index 2bea09e..6322068 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,5 @@ --- -name: Build and Test +name: πŸ§ͺ Test on: pull_request: @@ -9,8 +9,8 @@ on: permissions: {} jobs: - build-and-test: - name: Build and Test + test: + name: Test runs-on: ubuntu-latest permissions: contents: read @@ -23,8 +23,10 @@ jobs: id: setup_container_structure_test uses: ministryofjustice/github-actions/setup-container-structure-test@eaec2bb18f6dd19dd0fcb3cc48f7803a3731b7e5 # v18.1.5 - - name: Build and Test - id: build_and_test + - name: Test + id: test shell: bash + env: + IMAGE_TAG: ${{ github.sha }} run: | make test diff --git a/Dockerfile b/Dockerfile index f12adbe..5be71dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,10 +11,7 @@ ENV CONTAINER_USER="analyticalplatform" \ CONTAINER_GROUP="analyticalplatform" \ CONTAINER_GID="1000" \ DEBIAN_FRONTEND="noninteractive" \ - MLFLOW_ROOT="/mlflow" \ - MLFLOW_VERSION="2.16.2" \ - BOTO3_VERSION="1.35.25" \ - PSYCOPG2_BINARY_VERSION="2.9.9" + MLFLOW_ROOT="/opt/mlflow" SHELL ["/bin/bash", "-e", "-u", "-o", "pipefail", "-c"] @@ -45,18 +42,18 @@ apt-get clean --yes rm --force --recursive /var/lib/apt/lists/* -pip install --break-system-packages --no-cache-dir \ - "mlflow==${MLFLOW_VERSION}" \ - "boto3==${BOTO3_VERSION}" \ - "psycopg2-binary==${PSYCOPG2_BINARY_VERSION}" - install --directory --owner ${CONTAINER_USER} --group ${CONTAINER_GROUP} --mode 0755 ${MLFLOW_ROOT} EOF +COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} src${MLFLOW_ROOT}/requirements.txt ${MLFLOW_ROOT}/requirements.txt +RUN < [!IMPORTANT] +> MLflow on Analytical Platform is currently in discovery -The base container image is [Ubuntu 24.04 LTS](https://gallery.ecr.aws/ubuntu/ubuntu). +This repository contains the MLflow container image for use on the Analytical Platform. ## Running Locally ### Build ```bash -docker build --platform linux/amd64 --file Dockerfile --tag analytical-platform.service.justice.gov.uk/mlflow:local . +make build ``` -### Run +### Test ```bash -docker run -it --rm \ - --platform linux/amd64 \ - --publish 5000:5000 \ - --hostname mlflow \ - --name analytical-platform-mlflow \ - analytical-platform.service.justice.gov.uk/mlflow:local +make test ``` -### Use +### Run + +```bash +make run +``` Open a browser @@ -35,17 +35,17 @@ Open a browser ### Ubuntu -Generally Dependabot does this, but the following command will return the digest: +Dependabot is configured to do this in [`.github/dependabot.yml`](.github/dependabot.yml), but if you need to get the digest, do the following ```bash docker pull --platform linux/amd64 public.ecr.aws/ubuntu/ubuntu:24.04 -docker image inspect --format='{{index .RepoDigests 0}}' public.ecr.aws/ubuntu/ubuntu:24.04 +docker image inspect --format='{{ index .RepoDigests 0 }}' public.ecr.aws/ubuntu/ubuntu:24.04 ``` ### APT Packages -To find latest APT package versions, you can run the following: +The latest versions of the APT packages can be obtained by running the following ```bash docker run -it --rm --platform linux/amd64 public.ecr.aws/ubuntu/ubuntu:24.04 @@ -57,8 +57,4 @@ apt-cache policy ${PACKAGE} # for example curl, git or gpg ### MLflow - - -### Prometheus Flask Exporter - - +Dependabot is configured to manage the packages in [`src/opt/mlflow/requirements.txt`](src/opt/analytical-platform/requirements.txt) diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 112022c..ab74ea2 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -3,8 +3,8 @@ apiVersion: v2 name: mlflow description: MLflow Tracking Server type: application -version: 2.16.2-rc1 -appVersion: 2.16.2-rc1 +version: 2.16.2-rc2 +appVersion: 2.16.2-rc2 home: https://github.com/ministryofjustice/analytical-platform-mlflow sources: - https://github.com/mlflow/mlflow diff --git a/chart/values.yaml b/chart/values.yaml index 27ef830..c8f5268 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -6,7 +6,7 @@ fullnameOverride: "" image: repository: ghcr.io/ministryofjustice/analytical-platform-mlflow pullPolicy: IfNotPresent - tag: 2.16.2-rc1 + tag: 2.16.2-rc2 imagePullSecrets: [] diff --git a/src/mlflow/auth.ini b/src/opt/mlflow/auth.ini similarity index 100% rename from src/mlflow/auth.ini rename to src/opt/mlflow/auth.ini diff --git a/src/opt/mlflow/requirements.txt b/src/opt/mlflow/requirements.txt new file mode 100644 index 0000000..c9c81f3 --- /dev/null +++ b/src/opt/mlflow/requirements.txt @@ -0,0 +1,3 @@ +boto3==1.35.27 +mlflow==2.16.2 +psycopg2-binary==2.9.9 diff --git a/src/usr/local/bin/entrypoint.sh b/src/usr/local/bin/entrypoint.sh index 9a773f7..bb37ac2 100755 --- a/src/usr/local/bin/entrypoint.sh +++ b/src/usr/local/bin/entrypoint.sh @@ -9,7 +9,6 @@ export MLFLOW_SERVER_DEV_MODE="${MLFLOW_SERVER_DEV_MODE:-"false"}" export MLFLOW_SERVER_HOST="${MLFLOW_SERVER_HOST:-"0.0.0.0"}" export MLFLOW_SERVER_PORT="${MLFLOW_SERVER_PORT:-"5000"}" export MLFLOW_SERVER_WORKERS="${MLFLOW_SERVER_WORKERS:-"1"}" -export MLFLOW_SERVER_METRICS_PATH="${MLFLOW_SERVER_METRICS_PATH:-"${MLFLOW_ROOT}/metrics"}" export MLFLOW_SERVER_BACKEND_STORE_URI="${MLFLOW_SERVER_BACKEND_STORE_URI:-"sqlite:///mlflow.db"}" export MLFLOW_SERVER_DEFAULT_ARTIFACT_ROOT="${MLFLOW_SERVER_DEFAULT_ARTIFACT_ROOT:-"${MLFLOW_ROOT}/mlruns"}" @@ -28,7 +27,6 @@ mlflow server ${MLFLOW_SERVER_DEV_MODE_FLAG} \ --host "${MLFLOW_SERVER_HOST}" \ --port "${MLFLOW_SERVER_PORT}" \ --workers "${MLFLOW_SERVER_WORKERS}" \ - --expose-prometheus "${MLFLOW_SERVER_METRICS_PATH}" \ --app-name basic-auth \ --backend-store-uri "${MLFLOW_SERVER_BACKEND_STORE_URI}" \ --default-artifact-root "${MLFLOW_SERVER_DEFAULT_ARTIFACT_ROOT}" diff --git a/test/container-structure-test.yml b/test/container-structure-test.yml index 338b7c1..9ae3578 100644 --- a/test/container-structure-test.yml +++ b/test/container-structure-test.yml @@ -40,12 +40,16 @@ commandTests: expectedOutput: ["mlflow, version 2.16.2"] fileExistenceTests: - - name: "/mlflow" - path: "/mlflow" + - name: "/opt/mlflow" + path: "/opt/mlflow" shouldExist: true - - name: "/mlflow/auth.ini" - path: "/mlflow/auth.ini" + - name: "/opt/mlflow/auth.ini" + path: "/opt/mlflow/auth.ini" + shouldExist: true + + - name: "/opt/mlflow/requirements.txt" + path: "/opt/mlflow/requirements.txt" shouldExist: true - name: "/usr/local/bin/entrypoint.sh" diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 8f09289..39aa8cf 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -1,7 +1,7 @@ --- services: mlflow_auth_db: - image: public.ecr.aws/docker/library/postgres:16.3-alpine3.20 + image: public.ecr.aws/docker/library/postgres:16.4-alpine3.20 environment: POSTGRES_USER: mlflow POSTGRES_PASSWORD: mlflow @@ -12,7 +12,7 @@ services: timeout: 5s retries: 5 mlflow_db: - image: public.ecr.aws/docker/library/postgres:16.3-alpine3.20 + image: public.ecr.aws/docker/library/postgres:16.4-alpine3.20 environment: POSTGRES_USER: mlflow POSTGRES_PASSWORD: mlflow @@ -23,7 +23,7 @@ services: timeout: 5s retries: 5 mlflow: - image: analytical-platform.service.justice.gov.uk/mlflow:local + image: ghcr.io/ministryofjustice/analytical-platform-mlflow:local build: . environment: # MLFLOW_SERVER_DEV_MODE: "true"