♻️ Refactor project structure (#62)

Signed-off-by: Jacob Woffenden <[email protected]>
ministryofjustice · Sep 26, 2024 · 4e33e57 · 4e33e57
1 parent 01c5d78
commit 4e33e57
Show file tree

Hide file tree

Showing 18 changed files with 146 additions and 105 deletions.
diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json
@@ -24,4 +24,4 @@
       "integrity": "sha256:e81d52725655c8ffb861605feac7ad155b447d51af65f6c3a03cab32d59f1e16"
     }
   }
-}
+}
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -6,11 +6,27 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+    commit-message:
+      prefix: ":dependabot: github-actions"
+      include: "scope"
   - package-ecosystem: "devcontainers"
     directory: "/"
     schedule:
       interval: "daily"
+    commit-message:
+      prefix: ":dependabot: devcontainers"
+      include: "scope"
   - package-ecosystem: "docker"
     directory: "/"
     schedule:
       interval: "daily"
+    commit-message:
+      prefix: ":dependabot: docker"
+      include: "scope"
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    commit-message:
+      prefix: ":dependabot: pip"
+      include: "scope"
diff --git a/.github/workflows/chart-lint.yml b/.github/workflows/chart-lint.yml
@@ -1,5 +1,5 @@
 ---
-name: Chart Lint
+name: 🗺️ Chart Lint
 
 on:
   pull_request:

diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
@@ -1,5 +1,5 @@
 ---
-name: Dependency Review
+name: 🔍 Dependency Review
 
 on:
   pull_request:
@@ -24,7 +24,7 @@ jobs:
         id: checkout
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
-      - name: Dependency review
+      - name: Dependency Review
         id: dependency_review
         uses: actions/dependency-review-action@5a2ce3f5b92ee19cbb1541a4984c76d921601d7c # v4.3.4
         with:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,5 +1,5 @@
 ---
-name: Release
+name: 🔖 Release
 
 on:
   push:
@@ -13,7 +13,9 @@ jobs:
     name: Release Image
     runs-on: ubuntu-latest
     permissions:
-      contents: read
+      actions: read
+      attestations: write
+      contents: write
       id-token: write
       packages: write
     steps:
@@ -26,7 +28,7 @@ jobs:
         uses: sigstore/cosign-installer@4959ce089c160fddf62f7b42464195ba1a56d382 # v3.6.0
 
       - name: Log in to GitHub Container Registry
-        id: login_ghcr
+        id: ghcr_login
         uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
@@ -38,21 +40,55 @@ jobs:
         uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
         with:
           push: true
-          tags: ghcr.io/ministryofjustice/analytical-platform-mlflow:${{ github.ref_name }}
+          tags: ghcr.io/${{ github.repository }}:${{ github.ref_name }}
 
       - name: Sign
         id: sign
         shell: bash
         run: |
-          cosign sign --yes ghcr.io/ministryofjustice/analytical-platform-mlflow@${{ steps.build_and_push.outputs.digest }}
+          cosign sign --yes ghcr.io/${{ github.repository }}@${{ steps.build_and_push.outputs.digest }}
+
+      - name: Generate SBOM
+        id: generate_sbom
+        uses: anchore/sbom-action@61119d458adab75f756bc0b9e4bde25725f86a7a # v0.17.2
+        with:
+          image: ghcr.io/${{ github.repository }}:${{ github.ref_name }}
+          format: cyclonedx-json
+          output-file: "sbom.cyclonedx.json"
+
+      - name: Attest
+        uses: actions/attest-build-provenance@1c608d11d69870c2092266b3f9a6f3abbf17002c # v1.4.3
+        id: attest
+        with:
+          subject-name: ghcr.io/${{ github.repository }}
+          subject-digest: ${{ steps.build_and_push.outputs.digest }}
+          push-to-registry: true
 
-      - name: Verify
-        id: verify
+      - name: Attest SBOM
+        uses: actions/attest-sbom@5026d3663739160db546203eeaffa6aa1c51a4d6 # v1.4.1
+        id: attest_sbom
+        with:
+          subject-name: ghcr.io/${{ github.repository }}
+          subject-digest: ${{ steps.build_and_push.outputs.digest }}
+          sbom-path: sbom.cyclonedx.json
+          push-to-registry: true
+
+      - name: cosign Verify
+        id: cosign_verify
+        shell: bash
         run: |
           cosign verify \
             --certificate-oidc-issuer=https://token.actions.githubusercontent.com \
-            --certificate-identity=https://github.com/ministryofjustice/analytical-platform-mlflow/.github/workflows/release.yml@refs/tags/${{ github.ref_name }} \
-            ghcr.io/ministryofjustice/analytical-platform-mlflow@${{ steps.build_and_push.outputs.digest }}
+            --certificate-identity=https://github.com/${{ github.workflow_ref }} \
+            ghcr.io/${{ github.repository }}@${{ steps.build_and_push.outputs.digest }}
+
+      - name: GitHub Attestation Verify
+        id: gh_attestation_verify
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh attestation verify oci://ghcr.io/${{ github.repository }}:${{ github.ref_name }} --repo ${{ github.repository }}
 
   release-chart:
     name: Release Chart

diff --git a/.github/workflows/scan-image.yml b/.github/workflows/scan-image.yml
@@ -1,5 +1,5 @@
 ---
-name: Scan Image
+name: 🩻 Scan
 
 on:
   pull_request:
@@ -9,49 +9,29 @@ on:
 permissions: {}
 
 jobs:
-  scan-image:
-    name: Scan Image
+  scan:
+    name: Scan
     runs-on: ubuntu-latest
     permissions:
       contents: read
-      security-events: write
     steps:
       - name: Checkout
         id: checkout
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
-      - name: Build Image
-        id: build_image
-        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
-        with:
-          push: false
-          load: true
-          tags: mlflow
+      - name: Build
+        id: build
+        shell: bash
+        env:
+          IMAGE_NAME: ghcr.io/${{ github.repository }}
+          IMAGE_TAG: ${{ github.sha }}
+        run: |
+          make build
 
-      - name: Scan Image
-        id: scan_image
+      - name: Scan
+        id: scan
         uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0
         with:
-          image-ref: mlflow
+          image-ref: ghcr.io/${{ github.repository }}:${{ github.sha }}
+          severity: HIGH,CRITICAL
           exit-code: 1
-          format: sarif
-          output: trivy-results.sarif
-          severity: CRITICAL
-          limit-severities-for-sarif: true
-
-      - name: Scan Image (On SARIF Scan Failure)
-        if: failure() && steps.scan_image.outcome == 'failure'
-        id: scan_image_on_failure
-        uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0
-        with:
-          image-ref: mlflow
-          exit-code: 1
-          format: table
-          severity: CRITICAL
-
-      - name: Upload SARIF
-        if: always()
-        id: upload_sarif
-        uses: github/codeql-action/upload-sarif@294a9d92911152fe08befb9ec03e240add280cb3 # v3.26.8
-        with:
-          sarif_file: trivy-results.sarif
diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml
@@ -1,5 +1,5 @@
 ---
-name: Super-Linter
+name: 🦝 Super-Linter
 
 on:
   pull_request:
@@ -27,7 +27,7 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Run Super-Linter
+      - name: Super-Linter
         id: super_linter
         uses: super-linter/super-linter/slim@b92721f792f381cedc002ecdbb9847a15ece5bb8 # v7.1.0
         env:

diff --git a/.github/workflows/build-and-test.yml → .github/workflows/test.yml b/.github/workflows/build-and-test.yml → .github/workflows/test.yml
@@ -1,5 +1,5 @@
 ---
-name: Build and Test
+name: 🧪 Test
 
 on:
   pull_request:
@@ -9,8 +9,8 @@ on:
 permissions: {}
 
 jobs:
-  build-and-test:
-    name: Build and Test
+  test:
+    name: Test
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -23,8 +23,10 @@ jobs:
         id: setup_container_structure_test
         uses: ministryofjustice/github-actions/setup-container-structure-test@eaec2bb18f6dd19dd0fcb3cc48f7803a3731b7e5 # v18.1.5
 
-      - name: Build and Test
-        id: build_and_test
+      - name: Test
+        id: test
         shell: bash
+        env:
+          IMAGE_TAG: ${{ github.sha }}
         run: |
           make test
diff --git a/Dockerfile b/Dockerfile
@@ -11,10 +11,7 @@ ENV CONTAINER_USER="analyticalplatform" \
     CONTAINER_GROUP="analyticalplatform" \
     CONTAINER_GID="1000" \
     DEBIAN_FRONTEND="noninteractive" \
-    MLFLOW_ROOT="/mlflow" \
-    MLFLOW_VERSION="2.16.2" \
-    BOTO3_VERSION="1.35.25" \
-    PSYCOPG2_BINARY_VERSION="2.9.9"
+    MLFLOW_ROOT="/opt/mlflow"
 
 SHELL ["/bin/bash", "-e", "-u", "-o", "pipefail", "-c"]
 
@@ -45,18 +42,18 @@ apt-get clean --yes
 
 rm --force --recursive /var/lib/apt/lists/*
 
-pip install --break-system-packages --no-cache-dir \
-  "mlflow==${MLFLOW_VERSION}" \
-  "boto3==${BOTO3_VERSION}" \
-  "psycopg2-binary==${PSYCOPG2_BINARY_VERSION}"
-
 install --directory --owner ${CONTAINER_USER} --group ${CONTAINER_GROUP} --mode 0755 ${MLFLOW_ROOT}
 EOF
 
+COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} src${MLFLOW_ROOT}/requirements.txt ${MLFLOW_ROOT}/requirements.txt
+RUN <<EOF
+pip install --break-system-packages --no-cache-dir --requirement ${MLFLOW_ROOT}/requirements.txt
+EOF
+
 USER ${CONTAINER_USER}
 WORKDIR ${MLFLOW_ROOT}
 EXPOSE 5000
-COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} src/mlflow/auth.ini /mlflow/auth.ini
+COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} src${MLFLOW_ROOT}/auth.ini ${MLFLOW_ROOT}/auth.ini
 COPY --chown=nobody:nobody --chmod=0755 src/usr/local/bin/entrypoint.sh /usr/local/bin/entrypoint.sh
 COPY --chown=nobody:nobody --chmod=0755 src/usr/local/bin/healthcheck.sh /usr/local/bin/healthcheck.sh
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

diff --git a/Makefile b/Makefile
@@ -1,18 +1,27 @@
-IMAGE_NAME = ghcr.io/ministryofjustice/analytical-platform-mlflow:latest
+.PHONY: test build run scan ct
+
+IMAGE_NAME ?= ghcr.io/ministryofjustice/analytical-platform-mlflow
+IMAGE_TAG  ?= local
+
+ct:
+	ct lint --charts chart
+
+scan: build
+	trivy image --vuln-type os,library --severity CRITICAL --exit-code 1 $(IMAGE_NAME)
+
+run: build
+	docker-compose --project-directory test --file test/docker-compose.yml up
 
 test: build
-	container-structure-test test --platform linux/amd64 --config test/container-structure-test.yml --image $(IMAGE_NAME)
+	container-structure-test test --platform linux/amd64 --config test/container-structure-test.yml --image $(IMAGE_NAME):$(IMAGE_TAG)
 
 build:
-	@ARCH=`uname -m`; \
+	@ARCH=`uname --machine`; \
 	case $$ARCH in \
 	aarch64 | arm64) \
 		echo "Building on $$ARCH architecture"; \
-		docker build --platform linux/amd64 --file Dockerfile --tag $(IMAGE_NAME) . ;; \
+		docker build --platform linux/amd64 --file Dockerfile --tag $(IMAGE_NAME):$(IMAGE_TAG) . ;; \
 	*) \
 		echo "Building on $$ARCH architecture"; \
-		docker build --file Dockerfile --tag $(IMAGE_NAME) . ;; \
+		docker build --file Dockerfile --tag $(IMAGE_NAME):$(IMAGE_TAG) . ;; \
 	esac
-
-ct:
-	ct lint --charts chart
-Original file line number
+Diff line change
@@ Expand Up / @@ -24,4 +24,4 @@ @@
           "integrity": "sha256:e81d52725655c8ffb861605feac7ad155b447d51af65f6c3a03cab32d59f1e16"
         }
       }
-    }
+    }