[Project] Add option to enrich project on load from project_setup.py file if it exists

.github/actions/image-matrix-prep/action.yaml

@@ -0,0 +1,38 @@
+# Copyright 2018 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Image Matrix Prep
+description: Prepares the matrix of images to build
+inputs:
+  skip_images:
+    description: 'Comma separated list of images to skip'
+    required: false
+    default: ''
+outputs:
+  matrix:
+    description: 'The matrix of images to build'
+    value: ${{ steps.set-matrix.outputs.matrix }}
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v3
+    - id: set-matrix
+      run: |
+        skipImages=",$INPUT_SKIP_IMAGES,"
+        matrix=$(jq --arg skipImages "$skipImages" 'map(. | select(",\(."image-name")," | inside($skipImages)|not))' ./.github/workflows/build-workflow-matrix.json)
+        echo "matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT
+      shell: bash
+      env:
+        INPUT_SKIP_IMAGES: ${{ inputs.skip_images }}

.github/workflows/build.yaml

@@ -14,6 +14,7 @@
 #
 # This name is referenced in the release.yaml workflow, if you're changing here - change there
 name: Build
+run-name: Building ${{ inputs.version }} ${{ github.ref_name }}
 
 on:
   push:
@@ -43,6 +44,7 @@ on:
         description: 'Whether to build images from cache or not. Default: true, set to false only if required because that will cause a significant increase in build time'
         required: true
         default: 'true'
+
 jobs:
   matrix_prep:
     runs-on: ubuntu-latest
@@ -51,12 +53,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - id: set-matrix
-        run: |
-          skipImages=",$INPUT_SKIP_IMAGES,"
-          matrix=$(jq --arg skipImages "$skipImages" 'map(. | select(",\(."image-name")," | inside($skipImages)|not))' ./.github/workflows/build-workflow-matrix.json)
-          echo "matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT
-        env:
-          INPUT_SKIP_IMAGES: ${{ github.event.inputs.skip_images }}
+        uses: ./.github/actions/image-matrix-prep
+        with:
+          skip_images: ${{ github.event.inputs.skip_images }}
+
   build-images:
     name: Build and push image - ${{ matrix.image-name }} (Python ${{ matrix.python-version }})
     runs-on: ubuntu-latest

.github/workflows/ci.yaml

@@ -19,6 +19,7 @@ on:
     branches:
     - development
     - '[0-9]+.[0-9]+.x'
+    - 'feature/**'
 
   # Run CI also on push to backport release branches - we sometimes push code there by cherry-picking, meaning it
   # doesn't go through CI (no PR)
@@ -29,6 +30,9 @@ on:
     - master
     - '[0-9]+.[0-9]+.x'
 
+env:
+  NAMESPACE: mlrun-integ-test
+
 jobs:
   lint:
     name: Lint code (Python ${{ matrix.python-version }})
@@ -127,6 +131,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      # since github-actions gives us 14G only, and fills it up with some garbage
+      - name: Freeing up disk space
+        run: |
+          "${GITHUB_WORKSPACE}/automation/scripts/github_workflow_free_space.sh"
       - uses: manusa/[email protected]
         with:
           minikube version: "v1.28.0"
@@ -140,6 +148,19 @@ jobs:
       - name: Run GO tests
         run: |
           make test-go-integration-dockerized
+      - name: Output some logs in case of failure
+        if: ${{ failure() }}
+        # add set -x to print commands before executing to make logs reading easier
+        run: |
+          set -x
+          minikube ip
+          minikube logs
+          minikube kubectl -- --namespace ${NAMESPACE} get events
+          minikube kubectl -- --namespace ${NAMESPACE} logs -l app.kubernetes.io/component=api,app.kubernetes.io/name=mlrun --tail=-1
+          minikube kubectl -- --namespace ${NAMESPACE} get pods
+          minikube kubectl -- --namespace ${NAMESPACE} get pods -o yaml
+          minikube kubectl -- --namespace ${NAMESPACE} describe pods
+          set +x
 
   migrations-tests:
     name: Run Dockerized Migrations Tests

.github/workflows/periodic-rebuild.yaml

@@ -0,0 +1,53 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: Scheduled Re-Build Images
+
+on:
+  schedule:
+  # every night at 2am
+  - cron: "0 2 * * *"
+
+jobs:
+  re-build-images:
+    # let's not run this on every fork, change to your fork when developing
+    if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch'
+    strategy:
+      fail-fast: false
+      matrix:
+        repo: ["mlrun","ui"]
+        branch: ["development","1.3.x"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Re-Build MLRun Image
+        if: matrix.repo == 'mlrun'
+        uses: convictional/[email protected]
+        with:
+            owner: mlrun
+            repo: mlrun
+            github_token: ${{ secrets.RELEASE_GITHUB_ACCESS_TOKEN }}
+            workflow_file_name: build.yaml
+            ref: ${{ matrix.branch }}
+            wait_interval: 60
+            client_payload: '{"skip_images": "models-gpu,models,base,tests", "build_from_cache": "false"}'
+      - name: Re-Build UI Image
+        if: matrix.repo == 'ui'
+        uses: convictional/[email protected]
+        with:
+          owner: mlrun
+          repo: ui
+          github_token: ${{ secrets.RELEASE_GITHUB_ACCESS_TOKEN }}
+          workflow_file_name: build.yaml
+          ref: ${{ matrix.branch }}
+          wait_interval: 60

.github/workflows/release.yaml

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
 name: Release
+run-name: Releasing ${{ inputs.version }}
 
 on:
   workflow_dispatch:
@@ -157,3 +159,21 @@ jobs:
           allowUpdates: true
           prerelease: ${{ github.event.inputs.pre_release }}
           body: ${{ steps.resolve-release-notes.outputs.body }}
+
+
+  update-tutorials:
+    name: Bundle tutorials
+    needs: create-releases
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Create tutorials tar
+        run: |
+          tar -cvf mlrun-tutorials.tar docs/tutorial
+      - name: Add tutorials tar to release
+        uses: ncipollo/release-action@v1
+        with:
+          allowUpdates: true
+          tag: v${{ github.event.inputs.version }}
+          token: ${{ secrets.RELEASE_GITHUB_ACCESS_TOKEN }}
+          artifacts: mlrun-tutorials.tar

.github/workflows/security_scan.yaml

@@ -0,0 +1,153 @@
+# Copyright 2018 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Currently supported running against prebuilt images
+name: Security Scan
+run-name: Scanning ${{ inputs.tag }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'MLRun image tag to scan (unstable-cache, unstable-cache-13x, 1.3.0-wwwwwwww, 1.3.0 or any other tag)'
+        required: false
+        default: 'unstable'
+      registry:
+        description: 'MLRun image registry'
+        required: false
+        default: 'ghcr.io/'
+      skip_images:
+        description: 'Comma separated list of images to skip scanning'
+        required: false
+
+        # disabling gpu images for now as scanning them takes more disk space than we can afford
+        # test images are not scanned as they are not used in production
+        default: 'test,models-gpu'
+      publish_results:
+        description: 'Whether to publish results to Github or not (default empty - no publish)'
+        required: false
+        default: ''
+      severity_threshold:
+        description: 'The minimum severity of vulnerabilities to report ("negligible", "low", "medium", "high" and "critical".)'
+        required: false
+        default: 'medium'
+      only_fixed:
+        description: 'Whether to scan only fixed vulnerabilities ("true" or "false")'
+        required: false
+        default: 'true'
+
+jobs:
+  matrix_prep:
+    name: Prepare image list
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - id: set-matrix
+        uses: ./.github/actions/image-matrix-prep
+        with:
+          skip_images: ${{ github.event.inputs.skip_images }}
+
+  build_and_scan_docker_images:
+    name: Scan ${{ matrix.image-name }} (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    needs: matrix_prep
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Cleanup disk
+        run: |
+          "${GITHUB_WORKSPACE}/automation/scripts/github_workflow_free_space.sh"
+
+      - name: Resolving image name
+        id: resolve_image_name
+        run: |
+          echo "image_name=$(make pull-${{ matrix.image-name }} | tail -1)" >> $GITHUB_OUTPUT
+        env:
+          MLRUN_DOCKER_REGISTRY: ${{ github.event.inputs.registry }}
+          MLRUN_VERSION: ${{ github.event.inputs.tag }}
+          MLRUN_PYTHON_VERSION: ${{ matrix.python-version }}
+
+      - name: Define output format
+        id: output-format
+
+        # this section is duplicated in the other jobs.
+        # make sure to update all when changed.
+        run: |
+          if [[ -n "${{ github.event.inputs.publish_results }}" ]]; then \
+            echo "format=sarif" >> $GITHUB_OUTPUT; \
+            echo "fail_build=false" >> $GITHUB_OUTPUT; \
+          else \
+            echo "format=table" >> $GITHUB_OUTPUT; \
+            echo "fail_build=true" >> $GITHUB_OUTPUT; \
+          fi
+
+      - name: Scan image
+        uses: anchore/scan-action@v3
+        id: scan
+        with:
+          image: ${{ steps.resolve_image_name.outputs.image_name }}
+          only-fixed: ${{ github.event.inputs.only_fixed }}
+          output-format: ${{ steps.output-format.outputs.format }}
+          fail-build: ${{ steps.output-format.outputs.fail_build }}
+          severity-cutoff: ${{ github.event.inputs.severity_threshold }}
+
+      - name: Upload scan results
+        if: github.event.inputs.publish_results != ''
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: ${{ steps.scan.outputs.sarif }}
+          category: ${{ matrix.image-name }}-${{ matrix.python-version }})
+
+  scan_fs:
+    name: Scan file system
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Define output format
+        id: output-format
+
+        # this section is duplicated in the other jobs.
+        # make sure to update all when changed.
+        run: |
+          if [[ -n "${{ github.event.inputs.publish_results }}" ]]; then \
+            echo "format=sarif" >> $GITHUB_OUTPUT; \
+            echo "fail_build=false" >> $GITHUB_OUTPUT; \
+          else \
+            echo "format=table" >> $GITHUB_OUTPUT; \
+            echo "fail_build=true" >> $GITHUB_OUTPUT; \
+          fi
+
+      - name: Scan fs
+        uses: anchore/scan-action@v3
+        id: scan
+        with:
+          path: "."
+          only-fixed: ${{ github.event.inputs.only_fixed }}
+          output-format: ${{ steps.output-format.outputs.format }}
+          fail-build: ${{ steps.output-format.outputs.fail_build }}
+          severity-cutoff: ${{ github.event.inputs.severity_threshold }}
+
+      - name: Upload scan results
+        if: github.event.inputs.publish_results != ''
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: ${{ steps.scan.outputs.sarif }}
+          category: "repository"