Merge branch 'main' into improvement/gitlab

.github/workflows/actions/build-docker-image/action.yml

@@ -0,0 +1,81 @@
+name: Build docker images
+description: Build Docker Images
+# NOTE: In composite actions, all parameters are strings,
+# thus flags are simply checked by being non empty strings,
+# where there the default is an empty string
+inputs:
+  dockerfile:
+    description: Dockerfile to build
+    required: true
+  tags:
+    description: Docker tags to publish
+    required: true
+  platforms:
+    description: Platforms to build (csv)
+    required: false
+    default: 'linux/arm64,linux/amd64'
+  test:
+    description: Test command to run on the created image (Optional)
+    required: false
+    default: ''
+  build-args:
+    description: Explicit docker build-args
+    required: false
+    default: ''
+  skip-init:
+    description: Skip docker init (if ran after another invocation of this action)
+    required: false
+    default: ''
+  docker-user:
+    required: true
+    description: Docker Hub User
+  docker-password:
+    required: true
+    description: Docker Hub User
+  skip-push:
+    required: false
+    description: Optionally skip push
+    default: ''
+  load-created-image:
+    required: false
+    description: Optionally load created docker image
+    default: ''
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+      if: ${{ inputs.skip-init == '' }}
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      if: ${{ inputs.skip-init == '' }}
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v3
+      if: ${{ inputs.skip-init == '' }}
+      with:
+        registry: ghcr.io
+        username: ${{ inputs.docker-user }}
+        password: ${{ inputs.docker-password }}
+
+    - name: Build Runner Image
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ${{ inputs.dockerfile }}
+        platforms: ${{ inputs.platforms }}
+        push: ${{ inputs.skip-push == '' }}
+        load: ${{ inputs.test != '' || inputs.load-created-image != '' }}
+        tags: ${{ inputs.tags }}
+        build-args: |
+          ${{ inputs.build-args }}
+
+    - name: Verify Built Image
+      shell: bash
+      if: ${{ inputs.test != '' }}
+      run: |
+        SINGLE_TAG=$(echo "${{ inputs.tags }}" | awk -F ',' '{print $1};' )
+        SINGLE_PLATFORM=$(echo "${{ inputs.platforms }}" | awk -F ',' '{print $1};' )
+        docker run --platform "${SINGLE_PLATFORM}" --rm --entrypoint bash "${SINGLE_TAG}" -c '${{ inputs.test }}'

.github/workflows/apply-release.yml

@@ -41,7 +41,7 @@ jobs:
           git config --local user.email "[email protected]"
           git config --local user.name "GitHub Action"
 
-          ./scripts/bump-all.sh ^${{ steps.version.outputs.version }}
+          ./scripts/bump-all.sh ${{ steps.version.outputs.version }}
 
 
       - name: Open pull request

.github/workflows/build-infra-images.yml

@@ -0,0 +1,34 @@
+name: Build infra images
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  detect-changes:
+    uses: ./.github/workflows/detect-changes-matrix.yml
+  build-infra:
+    runs-on: 'ubuntu-latest'
+    needs: detect-changes
+    if: ${{ needs.detect-changes.outputs.infra == 'true' }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Build Docker Image
+        uses: ./.github/workflows/actions/build-docker-image
+        with:
+          dockerfile: ./integrations/_infra/Dockerfile.base.builder
+          platforms: linux/amd64,linux/arm64
+          tags: ghcr.io/port-labs/port-ocean-base-builder:latest
+          docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
+          docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+
+      - name: Build Docker Image
+        uses: ./.github/workflows/actions/build-docker-image
+        with:
+          dockerfile: ./integrations/_infra/Dockerfile.base.runner
+          platforms: linux/amd64,linux/arm64
+          tags: ghcr.io/port-labs/port-ocean-base-runner:latest
+          docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
+          docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+          skip-init: 'true'

.github/workflows/ci.yml

@@ -37,7 +37,6 @@ jobs:
           echo $(echo ${integrations_to_build[@]} | jq -R -c 'split(" ")')
           echo "INTEGRATIONS_MATRIX=$(echo ${integrations_to_build[@]} | jq -R -c 'split(" ")')" >> $GITHUB_OUTPUT
 
-
   build-integration:
     runs-on: ubuntu-latest
     if: needs.prepare-matrix.outputs.matrix != '[]'
@@ -47,17 +46,16 @@ jobs:
       contents: read
     needs: [prepare-matrix]
     strategy:
+      max-parallel: 10
       matrix:
         integration: ${{fromJson(needs.prepare-matrix.outputs.matrix)}}
+        platform:
+          - linux/amd64
+          - linux/arm64
     steps:
       - name: Check out code
         uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: linux/amd64,linux/arm64
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+
       - name: Prepare Docker images tags
         id: prepare_tags
         run: |
@@ -73,6 +71,7 @@ jobs:
 
           dockerfile_path=integrations/_infra/Dockerfile
           if test -e $folder/../Dockerfile; then
+            echo "Choosing a custom Dockerfile for ${{ matrix.integration }}"
             dockerfile_path=$folder/../Dockerfile
           fi
           echo "dockerfile_path=$dockerfile_path" >> $GITHUB_OUTPUT
@@ -91,14 +90,15 @@ jobs:
             echo "is_dev_version=false" >> $GITHUB_OUTPUT
           fi
 
-      - name: Build
-        uses: docker/build-push-action@v6
+      - name: Build Docker Image
+        uses: ./.github/workflows/actions/build-docker-image
         with:
-          context: .
-          file: ${{ steps.prepare_tags.outputs.dockerfile_path }}
-          platforms: linux/amd64,linux/arm64
-          push: false
+          dockerfile: ${{ steps.prepare_tags.outputs.dockerfile_path }}
+          platforms: ${{ matrix.platform }}
           tags: ${{ steps.prepare_tags.outputs.tags }}
           build-args: |
             BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }}
             INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }}
+          docker-user: ${{ secrets.DOCKER_MACHINE_USER }}
+          docker-password: ${{ secrets.DOCKER_MACHINE_TOKEN }}
+          skip-push: 'true'

.github/workflows/core-test.yml

@@ -0,0 +1,88 @@
+name: 🌊 Ocean Core Tests
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  detect-changes:
+    uses: ./.github/workflows/detect-changes-matrix.yml
+  test:
+    name: 🌊 Ocean Core Tests
+    needs: detect-changes
+    runs-on: ubuntu-latest
+    if: ${{ needs.detect-changes.outputs.core == 'true' }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'poetry'
+
+      - name: Install dependencies
+        run: |
+          make install
+
+      - name: Unit Test Core
+        env:
+          PYTEST_ADDOPTS: --junitxml=junit/unit-test-results-ocean/core.xml
+        run: |
+          make test
+
+      - name: Build core for smoke test
+        run: |
+          make build
+
+      - name: Run fake integration for core test
+        env:
+          PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }}
+          PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }}
+          PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }}
+          SMOKE_TEST_SUFFIX: ${{ github.run_id }}
+        run: |
+          ./scripts/run-smoke-test.sh
+
+      - name: Smoke Test Core
+        env:
+          PYTEST_ADDOPTS: --junitxml=junit/smoke-test-results-ocean/core.xml
+          PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }}
+          PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }}
+          PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }}
+          SMOKE_TEST_SUFFIX: ${{ github.run_id }}
+        run: |
+          make test/smoke
+
+      - name: Cleanup Smoke Test
+        env:
+          PYTEST_ADDOPTS: --junitxml=junit/smoke-test-results-ocean/core.xml
+          PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }}
+          PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }}
+          PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }}
+          SMOKE_TEST_SUFFIX: ${{ github.run_id }}
+        run: |
+          make test/smoke
+
+      - name: Install current core for all integrations
+        run: |
+          echo "Installing local core for all integrations"
+          SCRIPT_TO_RUN='make install/local-core' make execute/all
+
+      - name: Test all integrations with current core
+        run: |
+          echo "Testing all integrations with local core"
+          SCRIPT_TO_RUN="PYTEST_ADDOPTS=--junitxml=${PWD}/junit/test-results-core-change/\`pwd | xargs basename\`.xml make test" make execute/all
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: ${{ always() }}
+        with:
+          report_paths: '**/junit/**-test-results-**/*.xml'
+          include_passed: true
+          require_tests: true
+          fail_on_failure: true

.github/workflows/create-new-sonarcloud-project.yml

@@ -20,7 +20,7 @@ jobs:
           fetch-depth: 0
       - name: Get all changed integrations
         id: changed-integrations
-        uses: tj-actions/changed-files@v44
+        uses: tj-actions/changed-files@v45
         with:
           json: true
           dir_names: true
@@ -42,6 +42,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Create new project for integrations
     strategy:
+      max-parallel: 5
       matrix: ${{ fromJson(needs.pre-run.outputs.changed_integrations) }}
     steps:
       - name: Create integration variable

.github/workflows/detect-changes-matrix.yml

@@ -0,0 +1,66 @@
+name: Detect Changes
+on:
+  workflow_call:
+    outputs:
+      matrix:
+        value: ${{ jobs.detect-changes.outputs.matrix }}
+        description: "Matrix of changed integrations / Ocean Core per git commit changes"
+      integrations:
+        description: "Matrix of changed integrations per git commit changes"
+        value: ${{ jobs.detect-changes.outputs.integrations }}
+      core:
+        value: ${{ jobs.detect-changes.outputs.core }}
+        description: "Determine if any core changes per git commit changes"
+      infra:
+        value: ${{ jobs.detect-changes.outputs.infra }}
+        description: "Determine if any changes to docker infra"
+
+jobs:
+  detect-changes:
+    name: Detect changes
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-all-matrix.outputs.matrix }}
+      integrations: ${{ steps.set-all-matrix.outputs.integrations }}
+      core: ${{ steps.set-all-matrix.outputs.core }}
+      infra: ${{ steps.set-all-matrix.outputs.infra}}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - name: Get list of changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v45
+        with:
+          dir_names: true
+          json: true
+          dir_names_max_depth: 2
+          escape_json: false
+          files_yaml: |
+            core:
+              - '!integrations/**'
+              - '!scripts/*'
+              - '!scripts/*'
+              - '!./*.md'
+            integrations:
+              - 'integrations/**'
+              - '!integrations/**/*.md'
+              - '!integrations/_infra/*'
+            infra:
+              - 'integrations/_infra/*'
+
+      - name: Set integrations and all matrix
+        id: set-all-matrix
+        run: |
+          INTEGRATIONS=$(node -e 'integrations=${{ steps.changed-files.outputs.integrations_all_changed_files }};console.log(JSON.stringify(integrations.map(integration => integration.split("/")[1])))')
+          HAS_CORE=${{ steps.changed-files.outputs.core_all_changed_files != '[]' }}
+          echo "Core changes : ${HAS_CORE}"
+          MATRIX=$(node -e "integrations=${INTEGRATIONS}; hasCore=${HAS_CORE}; console.log(JSON.stringify(hasCore ? integrations.concat(['.']) : integrations))")
+          HAS_INFRA=${{ steps.changed-files.outputs.infra_all_changed_files != '[]' }}
+          echo "Infra changes : ${HAS_INFRA}"
+          echo "Integration changes : ${INTEGRATIONS}"
+          echo "All changes : ${MATRIX}"
+          echo "core=${HAS_CORE}" >> $GITHUB_OUTPUT
+          echo "integrations=${INTEGRATIONS}" >> $GITHUB_OUTPUT
+          echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT
+          echo "infra=${HAS_INFRA}" >> $GITHUB_OUTPUT