[feature] Multi-repo workflow

README.md

@@ -21,6 +21,35 @@ Build:
   extends: .build
 ```
 
-> Instead of `/main/`, you can specify a specific commit to ensure changes do not affect your CI. 
+> Instead of `/main/`, you can specify a specific commit to ensure changes do not affect your CI.
 
 The [`examples`](examples/) folder contains examples of `.gitlab-ci.yml` that can be assembled from the templates.
+
+## Multi-repository templates
+
+In `templates/multi-repo` the CI workflow differs from `basic` CI (which in `templates`) in the following key aspects:
+
+- In `multi-repo` workflow we can push to `dev` and `prod` registries separately with their own rules (see `jobs/multi-repo` and/or `examples/multi-repo-module.gitlab-ci.yml` for example jobs).
+- All werf's caches and other artifacts (from `build` stage) are stored in Gitlab's module's registry by default. And **only final images** are pushed to the dev/prod registries. So, even in dev-registry there **should be no** "build-time garbage" and/or some "extra" images/layers for each module.
+
+### Detailed differences between `multi-repo` and `basic` workflows
+
+- [General] There is additional stage `lint` before `build` and `cleanup` stage after `deploy`.
+- [General] All `only` sections (like `only: [tags, branches]`) replaced with corresponding `rules` section.
+- [General] Added `Scheduled cleanup` job to cleanup Gitlab's registry by pipeline schedule
+- [General] Added `Auto cleanup` job to cleanup Gitlab's registry BEFORE `build` stage. Can be disabled via `AUTO_CLEANUP="false"` variable.
+- [General] Added `.default_rules` hidden job (see `templates/multi-repo/Setup.gitlab-ci.yml`) for easy modification of this whole workflow.
+- [General] Added `.deploy-prod-rules` hidden job (see `templates/multi-repo/Deploy.gitlab-ci.yml`) for easy modification of `deploy to production` workflow.
+- [General] Added `jobs/multi-repo` jobs files which user can include and use in their own workflow.
+- [General] Added ability to specify which module's `EDITION` (`CE`, `EE`, etc) should be pushed to PRODUCTION registry.
+- [Refactor] Default `before_script` section (see `templates/Setup.gitlab-ci.yml`) moved to `.setup/before_script` job.
+- [Refactor] `dmt lint` job moved to `lint` stage in dedicated `templates/multi-repo/Lint.gitlab-ci.yml` file.
+- [Refactor] All werf's caches and other artifacts (from `build` stage) are stored in Gitlab's registry (`${CI_REGISTRY_IMAGE}/${MODULES_MODULE_NAME}`) by default.
+- [Refactor] Images publishing (via `crane copy`) and module's self-registration processes moved to dedicated hidden job `.publish` (see `templates/multi-repo/Deploy.gitlab-ci.yml`).
+
+## Variables
+
+`$MODULES_REGISTRY` - base URL for the registry, e.g. `registry.example.com`
+`$MODULES_REGISTRY_PATH` - path to modules repository in registry, e.g. `deckhouse/modules`
+`$MODULES_MODULE_NAME` (Optional) - module name, by default it is equal to the project name
+`$RELEASE_CHANNEL` - lowercase release channel name, e.g., `alpha`, `stable`, `early-access`

examples/multi-repo-module.gitlab-ci.yml

@@ -0,0 +1,59 @@
+include:
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Setup.gitlab-ci.yml'
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Lint.gitlab-ci.yml'
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Build.gitlab-ci.yml'
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Deploy.gitlab-ci.yml'
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Cleanup.gitlab-ci.yml'
+  # deploy jobs for DEV registry
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/jobs/multi-repo/Deploy_DEV.gitlab-ci.yml'
+  # deploy jobs for PROD registry
+  - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/jobs/multi-repo/Deploy_PROD.gitlab-ci.yml'
+    inputs:
+      # Editions used in your module. Array of following items:
+      # ce - Community edition
+      # ee - Enterprise edition
+      # fe - Flant edition (internal edition for Flant's engineers)
+      # se - Standard edition
+      # se-plus - Standard edition +
+      editions:
+        # All values must be in lowercase and quoted
+        - "ce"
+        - "ee"
+        - "fe"
+        - "se"
+        - "se-plus"
+
+variables:
+  # Do not forget to put these variables to your Gitlab CI secrets:
+  # They are REQUIRED and used for pulling/pushing images to the corresponding registry
+  # - DEV_MODULES_REGISTRY: DEV registry domain (like: registry.example.com)
+  # - DEV_MODULES_REGISTRY_PATH: path to modules repository in DEV registry (like: deckhouse/modules)
+  # - DEV_MODULES_REGISTRY_LOGIN: username to log in to DEV registry
+  # - DEV_MODULES_REGISTRY_PASSWORD: password to log in to DEV registry
+
+  # WARNING: If some of following variables are NOT SET, then there is NO production deployment jobs will be created in pipeline
+  # - PROD_MODULES_REGISTRY: PROD registry domain (like: registry.example.com)
+  # - PROD_MODULES_REGISTRY_PATH: path to modules repository in PROD registry (like: deckhouse/modules)
+  # - PROD_MODULES_REGISTRY_LOGIN: username to log in to PROD registry
+  # - PROD_MODULES_REGISTRY_PASSWORD: password to log in to PROD registry
+  WERF_VERSION: "2 stable"
+  BASE_IMAGES_VERSION: v0.2
+
+default:
+  tags:
+    - my-runner-tag
+
+
+###### LINT STAGE ######
+
+Lint:
+  extends: .lint
+
+###### END OF LINT STAGE ######
+
+###### BUILD STAGE ######
+
+Build:
+  extends: .build
+
+###### END OF BUILD STAGE ######

jobs/multi-repo/Debug.gitlab-ci.yml

@@ -0,0 +1,13 @@
+variables:
+  DEBUG_CI:
+    value: "false"
+    description: "Run debug job(s)"
+
+debug:printenv:
+  stage: build
+  rules:
+    # run if $DEBUG_CI variable is set to true
+    - if: $DEBUG_CI == "true" || $DEBUG_CI == "1"
+  script:
+    - |
+      printenv | sort

jobs/multi-repo/Deploy_DEV.gitlab-ci.yml

@@ -0,0 +1,66 @@
+# emulate same behaviour as in Deckhouse Github registry
+# when opened PRs will pushed to dev registry
+DEV | Publish merge request:
+  extends: .publish
+  variables:
+    MODULES_REGISTRY: ${DEV_MODULES_REGISTRY}
+    MODULES_REGISTRY_PATH: ${DEV_MODULES_REGISTRY_PATH}
+    MODULES_REGISTRY_LOGIN: ${DEV_MODULES_REGISTRY_LOGIN}
+    MODULES_REGISTRY_PASSWORD: ${DEV_MODULES_REGISTRY_PASSWORD}
+    # names as in Github: "pr" + merge request project-level ID instead of branch name
+    MODULES_MODULE_TAG: pr${CI_MERGE_REQUEST_IID}
+  rules:
+    # do not run if some required variables is empty
+    - if: '$DEV_MODULES_REGISTRY == null || $DEV_MODULES_REGISTRY == "" || $DEV_MODULES_REGISTRY_PATH == null || $DEV_MODULES_REGISTRY_PATH == ""'
+      when: never
+    # run only for merge requests
+    - if: $CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME != $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: on_success
+    # run when new branch is created and there are no opened merge requests for this branch and no commits to this branch yet (completely new branch from master/main)
+    # - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && ($CI_MERGE_REQUEST_IID == null || $CI_MERGE_REQUEST_IID == "") && $CI_COMMIT_BEFORE_SHA == "0000000000000000000000000000000000000000"
+    #   when: on_success
+    # do not run in other cases
+    - when: never
+
+DEV | Publish default branch:
+  extends: .publish
+  variables:
+    MODULES_REGISTRY: ${DEV_MODULES_REGISTRY}
+    MODULES_REGISTRY_PATH: ${DEV_MODULES_REGISTRY_PATH}
+    MODULES_REGISTRY_LOGIN: ${DEV_MODULES_REGISTRY_LOGIN}
+    MODULES_REGISTRY_PASSWORD: ${DEV_MODULES_REGISTRY_PASSWORD}
+    MODULES_MODULE_TAG: ${CI_DEFAULT_BRANCH}
+  rules:
+    # do not run if some required variables is empty
+    - if: '$DEV_MODULES_REGISTRY == null || $DEV_MODULES_REGISTRY == "" || $DEV_MODULES_REGISTRY_PATH == null || $DEV_MODULES_REGISTRY_PATH == ""'
+      when: never
+    # run only when push to default (main/master) branch
+    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: on_success
+    # do not run in other cases
+    - when: never
+
+DEV | Publish tags also to dev-registry:
+  stage: deploy
+  variables:
+    MODULES_REGISTRY: ${DEV_MODULES_REGISTRY}
+    MODULES_REGISTRY_PATH: ${DEV_MODULES_REGISTRY_PATH}
+    MODULES_REGISTRY_LOGIN: ${DEV_MODULES_REGISTRY_LOGIN}
+    MODULES_REGISTRY_PASSWORD: ${DEV_MODULES_REGISTRY_PASSWORD}
+    MODULES_MODULE_TAG: ${CI_COMMIT_TAG}
+  rules:
+    # do not run if some required variables is empty
+    - if: '$DEV_MODULES_REGISTRY == null || $DEV_MODULES_REGISTRY == "" || $DEV_MODULES_REGISTRY_PATH == null || $DEV_MODULES_REGISTRY_PATH == ""'
+      when: never
+    # deploy tags to dev-registry (as in prod registry) when tag specified
+    - if: '$CI_COMMIT_TAG && ($NO_DEPLOY_TAGS_TO_DEV == null || $NO_DEPLOY_TAGS_TO_DEV == "")'
+      when: on_success
+    # do not run in other cases
+    - when: never
+  script:
+    - |
+      if [ "$DEBUG_CI" = "true" -o "$DEBUG_CI" = "1" ]; then
+        printenv | sort
+      fi
+    # publish final images to dev registry and register module with $MODULES_MODULE_TAG
+    - !reference [.publish, script]

jobs/multi-repo/Deploy_PROD.gitlab-ci.yml

@@ -0,0 +1,54 @@
+# https://docs.gitlab.com/ci/inputs/
+spec:
+  inputs:
+    editions:
+      type: array
+      description: List of module editions
+      default:
+        - ee
+        - fe
+        - se
+        - se-plus
+
+---
+
+PROD | Alpha:
+  extends: .deploy_prod
+  variables:
+    RELEASE_CHANNEL: alpha
+  parallel:
+    matrix:
+      - EDITION: $[[ inputs.editions ]]
+
+PROD | Beta:
+  extends: .deploy_prod
+  variables:
+    RELEASE_CHANNEL: beta
+  parallel:
+    matrix:
+      - EDITION: $[[ inputs.editions ]]
+
+PROD | EarlyAccess:
+  extends: .deploy_prod
+  variables:
+    RELEASE_CHANNEL: early-access
+  parallel:
+    matrix:
+      - EDITION: $[[ inputs.editions ]]
+
+PROD | Stable:
+  extends: .deploy_prod
+  variables:
+    RELEASE_CHANNEL: stable
+  parallel:
+    matrix:
+      - EDITION: $[[ inputs.editions ]]
+
+# because uppercased letters are ordered before lowercased, so put rock-solid job last as in stability level, not alphabetical
+PROD | rock-solid:
+  extends: .deploy_prod
+  variables:
+    RELEASE_CHANNEL: rock-solid
+  parallel:
+    matrix:
+      - EDITION: $[[ inputs.editions ]]

templates/CVE_Scan.gitlab-ci.yml

@@ -40,8 +40,23 @@
       echo "Preparing DOCKER_CONFIG and login to registries"
       mkdir -p "${workdir}/docker"
       export DOCKER_CONFIG="${workdir}/docker"
-      echo ${PROD_REGISTRY_PASSWORD} | docker login --username="${PROD_REGISTRY_USER}" --password-stdin ${PROD_REGISTRY}
-      echo ${DEV_REGISTRY_PASSWORD} | docker login --username="${DEV_REGISTRY_USER}" --password-stdin ${DEV_REGISTRY}
+
+      PROD_AUTH_STRING=$(echo -n "$PROD_REGISTRY_USER:$PROD_REGISTRY_PASSWORD" | base64 -w 0)
+      DEV_AUTH_STRING=$(echo -n "$DEV_REGISTRY_USER:$DEV_REGISTRY_PASSWORD" | base64 -w 0)
+
+      # Create config.json file
+      cat > ${DOCKER_CONFIG}/config.json << EOF
+      {
+        "auths": {
+          "$PROD_REGISTRY": {
+            "auth": "$PROD_AUTH_STRING"
+          },
+          "${DEV_REGISTRY}": {
+            "auth": "$DEV_AUTH_STRING"
+          }
+        }
+      }
+      EOF
       echo
       echo "======================================================="
       echo

templates/Setup.gitlab-ci.yml

@@ -110,4 +110,4 @@ before_script:
 
 stages:
   - build
-  - deploy
+  - deploy

templates/multi-repo/Build.gitlab-ci.yml

@@ -0,0 +1,33 @@
+.build:
+  stage: build
+  rules:
+    - !reference [.default_rules, rules]
+  before_script:
+    - !reference [.setup, before_script]
+  script:
+    # Build images
+    - |
+      werf build \
+        --save-build-report --build-report-path images_tags_werf.json
+  artifacts:
+    paths:
+      - images_tags_werf.json
+    expire_in: "30 days"
+
+.svace_rules_mr:
+  rules:
+    - if: '$CI_MERGE_REQUEST_LABELS =~ /(^|,)analyze\/svace(,|$)/'
+      variables:
+        SVACE_ENABLED: "true"
+
+.svace_rules_manual:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "web" && $SVACE_ENABLED == "true" && $CI_COMMIT_BRANCH
+      variables:
+        SVACE_ENABLED: "true"
+
+.svace_rules_schedule:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $SVACE_ENABLED == "true" && $CI_COMMIT_BRANCH
+      variables:
+        SVACE_ENABLED: "true"

templates/multi-repo/Cleanup.gitlab-ci.yml

@@ -0,0 +1,52 @@
+Scheduled cleanup:
+  stage: cleanup
+  timeout: 3h
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: on_success
+    - when: never
+  before_script:
+    - !reference [.setup, before_script]
+  script:
+    - |
+      if [[ -z "${NO_PRIVATE_REPO_PATCH}" ]]; then
+        echo "Apply git private repo patch... Set NO_PRIVATE_REPO_PATCH=1 to disable it"
+        export GOPRIVATE=${CI_SERVER_HOST}
+        git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/".insteadOf "git@${CI_SERVER_HOST}:"
+      fi
+
+      echo "Managed images which will be preserved during cleanup procedure:"
+      werf managed-images ls
+      echo "Starting cleanup..."
+      werf cleanup
+
+Auto cleanup:
+  stage: cleanup
+  allow_failure: true
+  timeout: 10 minutes
+  rules:
+    # do not run if this job is explicitly disabled by user
+    - if: $AUTO_CLEANUP == "false" || $AUTO_CLEANUP == "0" || $AUTO_CLEANUP == ""
+      when: never
+    # do not run if there is a tag (release workflow)
+    - if: $CI_COMMIT_TAG
+      when: never
+    - !reference [.default_rules, rules]
+  before_script:
+    - !reference [.setup, before_script]
+  script:
+    - |
+      if (( $(date +%s) % 10 == 0 )); then
+        echo "✨ Run auto cleanup"
+
+        if [[ -z "${NO_PRIVATE_REPO_PATCH}" ]]; then
+          echo "Apply git private repo patch... Set NO_PRIVATE_REPO_PATCH=1 to disable it"
+          export GOPRIVATE=${CI_SERVER_HOST}
+          git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/".insteadOf "git@${CI_SERVER_HOST}:"
+        fi
+
+        echo "Managed images which will be preserved during cleanup procedure:"
+        werf managed-images ls
+        echo "Starting cleanup..."
+        werf cleanup
+      fi