diff --git a/README.md b/README.md index 7d93525..7819b6a 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,35 @@ Build: extends: .build ``` -> Instead of `/main/`, you can specify a specific commit to ensure changes do not affect your CI. +> Instead of `/main/`, you can specify a specific commit to ensure changes do not affect your CI. The [`examples`](examples/) folder contains examples of `.gitlab-ci.yml` that can be assembled from the templates. + +## Multi-repository templates + +In `templates/multi-repo` the CI workflow differs from `basic` CI (which in `templates`) in the following key aspects: + +- In `multi-repo` workflow we can push to `dev` and `prod` registries separately with their own rules (see `jobs/multi-repo` and/or `examples/multi-repo-module.gitlab-ci.yml` for example jobs). +- All werf's caches and other artifacts (from `build` stage) are stored in Gitlab's module's registry by default. And **only final images** are pushed to the dev/prod registries. So, even in dev-registry there **should be no** "build-time garbage" and/or some "extra" images/layers for each module. + +### Detailed differences between `multi-repo` and `basic` workflows + +- [General] There is additional stage `lint` before `build` and `cleanup` stage after `deploy`. +- [General] All `only` sections (like `only: [tags, branches]`) replaced with corresponding `rules` section. +- [General] Added `Scheduled cleanup` job to cleanup Gitlab's registry by pipeline schedule +- [General] Added `Auto cleanup` job to cleanup Gitlab's registry BEFORE `build` stage. Can be disabled via `AUTO_CLEANUP="false"` variable. +- [General] Added `.default_rules` hidden job (see `templates/multi-repo/Setup.gitlab-ci.yml`) for easy modification of this whole workflow. +- [General] Added `.deploy-prod-rules` hidden job (see `templates/multi-repo/Deploy.gitlab-ci.yml`) for easy modification of `deploy to production` workflow. +- [General] Added `jobs/multi-repo` jobs files which user can include and use in their own workflow. +- [General] Added ability to specify which module's `EDITION` (`CE`, `EE`, etc) should be pushed to PRODUCTION registry. +- [Refactor] Default `before_script` section (see `templates/Setup.gitlab-ci.yml`) moved to `.setup/before_script` job. +- [Refactor] `dmt lint` job moved to `lint` stage in dedicated `templates/multi-repo/Lint.gitlab-ci.yml` file. +- [Refactor] All werf's caches and other artifacts (from `build` stage) are stored in Gitlab's registry (`${CI_REGISTRY_IMAGE}/${MODULES_MODULE_NAME}`) by default. +- [Refactor] Images publishing (via `crane copy`) and module's self-registration processes moved to dedicated hidden job `.publish` (see `templates/multi-repo/Deploy.gitlab-ci.yml`). + +## Variables + +`$MODULES_REGISTRY` - base URL for the registry, e.g. `registry.example.com` +`$MODULES_REGISTRY_PATH` - path to modules repository in registry, e.g. `deckhouse/modules` +`$MODULES_MODULE_NAME` (Optional) - module name, by default it is equal to the project name +`$RELEASE_CHANNEL` - lowercase release channel name, e.g., `alpha`, `stable`, `early-access` diff --git a/examples/multi-repo-module.gitlab-ci.yml b/examples/multi-repo-module.gitlab-ci.yml new file mode 100644 index 0000000..5a3a4b5 --- /dev/null +++ b/examples/multi-repo-module.gitlab-ci.yml @@ -0,0 +1,59 @@ +include: + - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Setup.gitlab-ci.yml' + - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Lint.gitlab-ci.yml' + - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Build.gitlab-ci.yml' + - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Deploy.gitlab-ci.yml' + - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/templates/multi-repo/Cleanup.gitlab-ci.yml' + # deploy jobs for DEV registry + - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/jobs/multi-repo/Deploy_DEV.gitlab-ci.yml' + # deploy jobs for PROD registry + - remote: 'https://raw.githubusercontent.com/deckhouse/modules-gitlab-ci/refs/heads/main/jobs/multi-repo/Deploy_PROD.gitlab-ci.yml' + inputs: + # Editions used in your module. Array of following items: + # ce - Community edition + # ee - Enterprise edition + # fe - Flant edition (internal edition for Flant's engineers) + # se - Standard edition + # se-plus - Standard edition + + editions: + # All values must be in lowercase and quoted + - "ce" + - "ee" + - "fe" + - "se" + - "se-plus" + +variables: + # Do not forget to put these variables to your Gitlab CI secrets: + # They are REQUIRED and used for pulling/pushing images to the corresponding registry + # - DEV_MODULES_REGISTRY: DEV registry domain (like: registry.example.com) + # - DEV_MODULES_REGISTRY_PATH: path to modules repository in DEV registry (like: deckhouse/modules) + # - DEV_MODULES_REGISTRY_LOGIN: username to log in to DEV registry + # - DEV_MODULES_REGISTRY_PASSWORD: password to log in to DEV registry + + # WARNING: If some of following variables are NOT SET, then there is NO production deployment jobs will be created in pipeline + # - PROD_MODULES_REGISTRY: PROD registry domain (like: registry.example.com) + # - PROD_MODULES_REGISTRY_PATH: path to modules repository in PROD registry (like: deckhouse/modules) + # - PROD_MODULES_REGISTRY_LOGIN: username to log in to PROD registry + # - PROD_MODULES_REGISTRY_PASSWORD: password to log in to PROD registry + WERF_VERSION: "2 stable" + BASE_IMAGES_VERSION: v0.2 + +default: + tags: + - my-runner-tag + + +###### LINT STAGE ###### + +Lint: + extends: .lint + +###### END OF LINT STAGE ###### + +###### BUILD STAGE ###### + +Build: + extends: .build + +###### END OF BUILD STAGE ###### diff --git a/jobs/multi-repo/Debug.gitlab-ci.yml b/jobs/multi-repo/Debug.gitlab-ci.yml new file mode 100644 index 0000000..376d13f --- /dev/null +++ b/jobs/multi-repo/Debug.gitlab-ci.yml @@ -0,0 +1,13 @@ +variables: + DEBUG_CI: + value: "false" + description: "Run debug job(s)" + +debug:printenv: + stage: build + rules: + # run if $DEBUG_CI variable is set to true + - if: $DEBUG_CI == "true" || $DEBUG_CI == "1" + script: + - | + printenv | sort diff --git a/jobs/multi-repo/Deploy_DEV.gitlab-ci.yml b/jobs/multi-repo/Deploy_DEV.gitlab-ci.yml new file mode 100644 index 0000000..0b5805e --- /dev/null +++ b/jobs/multi-repo/Deploy_DEV.gitlab-ci.yml @@ -0,0 +1,66 @@ +# emulate same behaviour as in Deckhouse Github registry +# when opened PRs will pushed to dev registry +DEV | Publish merge request: + extends: .publish + variables: + MODULES_REGISTRY: ${DEV_MODULES_REGISTRY} + MODULES_REGISTRY_PATH: ${DEV_MODULES_REGISTRY_PATH} + MODULES_REGISTRY_LOGIN: ${DEV_MODULES_REGISTRY_LOGIN} + MODULES_REGISTRY_PASSWORD: ${DEV_MODULES_REGISTRY_PASSWORD} + # names as in Github: "pr" + merge request project-level ID instead of branch name + MODULES_MODULE_TAG: pr${CI_MERGE_REQUEST_IID} + rules: + # do not run if some required variables is empty + - if: '$DEV_MODULES_REGISTRY == null || $DEV_MODULES_REGISTRY == "" || $DEV_MODULES_REGISTRY_PATH == null || $DEV_MODULES_REGISTRY_PATH == ""' + when: never + # run only for merge requests + - if: $CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME != $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "merge_request_event" + when: on_success + # run when new branch is created and there are no opened merge requests for this branch and no commits to this branch yet (completely new branch from master/main) + # - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && ($CI_MERGE_REQUEST_IID == null || $CI_MERGE_REQUEST_IID == "") && $CI_COMMIT_BEFORE_SHA == "0000000000000000000000000000000000000000" + # when: on_success + # do not run in other cases + - when: never + +DEV | Publish default branch: + extends: .publish + variables: + MODULES_REGISTRY: ${DEV_MODULES_REGISTRY} + MODULES_REGISTRY_PATH: ${DEV_MODULES_REGISTRY_PATH} + MODULES_REGISTRY_LOGIN: ${DEV_MODULES_REGISTRY_LOGIN} + MODULES_REGISTRY_PASSWORD: ${DEV_MODULES_REGISTRY_PASSWORD} + MODULES_MODULE_TAG: ${CI_DEFAULT_BRANCH} + rules: + # do not run if some required variables is empty + - if: '$DEV_MODULES_REGISTRY == null || $DEV_MODULES_REGISTRY == "" || $DEV_MODULES_REGISTRY_PATH == null || $DEV_MODULES_REGISTRY_PATH == ""' + when: never + # run only when push to default (main/master) branch + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: on_success + # do not run in other cases + - when: never + +DEV | Publish tags also to dev-registry: + stage: deploy + variables: + MODULES_REGISTRY: ${DEV_MODULES_REGISTRY} + MODULES_REGISTRY_PATH: ${DEV_MODULES_REGISTRY_PATH} + MODULES_REGISTRY_LOGIN: ${DEV_MODULES_REGISTRY_LOGIN} + MODULES_REGISTRY_PASSWORD: ${DEV_MODULES_REGISTRY_PASSWORD} + MODULES_MODULE_TAG: ${CI_COMMIT_TAG} + rules: + # do not run if some required variables is empty + - if: '$DEV_MODULES_REGISTRY == null || $DEV_MODULES_REGISTRY == "" || $DEV_MODULES_REGISTRY_PATH == null || $DEV_MODULES_REGISTRY_PATH == ""' + when: never + # deploy tags to dev-registry (as in prod registry) when tag specified + - if: '$CI_COMMIT_TAG && ($NO_DEPLOY_TAGS_TO_DEV == null || $NO_DEPLOY_TAGS_TO_DEV == "")' + when: on_success + # do not run in other cases + - when: never + script: + - | + if [ "$DEBUG_CI" = "true" -o "$DEBUG_CI" = "1" ]; then + printenv | sort + fi + # publish final images to dev registry and register module with $MODULES_MODULE_TAG + - !reference [.publish, script] diff --git a/jobs/multi-repo/Deploy_PROD.gitlab-ci.yml b/jobs/multi-repo/Deploy_PROD.gitlab-ci.yml new file mode 100644 index 0000000..89bf35a --- /dev/null +++ b/jobs/multi-repo/Deploy_PROD.gitlab-ci.yml @@ -0,0 +1,54 @@ +# https://docs.gitlab.com/ci/inputs/ +spec: + inputs: + editions: + type: array + description: List of module editions + default: + - ee + - fe + - se + - se-plus + +--- + +PROD | Alpha: + extends: .deploy_prod + variables: + RELEASE_CHANNEL: alpha + parallel: + matrix: + - EDITION: $[[ inputs.editions ]] + +PROD | Beta: + extends: .deploy_prod + variables: + RELEASE_CHANNEL: beta + parallel: + matrix: + - EDITION: $[[ inputs.editions ]] + +PROD | EarlyAccess: + extends: .deploy_prod + variables: + RELEASE_CHANNEL: early-access + parallel: + matrix: + - EDITION: $[[ inputs.editions ]] + +PROD | Stable: + extends: .deploy_prod + variables: + RELEASE_CHANNEL: stable + parallel: + matrix: + - EDITION: $[[ inputs.editions ]] + +# because uppercased letters are ordered before lowercased, so put rock-solid job last as in stability level, not alphabetical +PROD | rock-solid: + extends: .deploy_prod + variables: + RELEASE_CHANNEL: rock-solid + parallel: + matrix: + - EDITION: $[[ inputs.editions ]] diff --git a/templates/CVE_Scan.gitlab-ci.yml b/templates/CVE_Scan.gitlab-ci.yml index 1d89231..74788ae 100644 --- a/templates/CVE_Scan.gitlab-ci.yml +++ b/templates/CVE_Scan.gitlab-ci.yml @@ -40,8 +40,23 @@ echo "Preparing DOCKER_CONFIG and login to registries" mkdir -p "${workdir}/docker" export DOCKER_CONFIG="${workdir}/docker" - echo ${PROD_REGISTRY_PASSWORD} | docker login --username="${PROD_REGISTRY_USER}" --password-stdin ${PROD_REGISTRY} - echo ${DEV_REGISTRY_PASSWORD} | docker login --username="${DEV_REGISTRY_USER}" --password-stdin ${DEV_REGISTRY} + + PROD_AUTH_STRING=$(echo -n "$PROD_REGISTRY_USER:$PROD_REGISTRY_PASSWORD" | base64 -w 0) + DEV_AUTH_STRING=$(echo -n "$DEV_REGISTRY_USER:$DEV_REGISTRY_PASSWORD" | base64 -w 0) + + # Create config.json file + cat > ${DOCKER_CONFIG}/config.json << EOF + { + "auths": { + "$PROD_REGISTRY": { + "auth": "$PROD_AUTH_STRING" + }, + "${DEV_REGISTRY}": { + "auth": "$DEV_AUTH_STRING" + } + } + } + EOF echo echo "=======================================================" echo @@ -100,20 +115,29 @@ # Functions trivy_scan() { - ${workdir}/bin/trivy i --policy "${TRIVY_POLICY_URL}" --cache-dir "${workdir}/bin/trivy_cache" --skip-db-update --skip-java-db-update --exit-code 0 --severity "${severity}" --ignorefile "${module_workdir}/.trivyignore" --format ${1} ${2} ${3} --quiet ${4} --username "${trivy_registry_user}" --password "${trivy_registry_pass}" --image-src remote + ${workdir}/bin/trivy i --vex oci --show-suppressed --config-check "${TRIVY_POLICY_URL}" --cache-dir "${workdir}/bin/trivy_cache" --skip-db-update --skip-java-db-update --exit-code 0 --severity "${severity}" --ignorefile "${module_workdir}/.trivyignore" --format ${1} ${2} ${3} --quiet ${4} --username "${trivy_registry_user}" --password "${trivy_registry_pass}" --image-src remote } send_report() { + dd_scan_type="${1}" + dd_report_file_path="${2}" + dd_module_name="${3}" + dd_image_name="${4}" + dd_engagement_name="[$(echo "${dd_scan_type}" | tr '[:lower:]' '[:upper:]')] [IMAGES] [${dd_branch}]" + + tags_string="\"external_modules\",\"images\",\"${dd_scan_type}\",\"${dd_release_or_dev_tag}\",\"${dd_image_version}\"" + if [[ -n "${dd_short_release_tag}" && -n "${dd_full_release_tag}" ]]; then + tags_string+=",\"${dd_short_release_tag}\",\"${dd_full_release_tag}\"" + fi echo "" - echo " Uploading trivy ${1} report for image \"${IMAGE_NAME}\" of \"${MODULE_NAME}\" module" + echo " Uploading trivy ${dd_branch} report for image \"${dd_image_name}\" of \"${dd_module_name}\" module" echo "" - curl -s -S -o /dev/null --fail-with-body -X POST \ - --retry 5 \ - --retry-delay 10 \ + dd_upload_response=$(curl -sw "%{http_code}" -X POST \ + --retry 10 \ + --retry-delay 20 \ --retry-all-errors \ ${DD_URL}/api/v2/reimport-scan/ \ -H "accept: application/json" \ - -H "Content-Type: multipart/form-data" \ -H "Authorization: Token ${DD_TOKEN}" \ -F "auto_create_context=True" \ -F "minimum_severity=Info" \ @@ -123,21 +147,57 @@ -F "close_old_findings=true" \ -F "do_not_reactivate=false" \ -F "push_to_jira=false" \ - -F "file=@${2}" \ + -F "file=@${dd_report_file_path}" \ -F "product_type_name=External Modules" \ - -F "product_name=$MODULE_NAME" \ + -F "product_name=${dd_module_name}" \ -F "scan_date=${date_iso}" \ - -F "engagement_name=${1}" \ - -F "service=${MODULE_NAME} / ${IMAGE_NAME}" \ + -F "engagement_name=${dd_engagement_name}" \ + -F "service=${dd_module_name} / ${dd_image_name}" \ -F "group_by=component_name+component_version" \ -F "deduplication_on_engagement=false" \ - -F "tags=external_module,module:${MODULE_NAME},image:${IMAGE_NAME},branch:${module_tag},${dd_short_release_tag},${dd_full_release_tag},${dd_default_branch_tag}" \ - -F "test_title=[${MODULE_NAME}]: ${IMAGE_NAME}:${module_tag}" \ + -F "tags=external_module,${dd_scan_type},module:${dd_module_name},image:${dd_image_name},branch:${dd_branch},${dd_short_release_tag},${dd_full_release_tag},${dd_default_branch_tag},${dd_release_or_dev_tag}" \ + -F "test_title=[${dd_module_name}]: ${dd_image_name}:${dd_image_version}" \ -F "version=${dd_image_version}" \ -F "build_id=${IMAGE_HASH}" \ -F "commit_hash=${CI_COMMIT_SHA}" \ -F "branch_tag=${module_tag}" \ - -F "apply_tags_to_findings=true" + -F "apply_tags_to_findings=true") + + dd_return_code="${dd_upload_response: -3}" + dd_return_body="${dd_upload_response:0: -3}" + if [ ${dd_return_code} -eq 201 ]; then + dd_engagement_id=$(echo ${dd_return_body} | jq ".engagement_id" ) + echo "dd_engagement_id: ${dd_engagement_id}" + echo "Update with tags: ${tags_string}" + # Updating engagement + dd_eng_patch_response=$(curl -sw "%{http_code}" -X "PATCH" \ + --retry 10 \ + --retry-delay 20 \ + --retry-all-errors \ + "${DD_URL}/api/v2/engagements/${dd_engagement_id}/" \ + -H "accept: application/json" \ + -H "Authorization: Token ${DD_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{ + \"tags\": ["${tags_string}"], + \"version\": \"${dd_image_version}\", + \"branch_tag\": \"${dd_branch}\" + }") + if [ ${dd_eng_patch_response: -3} -eq 200 ]; then + echo "Engagemet \"${dd_engagement_name}\" updated successfully" + else + echo "!!!WARNING!!!" + echo "Engagemet \"${dd_engagement_name}\" WAS NOT UPDATED" + echo "HTTP_CODE: ${dd_eng_patch_response: -3}" + echo "DD_RESPONSE: ${dd_eng_patch_response:0: -3}" + fi + else + echo "!!!WARNING!!!" + echo "Report for image \"${dd_image_name}\" of \"${dd_module_name}\" module WAS NOT UPLOADED" + echo "HTTP_CODE: ${dd_return_code}" + echo "DD_RESPONSE: ${dd_return_body}" + fi + } # Scan in loop for provided list of tags @@ -145,7 +205,10 @@ dd_default_branch_tag="" dd_short_release_tag="" dd_full_release_tag="" + dd_release_or_dev_tag="dev" dd_image_version="${module_tag}" + dd_branch="${module_tag}" + date_iso=$(date -I) module_image="${DEV_REGISTRY_MODULE_BASEDIR}/${MODULE_NAME}" trivy_registry_user="${DEV_REGISTRY_USER}" trivy_registry_pass="${DEV_REGISTRY_PASSWORD}" @@ -159,6 +222,7 @@ trivy_registry_pass="${PROD_REGISTRY_PASSWORD}" dd_short_release_tag="release:$(echo ${module_tag} | cut -d '.' -f -2 | sed 's/^v//')" dd_full_release_tag="image_release_tag:${module_tag}" + dd_release_or_dev_tag="release" dd_image_version="$(echo ${dd_short_release_tag} | sed 's/^release\://')" fi module_workdir="${workdir}/${MODULE_NAME}_${module_tag}" @@ -178,7 +242,7 @@ digests=$(echo "${digests}"|jq --arg i "${MODULE_NAME}" --arg s "${module_tag}" '. += { ($i): ($s) }') echo "Images to scan:" echo "${digests}" - date_iso=$(date -I) + while read -r line; do IMAGE_NAME=$(jq -rc '.key' <<< "${line}") if [[ "${IMAGE_NAME}" == "trivy" ]]; then @@ -220,8 +284,8 @@ fi echo " Done" - send_report "CVE" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" - send_report "License" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" + send_report "CVE" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${MODULE_NAME}" "${IMAGE_NAME}" + send_report "License" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${MODULE_NAME}" "${IMAGE_NAME}" done < <(jq -rc 'to_entries[]' <<< "${digests}") done rm -rf ${workdir} diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index 5eedee1..8aadfaf 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -110,4 +110,4 @@ before_script: stages: - build - - deploy + - deploy \ No newline at end of file diff --git a/templates/multi-repo/Build.gitlab-ci.yml b/templates/multi-repo/Build.gitlab-ci.yml new file mode 100644 index 0000000..c34de2b --- /dev/null +++ b/templates/multi-repo/Build.gitlab-ci.yml @@ -0,0 +1,33 @@ +.build: + stage: build + rules: + - !reference [.default_rules, rules] + before_script: + - !reference [.setup, before_script] + script: + # Build images + - | + werf build \ + --save-build-report --build-report-path images_tags_werf.json + artifacts: + paths: + - images_tags_werf.json + expire_in: "30 days" + +.svace_rules_mr: + rules: + - if: '$CI_MERGE_REQUEST_LABELS =~ /(^|,)analyze\/svace(,|$)/' + variables: + SVACE_ENABLED: "true" + +.svace_rules_manual: + rules: + - if: $CI_PIPELINE_SOURCE == "web" && $SVACE_ENABLED == "true" && $CI_COMMIT_BRANCH + variables: + SVACE_ENABLED: "true" + +.svace_rules_schedule: + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $SVACE_ENABLED == "true" && $CI_COMMIT_BRANCH + variables: + SVACE_ENABLED: "true" \ No newline at end of file diff --git a/templates/multi-repo/Cleanup.gitlab-ci.yml b/templates/multi-repo/Cleanup.gitlab-ci.yml new file mode 100644 index 0000000..db7dcbd --- /dev/null +++ b/templates/multi-repo/Cleanup.gitlab-ci.yml @@ -0,0 +1,52 @@ +Scheduled cleanup: + stage: cleanup + timeout: 3h + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + when: on_success + - when: never + before_script: + - !reference [.setup, before_script] + script: + - | + if [[ -z "${NO_PRIVATE_REPO_PATCH}" ]]; then + echo "Apply git private repo patch... Set NO_PRIVATE_REPO_PATCH=1 to disable it" + export GOPRIVATE=${CI_SERVER_HOST} + git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/".insteadOf "git@${CI_SERVER_HOST}:" + fi + + echo "Managed images which will be preserved during cleanup procedure:" + werf managed-images ls + echo "Starting cleanup..." + werf cleanup + +Auto cleanup: + stage: cleanup + allow_failure: true + timeout: 10 minutes + rules: + # do not run if this job is explicitly disabled by user + - if: $AUTO_CLEANUP == "false" || $AUTO_CLEANUP == "0" || $AUTO_CLEANUP == "" + when: never + # do not run if there is a tag (release workflow) + - if: $CI_COMMIT_TAG + when: never + - !reference [.default_rules, rules] + before_script: + - !reference [.setup, before_script] + script: + - | + if (( $(date +%s) % 10 == 0 )); then + echo "✨ Run auto cleanup" + + if [[ -z "${NO_PRIVATE_REPO_PATCH}" ]]; then + echo "Apply git private repo patch... Set NO_PRIVATE_REPO_PATCH=1 to disable it" + export GOPRIVATE=${CI_SERVER_HOST} + git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/".insteadOf "git@${CI_SERVER_HOST}:" + fi + + echo "Managed images which will be preserved during cleanup procedure:" + werf managed-images ls + echo "Starting cleanup..." + werf cleanup + fi diff --git a/templates/multi-repo/Deploy.gitlab-ci.yml b/templates/multi-repo/Deploy.gitlab-ci.yml new file mode 100644 index 0000000..184e1ed --- /dev/null +++ b/templates/multi-repo/Deploy.gitlab-ci.yml @@ -0,0 +1,144 @@ +.publish: + stage: deploy + script: + - | + if [[ "x${MODULES_REGISTRY}" == "x" || "x${MODULES_REGISTRY_LOGIN}" == "x" || "x${MODULES_REGISTRY_PASSWORD}" == "x" || "x${MODULES_REGISTRY_PATH}" == "x" ]]; then + echo "One or more required variables of source registry is empty: MODULES_REGISTRY, MODULES_REGISTRY_LOGIN, MODULES_REGISTRY_PASSWORD, MODULES_REGISTRY_PATH" + exit 5 + fi + + # set defaults for target registry variables from source registry variables + export MODULES_TARGET_REGISTRY="${MODULES_TARGET_REGISTRY:-$MODULES_REGISTRY}" + export MODULES_TARGET_REGISTRY_LOGIN="${MODULES_TARGET_REGISTRY_LOGIN:-$MODULES_REGISTRY_LOGIN}" + export MODULES_TARGET_REGISTRY_PASSWORD="${MODULES_TARGET_REGISTRY_PASSWORD:-$MODULES_REGISTRY_PASSWORD}" + export MODULES_TARGET_REGISTRY_PATH="${MODULES_TARGET_REGISTRY_PATH:-$MODULES_REGISTRY_PATH}" + + - | + # Login to Gitlab registry just in case + if [[ "x${MODULES_REGISTRY}" != "x${CI_REGISTRY}" && "x${CI_REGISTRY}" != "x" && "x${CI_REGISTRY_USER}" != "x" && "x${CI_REGISTRY_PASSWORD}" != "x" ]]; then + echo "Login to Gitlab ${CI_REGISTRY} just in case..." + werf cr login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} + fi + + # Login to source registry + echo "Login to source registry ${MODULES_REGISTRY}..." + werf cr login -u ${MODULES_REGISTRY_LOGIN} -p ${MODULES_REGISTRY_PASSWORD} ${MODULES_REGISTRY} + + # Login to target registry + echo "Login to target registry ${MODULES_TARGET_REGISTRY}..." + werf cr login -u ${MODULES_TARGET_REGISTRY_LOGIN} -p ${MODULES_TARGET_REGISTRY_PASSWORD} ${MODULES_TARGET_REGISTRY} + # generate MODULES_MODULE_TARGET + - | + export MODULES_MODULE_TARGET="${MODULES_TARGET_REGISTRY}/${MODULES_TARGET_REGISTRY_PATH}" + + # Module images + - | + jq -r '.Images | values[] | select (.Final==true) | .WerfImageName + " " + .DockerImageName + " " + .DockerTag + " " + .DockerImageDigest' < images_tags_werf.json | while read line; do + image="$(cut -d " " -f 1 <<< ${line})" + docker_image="$(cut -d " " -f 2 <<< ${line})" + docker_tag="$(cut -d " " -f 3 <<< ${line})" + shasum="$(cut -d " " -f 4 <<< ${line})" + name="$(tr '[:lower:]' '[:upper:]' <<< ${image} | sed 's|[-/]|_|g')" + + # image: pg-images-17.3-bookworm-standard; + # name: PG_IMAGES_17.3_BOOKWORM_STANDARD; + # shasum: sha256:cd6daa30c94ec77b352156cbf1c05c8e98b44d1d3a47fa2b0a1083587247f730; + # docker_image: registry.flant.com/team/managed-services/managed-psql/managed-psql-d8/managed-postgres:ed0388a743d61926309d1023e02c639c1006f7b7b56d78161f32b0e0-1744971955188; + # docker_tag: ed0388a743d61926309d1023e02c639c1006f7b7b56d78161f32b0e0-1744971955188 + + IMAGE_SRC="${docker_image}" + IMAGE_DST="${MODULES_MODULE_TARGET}/${MODULES_MODULE_NAME}:${docker_tag}" + + if [[ -z "${DEBUG_CI_DRY_RUN}" ]]; then + echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" + crane copy ${IMAGE_SRC} ${IMAGE_DST} + else + echo "[DEBUG_CI_DRY_RUN] ✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" + fi + done + + # Bundle image + - | + IMAGE_SRC="$(jq -r '.Images."bundle".DockerImageName' images_tags_werf.json)" + IMAGE_DST="${MODULES_MODULE_TARGET}/${MODULES_MODULE_NAME}:${MODULES_MODULE_TAG}" + + if [[ -z "${DEBUG_CI_DRY_RUN}" ]]; then + echo "✨ Pushing BUNDLE ${IMAGE_SRC} to ${IMAGE_DST}" + crane copy ${IMAGE_SRC} ${IMAGE_DST} + else + echo "[DEBUG_CI_DRY_RUN] ✨ Pushing BUNDLE ${IMAGE_SRC} to ${IMAGE_DST}" + fi + # Release-channel image + - | + IMAGE_SRC="$(jq -r '.Images."release-channel-version".DockerImageName' images_tags_werf.json)" + IMAGE_DST="${MODULES_MODULE_TARGET}/${MODULES_MODULE_NAME}/release:${MODULES_MODULE_TAG}" + + if [[ -z "${DEBUG_CI_DRY_RUN}" ]]; then + echo "✨ Pushing RELEASE ${IMAGE_SRC} to ${IMAGE_DST}" + crane copy ${IMAGE_SRC} ${IMAGE_DST} + else + echo "[DEBUG_CI_DRY_RUN] ✨ Pushing RELEASE ${IMAGE_SRC} to ${IMAGE_DST}" + fi + # Register module + - | + if [[ -z "${DEBUG_CI_DRY_RUN}" ]]; then + echo "✨ Register the module ${MODULES_MODULE_NAME} in ${MODULES_MODULE_TARGET} registry" + crane append \ + --oci-empty-base \ + --new_layer "" \ + --new_tag "${MODULES_MODULE_TARGET}:${MODULES_MODULE_NAME}" + else + echo "[DEBUG_CI_DRY_RUN] ✨ Register the module ${MODULES_MODULE_NAME} in ${MODULES_MODULE_TARGET} registry" + fi + +.deploy: + stage: deploy + script: + - | + # set defaults for target registry variables from source registry variables + export MODULES_TARGET_REGISTRY="${MODULES_TARGET_REGISTRY:-$MODULES_REGISTRY}" + export MODULES_TARGET_REGISTRY_PATH="${MODULES_TARGET_REGISTRY_PATH:-$MODULES_REGISTRY_PATH}" + + REPO="${MODULES_TARGET_REGISTRY}/${MODULES_TARGET_REGISTRY_PATH}/${MODULES_MODULE_NAME}/release" + + IMAGE_SRC="${REPO}:${MODULES_MODULE_TAG}" + IMAGE_DST="${REPO}:${RELEASE_CHANNEL}" + + if [[ -z "${DEBUG_CI_DRY_RUN}" ]]; then + echo "✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" + crane copy "${IMAGE_SRC}" "${IMAGE_DST}" + else + echo "[DEBUG_CI_DRY_RUN] ✨ Pushing ${IMAGE_SRC} to ${IMAGE_DST}" + fi + +.deploy_prod_rules: + rules: + # add MANUAL deploy job if $FORCE_CI variable is set + - if: $FORCE_CI == "true" || $FORCE_CI == "1" + when: manual + # do not run if some required variables is empty + - if: '$PROD_MODULES_REGISTRY == null || $PROD_MODULES_REGISTRY == "" || $EDITION == null || $EDITION == "" || $RELEASE_CHANNEL == null || $RELEASE_CHANNEL == ""' + when: never + # add MANUAL deploy job only if it is a tag defined (release workflow) + - if: '$CI_COMMIT_TAG' + when: manual + +.deploy_prod: + stage: deploy + rules: + - !reference [.deploy_prod_rules, rules] + variables: + MODULES_TARGET_REGISTRY: $PROD_MODULES_REGISTRY + MODULES_TARGET_REGISTRY_LOGIN: $PROD_MODULES_REGISTRY_LOGIN + MODULES_TARGET_REGISTRY_PASSWORD: $PROD_MODULES_REGISTRY_PASSWORD + # path in PROD registry must be hardcoded + MODULES_TARGET_REGISTRY_PATH: deckhouse/${EDITION}/modules + script: + - | + if [ "$DEBUG_CI" = "true" -o "$DEBUG_CI" = "1" ]; then + printenv | sort + fi + # publish final images to prod registry and register module with $MODULES_MODULE_TAG + - !reference [.publish, script] + # make 'symlink' to published module tag for specified release-channel + - !reference [.deploy, script] diff --git a/templates/multi-repo/Lint.gitlab-ci.yml b/templates/multi-repo/Lint.gitlab-ci.yml new file mode 100644 index 0000000..5ac4269 --- /dev/null +++ b/templates/multi-repo/Lint.gitlab-ci.yml @@ -0,0 +1,10 @@ +.lint: + stage: lint + rules: + - !reference [.default_rules, rules] + allow_failure: true + before_script: + - !reference [.setup, before_script] + script: + - | + dmt lint ./ diff --git a/templates/multi-repo/Setup.gitlab-ci.yml b/templates/multi-repo/Setup.gitlab-ci.yml new file mode 100644 index 0000000..43b8755 --- /dev/null +++ b/templates/multi-repo/Setup.gitlab-ci.yml @@ -0,0 +1,173 @@ +# Required CI/CD variables (must be defined in Gitlab's project/group settings): +# $DEV_MODULES_REGISTRY - dev registry URL (WITHOUT http/https prefix and path part) (like: registry.example.com) +# $DEV_MODULES_REGISTRY_PATH - dev registry path (to modules folder) part (like: sys/deckhouse-oss/modules) +# $DEV_MODULES_REGISTRY_LOGIN - login for dev registry (same as module name for most cases) +# $DEV_MODULES_REGISTRY_PASSWORD - password for dev registry + +# $PROD_MODULES_REGISTRY - production registry URL (WITHOUT http/https prefix and path part) +# $PROD_MODULES_REGISTRY_PATH - prod registry path (to modules folder) part +# $PROD_MODULES_REGISTRY_LOGIN - login for prod registry +# $PROD_MODULES_REGISTRY_PASSWORD - password for prod registry + + +# SVACE_ANALYZE_HOST - hostname of the svace analyze vm +# SVACE_ANALYZE_SSH_USER: - ssh user to connect with to svace analyze vm +# SVACE_ANALYZE_SSH_PRIVATE_KEY_B64 - svace analyze server ssh private key + + +variables: + ############################## + # User default settings + ############################## + + MODULES_MODULE_NAME: "${CI_PROJECT_NAME}" + MODULES_MODULE_TAG: ${CI_COMMIT_REF_NAME} + # Enable auto-cleanup job by default. + # `Auto cleanup` job is run randomly, only if current second is divisible by 10 + AUTO_CLEANUP: + value: "true" + options: + - "true" + - "false" + description: "`Auto cleanup` job is run randomly (if enabled), only if current second is divisible by 10" + + SVACE_ENABLED: + value: "false" + description: "Enable Svace static analysis" + options: + - "true" + - "false" + + ############################## + # Internal default settings + ############################## + BASE_IMAGES_VERSION: v0.5.9 + FORCE_CI: + value: "false" + options: + - "true" + - "false" + description: "Set to true if need force run workflow" + + # use module's container registry (on Gitlab) as werf's intermediate/cache images registry (repo with all build-time artifacts (garbage)) + WERF_REPO: + description: "Container registry storage address" + value: ${CI_REGISTRY_IMAGE}/${MODULES_MODULE_NAME} + WERF_VERSION: "2 stable" + +stages: + - lint + - cleanup + - build + - deploy + +.default_rules: + rules: + # run if $FORCE_CI variable is set to true + - if: $FORCE_CI == "true" || $FORCE_CI == "1" + # run if there is a tag defined (module release workflow) + - if: $CI_COMMIT_TAG + # run if this is a merge request event + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + # run when push to default (main/master) branch (for example, when merge request is merged to `master` branch) + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + # run when new branch is created and there are no opened merge requests for this branch and no commits to this branch yet (completely new branch from master/main) + # this rule is required for job `Publish merge request to DEV` to work properly + # - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && ($CI_MERGE_REQUEST_IID == null || $CI_MERGE_REQUEST_IID == "") && $CI_COMMIT_BEFORE_SHA == "0000000000000000000000000000000000000000" + # DO NOT run if this is a push to a branch and there are open merge requests (remove duplicated `branch` pipeline) + # - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS' + # when: never + # DO NOT run if this is a push (commits) to a branch (regardless of opened merge requests) + - if: $CI_COMMIT_BRANCH + when: never + +.setup: + before_script: + # Setup trdl + - | + trdl_version=$(curl -s https://tuf.trdl.dev/targets/channels/0/stable) + curl -sSLO "https://tuf.trdl.dev/targets/releases/$trdl_version/linux-amd64/bin/trdl" + install -D trdl ~/bin/trdl + rm trdl + export PATH=$PATH:~/bin + + # Setup werf + - | + trdl add werf https://tuf.werf.io 1 b7ff6bcbe598e072a86d595a3621924c8612c7e6dc6a82e919abe89707d7e3f468e616b5635630680dd1e98fc362ae5051728406700e6274c5ed1ad92bea52a2 + source $(trdl use werf ${WERF_VERSION:-1.2 stable}) + source $(werf ci-env gitlab --as-file) + + # Login to gitlab registry by default + if [[ "x${MODULES_REGISTRY}" == "x" ]]; then + MODULES_REGISTRY="${CI_REGISTRY}" + fi + if [[ "x${MODULES_REGISTRY_LOGIN}" == "x" ]]; then + MODULES_REGISTRY_LOGIN="${CI_REGISTRY_USER}" + fi + if [[ "x${MODULES_REGISTRY_PASSWORD}" == "x" ]]; then + MODULES_REGISTRY_PASSWORD="${CI_REGISTRY_PASSWORD}" + fi + echo "Logging in to registry ${MODULES_REGISTRY}..." + werf cr login -u ${MODULES_REGISTRY_LOGIN} -p ${MODULES_REGISTRY_PASSWORD} ${MODULES_REGISTRY} + + # Setup dmt + - | + trdl add dmt https://trrr.flant.dev/trdl-dmt/ 3 e77d785600a8c8612b84b93a5a2e4c48188d68f7478356d0708213e928bf67b024ed412e702dc32930da5c5bfc9b1c44be3ee7a292f923327815c91c6c3c3833 + source $(trdl use dmt 0 stable) + + - | + if [[ ! -z "${BASE_IMAGES_VERSION}" ]]; then + echo "Downloading base_images.yml ${BASE_IMAGES_VERSION}" + curl --fail -sSLO https://fox.flant.com/api/v4/projects/deckhouse%2Fbase-images/packages/generic/base_images/${BASE_IMAGES_VERSION}/base_images.yml + else + echo "DO NOT download base_images.yml because BASE_IMAGES_VERSION is empty" + fi + + # Download deckhouse lib-helm archive + - | + if [[ -n "${DECKHOUSE_LIB_HELM_VERSION}" ]]; then + mkdir charts + curl --fail -sSLO https://github.com/deckhouse/lib-helm/releases/download/deckhouse_lib_helm-${DECKHOUSE_LIB_HELM_VERSION}/deckhouse_lib_helm-${DECKHOUSE_LIB_HELM_VERSION}.tgz --output-dir ./charts + fi + + # Add ssh keys + - | + if [[ -n "${SOURCE_REPO_SSH_KEY}" || -n "${SVACE_ANALYZE_SSH_PRIVATE_KEY_B64}" ]]; then + + eval $(ssh-agent) + trap "kill -3 ${SSH_AGENT_PID}" ERR EXIT HUP INT QUIT TERM + export SSH_KNOWN_HOSTS=~/.ssh/known_hosts + mkdir -p ~/.ssh + touch ~/.ssh/known_hosts + + if [[ -n "${SOURCE_REPO_SSH_KEY}" ]]; then + ssh-add - <<< "${SOURCE_REPO_SSH_KEY}" + if [[ -n "${SOURCE_REPO}" ]]; then + HOST=$(grep -oP '(?<=@)[^/:]+' <<< ${SOURCE_REPO}) + HOST_KEYS=$(ssh-keyscan -H "$HOST" 2>/dev/null) + while IFS= read -r KEY_LINE; do + CONSTANT_PART=$(awk '{print $2, $3}' <<< "$KEY_LINE") + if ! grep -q "$CONSTANT_PART" ~/.ssh/known_hosts; then + echo "$KEY_LINE" >> ~/.ssh/known_hosts + fi + done <<< "$HOST_KEYS" + fi + fi + + if [[ -n "${SVACE_ANALYZE_SSH_PRIVATE_KEY_B64}" ]]; then + echo "${SVACE_ANALYZE_SSH_PRIVATE_KEY_B64}" | base64 -d | ssh-add - + if [[ -n "${SVACE_ANALYZE_HOST}" ]]; then + echo "Adding svace ssh key (ignoring errors)." + set +e + HOST=${SVACE_ANALYZE_HOST} + HOST_KEYS=$(ssh-keyscan -H "$HOST" 2>/dev/null) + while IFS= read -r KEY_LINE; do + CONSTANT_PART=$(awk '{print $2, $3}' <<< "$KEY_LINE") + if ! grep -q "$CONSTANT_PART" ~/.ssh/known_hosts; then + echo "$KEY_LINE" >> ~/.ssh/known_hosts + fi + done <<< "$HOST_KEYS" + set -e + fi + fi + fi