From e4e1f92e873e9180d80e2b798bb6d435ff31efd0 Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Thu, 16 Sep 2021 10:20:57 +0100 Subject: [PATCH 01/21] [ci] release automatically every 3 weeks [finishes #179484418] --- ci/pipelines/bbr/pipeline.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index 83be6d54c..fec48a865 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -94,6 +94,11 @@ resource_types: repository: teliaoss/github-pr-resource resources: +- name: every-3-weeks + type: time + source: + interval: 504h # 24*7*3 + - name: bbr-director-test-releases type: git source: @@ -562,6 +567,8 @@ jobs: serial_groups: [version] plan: - in_parallel: + - get: every-3-weeks + trigger: true - get: docs-bbr-main - get: cryogenics-ci - get: bosh-backup-and-restore-master From 7475f04c16c840fe6ac003f28e5a3fae58d82f8a Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Thu, 16 Sep 2021 10:38:49 +0100 Subject: [PATCH 02/21] [ci] Fix missing task input [#179484418] --- ci/pipelines/bbr/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index fec48a865..fde116728 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -605,6 +605,7 @@ jobs: file: cryogenics-ci/tasks/release-automation/release-notes/task.yml input_mapping: git-repo: bosh-backup-and-restore-master + cryogenics-concourse-tasks: cryogenics-ci params: USE_LATEST_PUBLISHED_TAG: true - task: format-release-notes From 56be754ac689c567b9357f24597beb776a25b0f5 Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Thu, 16 Sep 2021 12:35:32 +0100 Subject: [PATCH 03/21] [ci] fix task inputs [#179484418] --- ci/pipelines/bbr/pipeline.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index fde116728..52282fa8b 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -612,9 +612,10 @@ jobs: file: cryogenics-ci/tasks/release-automation/format-release-notes/task.yml input_mapping: template-folder: cryogenics-ci + cryogenics-concourse-tasks: cryogenics-ci params: VERSION: ((.:version-number)) - TEMPLATE_PATH: ci/release-templates/docs-release-notes-auto.md.erb + TEMPLATE_PATH: tasks/release-automation/release-notes-templates/docs-release-notes-auto.md.erb - task: commit-release-notes file: docs-bbr-main/ci/tasks/commit-new-release-notes.yml input_mapping: @@ -641,6 +642,7 @@ jobs: TITLE: ((.:version-number)) release notes input_mapping: repo: docs-bbr-main + cryogenics-concourse-tasks: cryogenics-ci - in_parallel: - put: bbr-pivnet params: @@ -669,6 +671,8 @@ jobs: View the pipeline <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|here> - task: create-story file: cryogenics-ci/tasks/tracker-automation/create-story/task.yml + input_mapping: + cryogenics-concourse-tasks: cryogenics-ci params: TRACKER_API_TOKEN: ((tracker.api_token)) TRACKER_PROJECT_ID: ((tracker.project_id)) From b5d35486298f65f82581ec37944f09b2a150c0a1 Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Thu, 16 Sep 2021 12:54:59 +0100 Subject: [PATCH 04/21] [ci] Replace concourse-time by gitgat-resource [#179484418] --- ci/pipelines/bbr/pipeline.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index 52282fa8b..70baa74b5 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -61,6 +61,12 @@ groups: - update-homebrew-formula resource_types: +- name: gitgat + type: docker-image + source: + repository: cryogenics/concourse-gitgat-resource + tag: latest + - name: pivnet type: docker-image source: @@ -95,9 +101,11 @@ resource_types: resources: - name: every-3-weeks - type: time - source: - interval: 504h # 24*7*3 + type: gitgat + source: + uri: https://github.com/cloudfoundry-incubator/bosh-backup-and-restore + branch: master + since: 3 weeks - name: bbr-director-test-releases type: git From c56cb91452aabbdd77928cf7494a9297fda00446 Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Thu, 16 Sep 2021 13:05:19 +0100 Subject: [PATCH 05/21] [ci] Update Homebrew formula after publishing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In practice this does not change anything. But visually, I have had a hard time reading the pipeline with publish jobs at the beginning... Hopefully this change will make pipeline flow more intuitive 🤞 [#179484418] --- ci/pipelines/bbr/pipeline.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index 70baa74b5..54faa8cd1 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -188,14 +188,6 @@ resources: repository: bosh-backup-and-restore access_token: ((github.access_token)) -- name: bbr-final-release - type: github-release - icon: github - source: - user: cloudfoundry-incubator - repository: bosh-backup-and-restore - access_token: ((github.access_token)) - - name: bbr-final-release-version type: semver icon: google @@ -745,19 +737,20 @@ jobs: AWS_SECRET_ACCESS_KEY: ((pivnet_bucket.secret_access_key)) AWS_PRODUCT_NAME: ((pivnet_bucket.user)) - - name: update-homebrew-formula serial: true plan: - in_parallel: - get: homebrew-tap - - get: bbr-final-release + - get: bbr-release + passed: + - build-and-publish-final trigger: true - get: bosh-backup-and-restore - task: update-homebrew-formula file: bosh-backup-and-restore/ci/tasks/bbr-update-homebrew-formula/task.yml input_mapping: - bbr-release: bbr-final-release + bbr-release: bbr-release - put: homebrew-tap params: repository: updated-homebrew-tap From 85f07a5a86800caee0f2e1f660dfb3aed59b7433 Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 10:12:13 +0100 Subject: [PATCH 06/21] create commit to test ci --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ae72d92aa..bfbe31fa1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +(test) # BOSH Backup and Restore BOSH Backup and Restore is a CLI utility for orchestrating the backup and restore of [BOSH](https://bosh.io/) deployments and BOSH directors. It orchestrates triggering the backup or restore process on the deployment or director, and transfers the backup artifact to and from the deployment or director. From 4e8d42cfb141da7dbef8d2ebdcb4809a1efaff43 Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 10:42:55 +0100 Subject: [PATCH 07/21] [ci] Split Github release bits onto its own job [#179484418 ] --- ci/pipelines/bbr/pipeline.yml | 133 +++++++++++++++--- ci/tasks/prepare-artefacts-for-github/task.sh | 34 +++++ .../prepare-artefacts-for-github/task.yml | 21 +++ ci/tasks/promote-release-candidate/task.sh | 58 ++++++++ ci/tasks/promote-release-candidate/task.yml | 18 +++ 5 files changed, 245 insertions(+), 19 deletions(-) create mode 100755 ci/tasks/prepare-artefacts-for-github/task.sh create mode 100644 ci/tasks/prepare-artefacts-for-github/task.yml create mode 100755 ci/tasks/promote-release-candidate/task.sh create mode 100644 ci/tasks/promote-release-candidate/task.yml diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index 54faa8cd1..64e5d2643 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -59,6 +59,7 @@ groups: - request-osl - publish-ga - update-homebrew-formula + - publish-to-github resource_types: - name: gitgat @@ -429,25 +430,6 @@ jobs: GITHUB_SDK_PRIVATE_KEY: ((github-sdk.key)) <<: *maru-lite-bosh-env -- name: build-rc - serial_groups: [version] - plan: - - in_parallel: - - get: bosh-backup-and-restore - resource: bosh-backup-and-restore-master - trigger: true - params: - submodules: none - disable_git_lfs: true - - get: bbr-final-release-version - params: {pre: rc} - - task: build - file: bosh-backup-and-restore/ci/tasks/bbr-build/task.yml - params: - GITHUB_SDK_PRIVATE_KEY: ((github-sdk.key)) - - put: rc - params: {file: bbr-build/bbr-*.tar} - - name: test-build-s3-config-validator plan: - in_parallel: @@ -563,6 +545,119 @@ jobs: TRACKER_PROJECT_ID: ((tracker.project_id)) GIT_REPOSITORY: cloudfoundry-incubator/bosh-backup-and-restore +- name: build-rc + serial_groups: [version] + plan: + - in_parallel: + - get: bosh-backup-and-restore + resource: bosh-backup-and-restore-master + trigger: true + params: + submodules: none + disable_git_lfs: true + - get: bbr-final-release-version + params: + pre: rc + - task: build-bbr + file: bosh-backup-and-restore/ci/tasks/bbr-build/task.yml + params: + GITHUB_SDK_PRIVATE_KEY: ((github-sdk.key)) + - task: build-s3-config-validator + file: bosh-backup-and-restore/ci/tasks/bbr-s3-config-validator-build/task.yml + input_mapping: + s3-config-validator-version: bbr-final-release-version + - put: bbr-final-release-version + params: { file: bbr-final-release-version/number } + - put: rc + params: {file: bbr-build/bbr-*.tar} + - put: bbr-s3-config-validator-test-artifacts + params: + file: bbr-s3-config-validator-test-artifacts/* + +- name: publish-to-github + serial: true + plan: + - in_parallel: + - get: bbr-final-release-version + params: + bump: final + passed: + - build-rc + - get: cryogenics-ci + - get: bosh-backup-and-restore-master + passed: [build-rc] + params: + submodules: none + disable_git_lfs: true + - get: docs-bbr-main + - get: ci + - get: rc + params: + unpack: true + passed: + - build-rc + - get: bbr-s3-config-validator-artifact + params: + unpack: true + - load_var: version-number + file: bbr-final-release-version/number + - task: promote-rc-binaries + file: ci/ci/tasks/promote-release-candidate/task.yml + input_mapping: + bbr-rc-artefacts: rc + s3-config-validator-rc-artefacts: bbr-s3-config-validator-artifact + - task: create-release-notes + file: cryogenics-ci/tasks/release-automation/release-notes/task.yml + input_mapping: + git-repo: bosh-backup-and-restore-master + cryogenics-concourse-tasks: cryogenics-ci + params: + USE_LATEST_PUBLISHED_TAG: true + - task: format-release-notes + file: cryogenics-ci/tasks/release-automation/format-release-notes/task.yml + input_mapping: + template-folder: cryogenics-ci + cryogenics-concourse-tasks: cryogenics-ci + params: + VERSION: ((.:version-number)) + TEMPLATE_PATH: tasks/release-automation/release-notes-templates/docs-release-notes-auto.md.erb + - task: commit-release-notes + file: docs-bbr-main/ci/tasks/commit-new-release-notes.yml + input_mapping: + docs-repo: docs-bbr-main + params: + VERSION: ((.:version-number)) + GIT_AUTHOR_NAME: "Cryogenics CI" + GIT_AUTHOR_EMAIL: "mapbu-cryogenics@groups.vmware.com" + - put: docs-bbr-main + params: + repository: docs-repo + rebase: false + merge: false + force: true + branch: ((.:version-number))-release-notes + on_success: + try: + task: create-docs-pull-request + file: cryogenics-ci/tasks/create-pr/task.yml + params: + GITHUB_TOKEN: ((github.access_token)) + BASE: master + BRANCH: ((.:version-number))-release-notes + TITLE: ((.:version-number)) release notes + input_mapping: + repo: docs-bbr-main + cryogenics-concourse-tasks: cryogenics-ci + - put: bbr-release + params: + name: bbr-final-release-version/number + tag: bbr-final-release-version/number + tag_prefix: v + body: release-notes/release-notes.md + commitish: bosh-backup-and-restore-master/.git/refs/heads/master + globs: + - bbr-final-release-artefacts/bbr* + - name: build-and-publish-final serial_groups: [version] plan: diff --git a/ci/tasks/prepare-artefacts-for-github/task.sh b/ci/tasks/prepare-artefacts-for-github/task.sh new file mode 100755 index 000000000..44cd6e0c3 --- /dev/null +++ b/ci/tasks/prepare-artefacts-for-github/task.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$PWD" +GITHUB_FOLDER="${ROOT_DIR}/github-release-with-metadata" +RELEASE_FOLDER="${ROOT_DIR}/release" +RELEASE_TAR_FOLDER="${ROOT_DIR}/release-tar" +VERSION=$(cat "version-folder/${VERSION_PATH}") + +function main { + create_tarball + copy_tarball_to_folder "${GITHUB_FOLDER}" + copy_release_files_to_folder "${GITHUB_FOLDER}" + delete_sha256_files + export_release_metadata_variables +} + +function create_tarball { + echo "Creating release tarball..." + export TAR_NAME="bbr-${VERSION}.tar" + tar -cf "${TAR_NAME}" -C "${RELEASE_TAR_FOLDER}" . +} + +function copy_tarball_to_folder { + echo "Adding tarball to: $1" + cp "${TAR_NAME}" "$1" +} + +function copy_release_files_to_folder { + cp -r "${RELEASE_FOLDER}/." "$1" +} + +main diff --git a/ci/tasks/prepare-artefacts-for-github/task.yml b/ci/tasks/prepare-artefacts-for-github/task.yml new file mode 100644 index 000000000..3a02f684b --- /dev/null +++ b/ci/tasks/prepare-artefacts-for-github/task.yml @@ -0,0 +1,21 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: {repository: pcfplatformrecovery/backup-and-restore} + +inputs: +- name: release +- name: release-tar +- name: version-folder +- name: bosh-backup-and-restore-master + +params: + VERSION_PATH: + +outputs: +- name: github-release-with-metadata + +run: + path: bosh-backup-and-restore-master/ci/tasks/prepare-artefacts-for-github/task.sh diff --git a/ci/tasks/promote-release-candidate/task.sh b/ci/tasks/promote-release-candidate/task.sh new file mode 100755 index 000000000..5444d2c56 --- /dev/null +++ b/ci/tasks/promote-release-candidate/task.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +set -eu +set -x + +VERSION=$(cat bbr-final-release-version/bbr-final) + +function main { + local release_folder='bbr-final-release-artefacts' + + copy_bbr_binaries_to "$release_folder" + copy_s3_config_validator_artefacts_to "$release_folder" + create_bbr_tarball_in "$release_folder" +} + +function create_bbr_tarball_in() { + local release_folder="${1:?}" + + local bbr_tarball_folder + bbr_tarball_folder="$(mktemp -d)" + + ls bbr-rc-artefacts/releases + + cp 'bbr-rc-artefacts/releases/bbr' "$bbr_tarball_folder" + cp 'bbr-rc-artefacts/releases/bbr-mac' "$bbr_tarball_folder" + cp 'bbr-rc-artefacts/releases/checksum.sha256' "$bbr_tarball_folder" + + cp 's3-config-validator-rc-artefacts/bbr-s3-config-validator' "$bbr_tarball_folder" + cp 's3-config-validator-rc-artefacts/README.md' "$bbr_tarball_folder/bbr-s3-config-validator.README.md" + + echo "$(cat s3-config-validator-rc-artefacts/bbr-s3-config-validator.sha256) bbr-s3-config-validator" >> "$bbr_tarball_folder/checksum.sha256" + + tar -cf "$release_folder/bbr-${VERSION}.tar" -C "$bbr_tarball_folder" . +} + +function copy_bbr_binaries_to { + local release_folder="${1:?}" + local binary_name + + binary_name="bbr-${VERSION}-linux-amd64" + cp 'bbr-rc-artefacts/releases/bbr' "$release_folder/${binary_name}" + grep 'bbr$' 'bbr-rc-artefacts/releases/checksum.sha256' | cut -d' ' -f1 > "$release_folder/${binary_name}.sha256" + + binary_name="bbr-$VERSION-darwin-amd64" + cp 'bbr-rc-artefacts/releases/bbr-mac' "$release_folder/${binary_name}" + grep 'bbr-mac$' 'bbr-rc-artefacts/releases/checksum.sha256' | cut -d' ' -f1 > "$release_folder/${binary_name}.sha256" +} + +function copy_s3_config_validator_artefacts_to { + local release_folder="${1:?}" + + cp 's3-config-validator-rc-artefacts/README.md' "$release_folder/bbr-s3-config-validator-$VERSION.README.md" + cp 's3-config-validator-rc-artefacts/bbr-s3-config-validator' "$release_folder/bbr-s3-config-validator-$VERSION-linux-amd64" + cp 's3-config-validator-rc-artefacts/bbr-s3-config-validator.sha256' "$release_folder/bbr-s3-config-validator-$VERSION-linux-amd64.sha256" +} + +main + diff --git a/ci/tasks/promote-release-candidate/task.yml b/ci/tasks/promote-release-candidate/task.yml new file mode 100644 index 000000000..8f345bb44 --- /dev/null +++ b/ci/tasks/promote-release-candidate/task.yml @@ -0,0 +1,18 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: {repository: pcfplatformrecovery/backup-and-restore} + +inputs: +- name: bbr-final-release-version +- name: bbr-rc-artefacts +- name: s3-config-validator-rc-artefacts +- name: ci + +outputs: +- name: bbr-final-release-artefacts + +run: + path: ci/ci/tasks/promote-release-candidate/task.sh From 748799f04f48669c8b4b9a80d51b77a82da58648 Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 10:46:41 +0100 Subject: [PATCH 08/21] [ci] fix promote-release-candidate task [#179484418 ] --- ci/tasks/promote-release-candidate/task.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tasks/promote-release-candidate/task.sh b/ci/tasks/promote-release-candidate/task.sh index 5444d2c56..3ce091c57 100755 --- a/ci/tasks/promote-release-candidate/task.sh +++ b/ci/tasks/promote-release-candidate/task.sh @@ -3,7 +3,7 @@ set -eu set -x -VERSION=$(cat bbr-final-release-version/bbr-final) +VERSION=$(cat bbr-final-release-version/number) function main { local release_folder='bbr-final-release-artefacts' From 37823644568886c6e26e64961e0aac3a9a8cdaaf Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 10:52:19 +0100 Subject: [PATCH 09/21] [ci] improve promote-release-candidate task [#179484418 ] --- ci/tasks/promote-release-candidate/task.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/tasks/promote-release-candidate/task.sh b/ci/tasks/promote-release-candidate/task.sh index 3ce091c57..8bd84977f 100755 --- a/ci/tasks/promote-release-candidate/task.sh +++ b/ci/tasks/promote-release-candidate/task.sh @@ -1,6 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash -set -eu +set -euo pipefail set -x VERSION=$(cat bbr-final-release-version/number) @@ -19,8 +19,6 @@ function create_bbr_tarball_in() { local bbr_tarball_folder bbr_tarball_folder="$(mktemp -d)" - ls bbr-rc-artefacts/releases - cp 'bbr-rc-artefacts/releases/bbr' "$bbr_tarball_folder" cp 'bbr-rc-artefacts/releases/bbr-mac' "$bbr_tarball_folder" cp 'bbr-rc-artefacts/releases/checksum.sha256' "$bbr_tarball_folder" From ceca74eabcf4b6c004b85a0a28a5e020d001dc3d Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 11:35:59 +0100 Subject: [PATCH 10/21] [ci] bump version on release promotion [#179484418 ] --- ci/pipelines/bbr/pipeline.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index 64e5d2643..df3aaeffd 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -606,6 +606,10 @@ jobs: input_mapping: bbr-rc-artefacts: rc s3-config-validator-rc-artefacts: bbr-s3-config-validator-artifact + on_success: + put: bbr-final-release-version + params: + bump: patch - task: create-release-notes file: cryogenics-ci/tasks/release-automation/release-notes/task.yml input_mapping: From 073ae9dc7158bac46cf3e263a4f5d3b82c6d10ea Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 11:36:41 +0100 Subject: [PATCH 11/21] [ci] Update Homebrew after Github publishing [#179484418 ] --- ci/pipelines/bbr/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index df3aaeffd..088383358 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -843,7 +843,7 @@ jobs: - get: homebrew-tap - get: bbr-release passed: - - build-and-publish-final + - publish-to-github trigger: true - get: bosh-backup-and-restore - task: update-homebrew-formula From 8f4917c17d2e7f720da00a5c67ca35ea613cccee Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 11:38:03 +0100 Subject: [PATCH 12/21] [ci] trigger Github release every 3 weeks [#179484418 ] --- ci/pipelines/bbr/pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index 088383358..056255179 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -578,6 +578,8 @@ jobs: serial: true plan: - in_parallel: + - get: every-3-weeks + trigger: true - get: bbr-final-release-version params: bump: final From 527ece69715d59b9c78c561d4c27ab2a85ac1d9f Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 11:39:32 +0100 Subject: [PATCH 13/21] [ci] Send message on Slack on Github publishing [#179484418 ] --- ci/pipelines/bbr/pipeline.yml | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index 056255179..1c0136cb5 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -663,6 +663,16 @@ jobs: commitish: bosh-backup-and-restore-master/.git/refs/heads/master globs: - bbr-final-release-artefacts/bbr* + on_success: + do: + - load_var: github-release-url + file: bbr-release/url + - put: slack-cryo-notification + params: + text: | + A new release for $BUILD_PIPELINE_NAME has been published! + Release `((.:version-number))` is now available <((.:github-release-url))|here> + View the pipeline <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|here> - name: build-and-publish-final serial_groups: [version] @@ -751,25 +761,8 @@ jobs: s3_filepath_prefix: product-files/bosh-backup-restore file_glob: pivnet-release-with-metadata/bbr* override: true - - put: bbr-release - params: - name: bbr-final-release-version/number - tag: bbr-final-release-version/number - tag_prefix: v - body: release-notes/release-notes.md - commitish: bosh-backup-and-restore-master/.git/refs/heads/master - globs: - - github-release-with-metadata/bbr* - - put: bbr-final-release-version - params: { file: bbr-final-release-version/number } - load_var: github-release-url file: bbr-release/url - - put: slack-cryo-notification - params: - text: | - A new release for $BUILD_PIPELINE_NAME has been published! - Release `((.:version-number))` is now available <((.:github-release-url))|here> - View the pipeline <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|here> - task: create-story file: cryogenics-ci/tasks/tracker-automation/create-story/task.yml input_mapping: From f2fab8e68a8ae5a4dc729f0b3f6e829f10b0fd33 Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 12:52:22 +0100 Subject: [PATCH 14/21] [ci] separate jobs for github and pivnet release [#179484418 ] --- ci/pipelines/bbr/pipeline.yml | 119 +++++------------- .../generate-bbr-release-metadata/task.sh | 73 +++++------ .../generate-bbr-release-metadata/task.yml | 14 +-- 3 files changed, 64 insertions(+), 142 deletions(-) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index 1c0136cb5..d9d6b9a98 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -32,7 +32,8 @@ groups: - system-test-director - build-rc - update-tracker - - build-and-publish-final + - publish-to-github + - publish-to-pivnet - request-osl - publish-ga - update-homebrew-formula @@ -55,11 +56,11 @@ groups: - name: shipit jobs: - build-rc - - build-and-publish-final + - publish-to-github + - publish-to-pivnet - request-osl - publish-ga - update-homebrew-formula - - publish-to-github resource_types: - name: gitgat @@ -674,95 +675,35 @@ jobs: Release `((.:version-number))` is now available <((.:github-release-url))|here> View the pipeline <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|here> -- name: build-and-publish-final - serial_groups: [version] +- name: publish-to-pivnet + serial: true plan: - in_parallel: - - get: every-3-weeks - trigger: true - - get: docs-bbr-main - get: cryogenics-ci - - get: bosh-backup-and-restore-master - passed: [build-rc] - params: - submodules: none - disable_git_lfs: true - - get: homebrew-tap - - get: bbr-s3-config-validator-artifact - get: bbr-final-release-version - params: {bump: patch} + passed: + - publish-to-github - get: ci - - task: check-for-changes - file: bosh-backup-and-restore-master/ci/tasks/check-for-changes/task.yml - input_mapping: - repo: bosh-backup-and-restore-master - - task: build-binaries - file: ci/ci/tasks/build-binaries/task.yml + - get: bbr-release + passed: + - publish-to-github - load_var: version-number file: bbr-final-release-version/number - - in_parallel: - - task: generate-artifact-and-metadata - file: ci/ci/tasks/generate-bbr-release-metadata/task.yml - input_mapping: - version-folder: bbr-final-release-version - template-folder: bosh-backup-and-restore-master - release: bbr-build - release-tar: bbr-release - params: - TEMPLATE_PATH: ci/tasks/generate-bbr-release-metadata/release.yml.erb - VERSION_PATH: number - - do: - - task: create-release-notes - file: cryogenics-ci/tasks/release-automation/release-notes/task.yml - input_mapping: - git-repo: bosh-backup-and-restore-master - cryogenics-concourse-tasks: cryogenics-ci - params: - USE_LATEST_PUBLISHED_TAG: true - - task: format-release-notes - file: cryogenics-ci/tasks/release-automation/format-release-notes/task.yml - input_mapping: - template-folder: cryogenics-ci - cryogenics-concourse-tasks: cryogenics-ci - params: - VERSION: ((.:version-number)) - TEMPLATE_PATH: tasks/release-automation/release-notes-templates/docs-release-notes-auto.md.erb - - task: commit-release-notes - file: docs-bbr-main/ci/tasks/commit-new-release-notes.yml - input_mapping: - docs-repo: docs-bbr-main - params: - VERSION: ((.:version-number)) - GIT_AUTHOR_NAME: "Cryogenics CI" - GIT_AUTHOR_EMAIL: "mapbu-cryogenics@groups.vmware.com" - - put: docs-bbr-main - params: - repository: docs-repo - rebase: false - merge: false - force: true - branch: ((.:version-number))-release-notes - on_success: - try: - task: create-docs-pull-request - file: cryogenics-ci/tasks/create-pr/task.yml - params: - GITHUB_TOKEN: ((github.access_token)) - BASE: master - BRANCH: ((.:version-number))-release-notes - TITLE: ((.:version-number)) release notes - input_mapping: - repo: docs-bbr-main - cryogenics-concourse-tasks: cryogenics-ci - - in_parallel: - - put: bbr-pivnet - params: - metadata_file: pivnet-release-with-metadata/release.yml - s3_filepath_prefix: product-files/bosh-backup-restore - file_glob: pivnet-release-with-metadata/bbr* - override: true - - load_var: github-release-url - file: bbr-release/url + - task: generate-artifact-and-metadata + file: ci/ci/tasks/generate-bbr-release-metadata/task.yml + input_mapping: + version: bbr-final-release-version + promoted-artefacts: bbr-release + repo: ci + params: + TEMPLATE_PATH: ci/tasks/generate-bbr-release-metadata/release.yml.erb + VERSION_PATH: number + - put: bbr-pivnet + params: + metadata_file: pivnet-release-with-metadata/release.yml + s3_filepath_prefix: product-files/bosh-backup-restore + file_glob: pivnet-artefacts/* + override: true - task: create-story file: cryogenics-ci/tasks/tracker-automation/create-story/task.yml input_mapping: @@ -780,11 +721,11 @@ jobs: plan: - get: cryogenics-ci passed: - - build-and-publish-final + - publish-to-pivnet - get: bbr-final-release-version trigger: true passed: - - build-and-publish-final + - publish-to-pivnet - load_var: version-number file: bbr-final-release-version/number - task: create-osm-release @@ -840,9 +781,9 @@ jobs: passed: - publish-to-github trigger: true - - get: bosh-backup-and-restore + - get: ci - task: update-homebrew-formula - file: bosh-backup-and-restore/ci/tasks/bbr-update-homebrew-formula/task.yml + file: ci/ci/tasks/bbr-update-homebrew-formula/task.yml input_mapping: bbr-release: bbr-release - put: homebrew-tap diff --git a/ci/tasks/generate-bbr-release-metadata/task.sh b/ci/tasks/generate-bbr-release-metadata/task.sh index 0520b70eb..3afdffe24 100755 --- a/ci/tasks/generate-bbr-release-metadata/task.sh +++ b/ci/tasks/generate-bbr-release-metadata/task.sh @@ -1,60 +1,45 @@ #!/usr/bin/env bash set -euo pipefail +set -x -ROOT_DIR="$PWD" -PIVNET_FOLDER="${ROOT_DIR}/pivnet-release-with-metadata" -REL_PIVNET_FOLDER="pivnet-release-with-metadata" -GITHUB_FOLDER="${ROOT_DIR}/github-release-with-metadata" -REL_GITHUB_FOLDER="github-release-with-metadata" -RELEASE_FOLDER="${ROOT_DIR}/release" -RELEASE_TAR_FOLDER="${ROOT_DIR}/release-tar" -VERSION=$(cat "version-folder/${VERSION_PATH}") +[ -d promoted-artefacts ] +[ -d pivnet-artefacts ] +[ -d repo ] + +: "${TEMPLATE_PATH:?}" +: "${RELEASE_TYPE:?}" + +VERSION=$(cat "version/number") function main { - create_tarball - copy_tarball_to_folder "${GITHUB_FOLDER}" - copy_tarball_to_folder "${PIVNET_FOLDER}" - copy_release_files_to_folder "${GITHUB_FOLDER}" - delete_sha256_files - copy_release_files_to_folder "${PIVNET_FOLDER}" - export_release_metadata_variables - - erb -T- "template-folder/${TEMPLATE_PATH}" > "${PIVNET_FOLDER}/release.yml" - - echo -e "\n > Generated Tanzunet release file" - cat "${PIVNET_FOLDER}/release.yml" + copy_pivnet_files 'promoted-artefacts' 'pivnet-artefacts' + generate_pivnet_metadata "repo/${TEMPLATE_PATH}" 'pivnet-artefacts' } -function create_tarball { - echo "Creating release tarball..." - export TAR_NAME="bbr-${VERSION}.tar" - tar -cf "${TAR_NAME}" -C "${RELEASE_TAR_FOLDER}" . -} +function copy_pivnet_files() { + local promoted_artefacts="${1:?}" + local pivnet_artefacts="${2:?}" -function copy_tarball_to_folder { - echo "Adding tarball to: $1" - cp "${TAR_NAME}" "$1" -} + ls "$promoted_artefacts" + cp "$promoted_artefacts"/* "$pivnet_artefacts" -function delete_sha256_files { - echo "Removing sha256 files from tanzunet release" - # Why? The previous concourse job has generated shasums for each product, - # we have bundled this as part of the tar and do no need these extra files. - rm $RELEASE_FOLDER/*.sha256 + rm "$pivnet_artefacts"/*.sha256 } -function copy_release_files_to_folder { - cp -r "${RELEASE_FOLDER}/." "$1" -} +function generate_pivnet_metadata() { + local metadata_template="${1:?}" + local pivnet_artefacts="${2:?}" + + export BBR_LINUX_BINARY="$pivnet_artefacts/bbr-${VERSION}-linux-amd64" + export BBR_DARWIN_BINARY="$pivnet_artefacts/bbr-${VERSION}-darwin-amd64" + export RELEASE_TAR="$pivnet_artefacts/bbr-build/bbr-${VERSION}.tar" + export BBR_S3_VALIDATOR_BINARY="$pivnet_artefacts/bbr-s3-config-validator-${VERSION}-linux-amd64" + export BBR_S3_VALIDATOR_README="$pivnet_artefacts/bbr-s3-config-validator-${VERSION}.README.md" + export VERSION + export RELEASE_TYPE -function export_release_metadata_variables { - export BBR_LINUX_BINARY="${REL_PIVNET_FOLDER}/bbr-${VERSION}-linux-amd64" - export BBR_DARWIN_BINARY="${REL_PIVNET_FOLDER}/bbr-${VERSION}-darwin-amd64" - export RELEASE_TAR="${REL_PIVNET_FOLDER}/${TAR_NAME}" - export BBR_S3_VALIDATOR_BINARY="${REL_PIVNET_FOLDER}/bbr-s3-config-validator-${VERSION}-linux-amd64" - export BBR_S3_VALIDATOR_README="${REL_PIVNET_FOLDER}/bbr-s3-config-validator-${VERSION}.README.md" - export VERSION="${VERSION}" + erb -T- "$metadata_template" > "$pivnet_artefacts/release.yml" } main diff --git a/ci/tasks/generate-bbr-release-metadata/task.yml b/ci/tasks/generate-bbr-release-metadata/task.yml index 3d5c6681b..32d926658 100644 --- a/ci/tasks/generate-bbr-release-metadata/task.yml +++ b/ci/tasks/generate-bbr-release-metadata/task.yml @@ -6,20 +6,16 @@ image_resource: source: {repository: pcfplatformrecovery/backup-and-restore} inputs: -- name: release -- name: release-tar -- name: version-folder -- name: template-folder -- name: bosh-backup-and-restore-master +- name: version +- name: promoted-artefacts +- name: repo params: TEMPLATE_PATH: - VERSION_PATH: RELEASE_TYPE: Maintenance outputs: -- name: pivnet-release-with-metadata -- name: github-release-with-metadata +- name: pivnet-artefacts run: - path: bosh-backup-and-restore-master/ci/tasks/generate-bbr-release-metadata/task.sh + path: repo/ci/tasks/generate-bbr-release-metadata/task.sh From cd5f92cd181722f845aa137e087916fe9eea4ebb Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 14:13:04 +0100 Subject: [PATCH 15/21] [ci] improve generate-bbr-release-metadata task [#179484418] --- ci/tasks/generate-bbr-release-metadata/task.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/tasks/generate-bbr-release-metadata/task.sh b/ci/tasks/generate-bbr-release-metadata/task.sh index 3afdffe24..2fff46091 100755 --- a/ci/tasks/generate-bbr-release-metadata/task.sh +++ b/ci/tasks/generate-bbr-release-metadata/task.sh @@ -3,9 +3,10 @@ set -euo pipefail set -x +[ -d version ] [ -d promoted-artefacts ] -[ -d pivnet-artefacts ] [ -d repo ] +[ -d pivnet-artefacts ] : "${TEMPLATE_PATH:?}" : "${RELEASE_TYPE:?}" From a97fa15a3c93c39e0217a0b1f516dbf498e4e8ee Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 14:23:21 +0100 Subject: [PATCH 16/21] [ci] rename task [#179484418 ] --- ci/pipelines/bbr/pipeline.yml | 4 ++-- .../release.yml.erb | 0 .../task.sh | 0 .../task.yml | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename ci/tasks/{generate-bbr-release-metadata => prepare-artefacts-for-pivnet}/release.yml.erb (100%) rename ci/tasks/{generate-bbr-release-metadata => prepare-artefacts-for-pivnet}/task.sh (100%) rename ci/tasks/{generate-bbr-release-metadata => prepare-artefacts-for-pivnet}/task.yml (82%) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index d9d6b9a98..23b3b3d95 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -690,13 +690,13 @@ jobs: - load_var: version-number file: bbr-final-release-version/number - task: generate-artifact-and-metadata - file: ci/ci/tasks/generate-bbr-release-metadata/task.yml + file: ci/ci/tasks/prepare-artefacts-for-pivnet/task.yml input_mapping: version: bbr-final-release-version promoted-artefacts: bbr-release repo: ci params: - TEMPLATE_PATH: ci/tasks/generate-bbr-release-metadata/release.yml.erb + TEMPLATE_PATH: ci/tasks/prepare-artefacts-for-pivnet/release.yml.erb VERSION_PATH: number - put: bbr-pivnet params: diff --git a/ci/tasks/generate-bbr-release-metadata/release.yml.erb b/ci/tasks/prepare-artefacts-for-pivnet/release.yml.erb similarity index 100% rename from ci/tasks/generate-bbr-release-metadata/release.yml.erb rename to ci/tasks/prepare-artefacts-for-pivnet/release.yml.erb diff --git a/ci/tasks/generate-bbr-release-metadata/task.sh b/ci/tasks/prepare-artefacts-for-pivnet/task.sh similarity index 100% rename from ci/tasks/generate-bbr-release-metadata/task.sh rename to ci/tasks/prepare-artefacts-for-pivnet/task.sh diff --git a/ci/tasks/generate-bbr-release-metadata/task.yml b/ci/tasks/prepare-artefacts-for-pivnet/task.yml similarity index 82% rename from ci/tasks/generate-bbr-release-metadata/task.yml rename to ci/tasks/prepare-artefacts-for-pivnet/task.yml index 32d926658..18ba1a8c6 100644 --- a/ci/tasks/generate-bbr-release-metadata/task.yml +++ b/ci/tasks/prepare-artefacts-for-pivnet/task.yml @@ -18,4 +18,4 @@ outputs: - name: pivnet-artefacts run: - path: repo/ci/tasks/generate-bbr-release-metadata/task.sh + path: repo/ci/tasks/prepare-artefacts-for-pivnet/task.sh From 778fd79c6faa3f3cddd13eec6b4420d41a83802d Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 14:38:18 +0100 Subject: [PATCH 17/21] [ci] add validation to promote-rc task [#179484418 ] --- ci/pipelines/bbr/pipeline.yml | 22 +++++++++++---------- ci/tasks/promote-release-candidate/task.sh | 11 +++++++++-- ci/tasks/promote-release-candidate/task.yml | 8 ++++---- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index 23b3b3d95..ee0ada2b2 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -604,15 +604,6 @@ jobs: unpack: true - load_var: version-number file: bbr-final-release-version/number - - task: promote-rc-binaries - file: ci/ci/tasks/promote-release-candidate/task.yml - input_mapping: - bbr-rc-artefacts: rc - s3-config-validator-rc-artefacts: bbr-s3-config-validator-artifact - on_success: - put: bbr-final-release-version - params: - bump: patch - task: create-release-notes file: cryogenics-ci/tasks/release-automation/release-notes/task.yml input_mapping: @@ -655,6 +646,17 @@ jobs: input_mapping: repo: docs-bbr-main cryogenics-concourse-tasks: cryogenics-ci + - task: promote-rc-binaries + file: ci/ci/tasks/promote-release-candidate/task.yml + input_mapping: + version: bbr-final-release-version + bbr-rc-artefacts: rc + s3-config-validator-rc-artefacts: bbr-s3-config-validator-artifact + repo: ci + on_success: + put: bbr-final-release-version + params: + bump: patch - put: bbr-release params: name: bbr-final-release-version/number @@ -663,7 +665,7 @@ jobs: body: release-notes/release-notes.md commitish: bosh-backup-and-restore-master/.git/refs/heads/master globs: - - bbr-final-release-artefacts/bbr* + - promoted-artefacts/bbr* on_success: do: - load_var: github-release-url diff --git a/ci/tasks/promote-release-candidate/task.sh b/ci/tasks/promote-release-candidate/task.sh index 8bd84977f..9e6d845a4 100755 --- a/ci/tasks/promote-release-candidate/task.sh +++ b/ci/tasks/promote-release-candidate/task.sh @@ -3,10 +3,17 @@ set -euo pipefail set -x -VERSION=$(cat bbr-final-release-version/number) +VERSION=$(cat version/number) + +[ -d 'version' ] +[ -d 'bbr-rc-artefacts' ] +[ -d 's3-config-validator-rc-artefacts' ] +[ -d 's3-config-validator-rc-artefacts' ] +[ -d 'repo' ] +[ -d 'promoted-artefacts' ] function main { - local release_folder='bbr-final-release-artefacts' + local release_folder='promoted-artefacts' copy_bbr_binaries_to "$release_folder" copy_s3_config_validator_artefacts_to "$release_folder" diff --git a/ci/tasks/promote-release-candidate/task.yml b/ci/tasks/promote-release-candidate/task.yml index 8f345bb44..b69a58bf0 100644 --- a/ci/tasks/promote-release-candidate/task.yml +++ b/ci/tasks/promote-release-candidate/task.yml @@ -6,13 +6,13 @@ image_resource: source: {repository: pcfplatformrecovery/backup-and-restore} inputs: -- name: bbr-final-release-version +- name: version - name: bbr-rc-artefacts - name: s3-config-validator-rc-artefacts -- name: ci +- name: repo outputs: -- name: bbr-final-release-artefacts +- name: promoted-artefacts run: - path: ci/ci/tasks/promote-release-candidate/task.sh + path: repo/ci/tasks/promote-release-candidate/task.sh From 1fa53b3ac46105282c6d3b6f6a2dfb58d76721ea Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 17:14:22 +0100 Subject: [PATCH 18/21] [ci] rename task input [#179484418 ] --- ci/tasks/bbr-build/task.sh | 4 +++- ci/tasks/bbr-build/task.yml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/tasks/bbr-build/task.sh b/ci/tasks/bbr-build/task.sh index a4419a700..dd5731f84 100755 --- a/ci/tasks/bbr-build/task.sh +++ b/ci/tasks/bbr-build/task.sh @@ -2,12 +2,14 @@ set -eu +[ -d 'version' ] + eval "$(ssh-agent)" echo -e "${GITHUB_SDK_PRIVATE_KEY}" > "${PWD}/github-sdk.key" chmod 400 "${PWD}/github-sdk.key" ssh-add "${PWD}/github-sdk.key" -VERSION=$(cat bbr-final-release-version/number) +VERSION=$(cat version/number) export VERSION BBR_REPO="bosh-backup-and-restore" diff --git a/ci/tasks/bbr-build/task.yml b/ci/tasks/bbr-build/task.yml index 1b7d37fdf..b657aea6a 100644 --- a/ci/tasks/bbr-build/task.yml +++ b/ci/tasks/bbr-build/task.yml @@ -7,7 +7,7 @@ image_resource: inputs: - name: bosh-backup-and-restore -- name: bbr-final-release-version +- name: version params: GITHUB_SDK_PRIVATE_KEY: From 4cf2f33953a1db8ce04df85bb6ae495ea8efda2b Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Fri, 17 Sep 2021 17:25:25 +0100 Subject: [PATCH 19/21] [ci] simplify s3-validator build task [#179484418 ] --- ci/tasks/bbr-s3-config-validator-build/task.sh | 10 ++++++++-- ci/tasks/bbr-s3-config-validator-build/task.yml | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ci/tasks/bbr-s3-config-validator-build/task.sh b/ci/tasks/bbr-s3-config-validator-build/task.sh index f7301398b..45883e833 100755 --- a/ci/tasks/bbr-s3-config-validator-build/task.sh +++ b/ci/tasks/bbr-s3-config-validator-build/task.sh @@ -1,13 +1,19 @@ #!/usr/bin/env bash set -euo pipefail +set -x + +[ -d version ] +[ -d repo ] + +VERSION="$(cat version/number)" ROOT_DIR=$PWD -cd bosh-backup-and-restore/s3-config-validator +cd repo/s3-config-validator make artifact cp -r \ build/artifact.tgz \ - "${ROOT_DIR}/bbr-s3-config-validator-test-artifacts/bbr-s3-config-validator.$(cat "${ROOT_DIR}"/s3-config-validator-version/number).tgz" + "${ROOT_DIR}/bbr-s3-config-validator-test-artifacts/bbr-s3-config-validator.$VERSION.tgz" diff --git a/ci/tasks/bbr-s3-config-validator-build/task.yml b/ci/tasks/bbr-s3-config-validator-build/task.yml index 988a907e1..7a4cb2a84 100644 --- a/ci/tasks/bbr-s3-config-validator-build/task.yml +++ b/ci/tasks/bbr-s3-config-validator-build/task.yml @@ -7,11 +7,11 @@ image_resource: repository: pcfplatformrecovery/backup-and-restore-minimal inputs: -- name: s3-config-validator-version -- name: bosh-backup-and-restore +- name: version +- name: repo outputs: - name: bbr-s3-config-validator-test-artifacts run: - path: bosh-backup-and-restore/ci/tasks/bbr-s3-config-validator-build/task.sh + path: repo/ci/tasks/bbr-s3-config-validator-build/task.sh From 6e6efc15e6c2903a54fb994e16afd175ad33bb77 Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Mon, 20 Sep 2021 11:09:26 +0100 Subject: [PATCH 20/21] [ci] extract promote-rc job [#179484418] --- ci/pipelines/bbr/pipeline.yml | 117 ++++++++++++++++++++++++---------- 1 file changed, 85 insertions(+), 32 deletions(-) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index ee0ada2b2..9beed7096 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -32,6 +32,7 @@ groups: - system-test-director - build-rc - update-tracker + - promote-latest-rc - publish-to-github - publish-to-pivnet - request-osl @@ -56,6 +57,7 @@ groups: - name: shipit jobs: - build-rc + - promote-latest-rc - publish-to-github - publish-to-pivnet - request-osl @@ -63,6 +65,10 @@ groups: - update-homebrew-formula resource_types: +- name: github-release + type: registry-image + source: {repository: concourse/github-release-resource} + - name: gitgat type: docker-image source: @@ -182,22 +188,44 @@ resources: access_key_id: ((aws_credentials.access_key_id)) secret_access_key: ((aws_credentials.secret_access_key)) -- name: bbr-release +- name: bbr-draft-release type: github-release icon: github source: + repository: bosh-backup-and-restore user: cloudfoundry-incubator + access_token: ((github.access_token)) + release: false + drafts: true + +- name: bbr-release + type: github-release + icon: github + source: repository: bosh-backup-and-restore + user: cloudfoundry-incubator access_token: ((github.access_token)) + release: true + pre_release: false - name: bbr-final-release-version type: semver icon: google source: bucket: release-versions - key: bbr-final + key: bosh-backup-and-restore/final + json_key: ((gcp/service_accounts/bucket_owner.json_key)) + initial_version: 1.9.13 + driver: gcs + +- name: release-candidate-version + type: semver + icon: google + source: + bucket: release-versions + key: bosh-backup-and-restore/release-candidate json_key: ((gcp/service_accounts/bucket_owner.json_key)) - initial_version: 1.9.0 + initial_version: 1.9.13-rc.1 driver: gcs - name: docs-bbr-main @@ -454,6 +482,9 @@ jobs: AWS_SECRET_KEY: ((s3_validator_test_bucket.secret_key)) - task: build file: bosh-backup-and-restore/ci/tasks/bbr-s3-config-validator-build/task.yml + input_mapping: + version: s3-config-validator-version + repo: bosh-backup-and-restore - put: s3-config-validator-version params: { file: s3-config-validator-version/number } - put: bbr-s3-config-validator-test-artifacts @@ -556,44 +587,38 @@ jobs: params: submodules: none disable_git_lfs: true - - get: bbr-final-release-version + - get: release-candidate-version params: pre: rc - task: build-bbr file: bosh-backup-and-restore/ci/tasks/bbr-build/task.yml params: GITHUB_SDK_PRIVATE_KEY: ((github-sdk.key)) + input_mapping: + version: release-candidate-version - task: build-s3-config-validator file: bosh-backup-and-restore/ci/tasks/bbr-s3-config-validator-build/task.yml input_mapping: - s3-config-validator-version: bbr-final-release-version - - put: bbr-final-release-version - params: { file: bbr-final-release-version/number } + version: release-candidate-version + repo: bosh-backup-and-restore - put: rc params: {file: bbr-build/bbr-*.tar} - put: bbr-s3-config-validator-test-artifacts params: file: bbr-s3-config-validator-test-artifacts/* -- name: publish-to-github +- name: promote-latest-rc serial: true plan: - in_parallel: - get: every-3-weeks trigger: true - - get: bbr-final-release-version + - get: ci + - get: release-candidate-version params: bump: final passed: - build-rc - - get: cryogenics-ci - - get: bosh-backup-and-restore-master - passed: [build-rc] - params: - submodules: none - disable_git_lfs: true - - get: docs-bbr-main - - get: ci - get: rc params: unpack: true @@ -602,6 +627,49 @@ jobs: - get: bbr-s3-config-validator-artifact params: unpack: true + - get: bosh-backup-and-restore-master + - task: promote-rc-binaries + file: ci/ci/tasks/promote-release-candidate/task.yml + input_mapping: + version: release-candidate-version + bbr-rc-artefacts: rc + s3-config-validator-rc-artefacts: bbr-s3-config-validator-artifact + repo: ci + - put: bbr-final-release-version + params: + file: release-candidate-version/number + - put: release-candidate-version + params: + bump: patch + pre: rc + - put: bbr-draft-release + params: + name: bbr-final-release-version/number + tag: bbr-final-release-version/number + tag_prefix: v + commitish: bosh-backup-and-restore-master/.git/refs/heads/master + globs: + - promoted-artefacts/bbr* + +- name: publish-to-github + serial: true + plan: + - in_parallel: + - get: cryogenics-ci + - get: bosh-backup-and-restore-master + passed: [promote-latest-rc] + params: + submodules: none + disable_git_lfs: true + - get: docs-bbr-main + - get: ci + - get: bbr-final-release-version + passed: + - promote-latest-rc + - get: bbr-draft-release + trigger: true + passed: + - promote-latest-rc - load_var: version-number file: bbr-final-release-version/number - task: create-release-notes @@ -646,26 +714,12 @@ jobs: input_mapping: repo: docs-bbr-main cryogenics-concourse-tasks: cryogenics-ci - - task: promote-rc-binaries - file: ci/ci/tasks/promote-release-candidate/task.yml - input_mapping: - version: bbr-final-release-version - bbr-rc-artefacts: rc - s3-config-validator-rc-artefacts: bbr-s3-config-validator-artifact - repo: ci - on_success: - put: bbr-final-release-version - params: - bump: patch - put: bbr-release params: name: bbr-final-release-version/number tag: bbr-final-release-version/number tag_prefix: v body: release-notes/release-notes.md - commitish: bosh-backup-and-restore-master/.git/refs/heads/master - globs: - - promoted-artefacts/bbr* on_success: do: - load_var: github-release-url @@ -778,7 +832,6 @@ jobs: serial: true plan: - in_parallel: - - get: homebrew-tap - get: bbr-release passed: - publish-to-github From e6bee8d8c00cfd481f22d8fc63389e64e98c45bd Mon Sep 17 00:00:00 2001 From: Diego Lemos Date: Mon, 20 Sep 2021 11:28:31 +0100 Subject: [PATCH 21/21] [ci] replace ci resource by bbr-repo Maybe this was serving some purpose in the past. I could not find any reason why this should be split. On the flip side, it makes the pipeline more complex and difficult to reason about. [#179484418] --- ci/pipelines/bbr/pipeline.yml | 70 +++++++++++++++-------------------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/ci/pipelines/bbr/pipeline.yml b/ci/pipelines/bbr/pipeline.yml index 9beed7096..e146ea64b 100644 --- a/ci/pipelines/bbr/pipeline.yml +++ b/ci/pipelines/bbr/pipeline.yml @@ -152,16 +152,6 @@ resources: branch: master disable_git_lfs: true -- name: ci - type: git - icon: github - source: - uri: git@github.com:cloudfoundry-incubator/bosh-backup-and-restore.git - private_key: ((github.ssh_key)) - branch: master - paths: - - ci - - name: homebrew-tap type: git icon: github @@ -613,7 +603,6 @@ jobs: - in_parallel: - get: every-3-weeks trigger: true - - get: ci - get: release-candidate-version params: bump: final @@ -628,13 +617,15 @@ jobs: params: unpack: true - get: bosh-backup-and-restore-master + passed: + - build-rc - task: promote-rc-binaries - file: ci/ci/tasks/promote-release-candidate/task.yml + file: bosh-backup-and-restore-master/ci/tasks/promote-release-candidate/task.yml input_mapping: version: release-candidate-version bbr-rc-artefacts: rc s3-config-validator-rc-artefacts: bbr-s3-config-validator-artifact - repo: ci + repo: bosh-backup-and-restore-master - put: bbr-final-release-version params: file: release-candidate-version/number @@ -662,7 +653,6 @@ jobs: submodules: none disable_git_lfs: true - get: docs-bbr-main - - get: ci - get: bbr-final-release-version passed: - promote-latest-rc @@ -739,20 +729,22 @@ jobs: - get: bbr-final-release-version passed: - publish-to-github - - get: ci + - get: bosh-backup-and-restore-master + passed: + - publish-to-github - get: bbr-release passed: - publish-to-github - load_var: version-number file: bbr-final-release-version/number - task: generate-artifact-and-metadata - file: ci/ci/tasks/prepare-artefacts-for-pivnet/task.yml + file: bosh-backup-and-restore-master/ci/tasks/prepare-artefacts-for-pivnet/task.yml input_mapping: version: bbr-final-release-version promoted-artefacts: bbr-release - repo: ci + repo: bosh-backup-and-restore-master params: - TEMPLATE_PATH: ci/tasks/prepare-artefacts-for-pivnet/release.yml.erb + TEMPLATE_PATH: bosh-backup-and-restore-master/tasks/prepare-artefacts-for-pivnet/release.yml.erb VERSION_PATH: number - put: bbr-pivnet params: @@ -772,12 +764,30 @@ jobs: STORY_LABELS: 'bbr' STORY_BLOCKERS: '{"description":"OSL file"}' +- name: update-homebrew-formula + serial: true + plan: + - in_parallel: + - get: bbr-release + passed: + - publish-to-github + trigger: true + - get: bosh-backup-and-restore-master + passed: + - publish-to-github + - task: update-homebrew-formula + file: bosh-backup-and-restore-master/ci/tasks/bbr-update-homebrew-formula/task.yml + input_mapping: + bbr-release: bbr-release + - put: homebrew-tap + params: + repository: updated-homebrew-tap + rebase: true + - name: request-osl serial_groups: [version] plan: - get: cryogenics-ci - passed: - - publish-to-pivnet - get: bbr-final-release-version trigger: true passed: @@ -798,8 +808,6 @@ jobs: plan: - in_parallel: - get: cryogenics-ci - passed: - - request-osl - get: bbr-final-release-version passed: - request-osl @@ -827,21 +835,3 @@ jobs: AWS_ACCESS_KEY_ID: ((pivnet_bucket.access_key_id)) AWS_SECRET_ACCESS_KEY: ((pivnet_bucket.secret_access_key)) AWS_PRODUCT_NAME: ((pivnet_bucket.user)) - -- name: update-homebrew-formula - serial: true - plan: - - in_parallel: - - get: bbr-release - passed: - - publish-to-github - trigger: true - - get: ci - - task: update-homebrew-formula - file: ci/ci/tasks/bbr-update-homebrew-formula/task.yml - input_mapping: - bbr-release: bbr-release - - put: homebrew-tap - params: - repository: updated-homebrew-tap - rebase: true