From ab53c586a57433fa9f5c79b9e84d4e239854bad5 Mon Sep 17 00:00:00 2001 From: alextruninn Date: Thu, 24 Aug 2023 10:59:37 +0200 Subject: [PATCH] =?UTF-8?q?CLOUD-64700:=20[kms]=20=D0=90=D0=B2=D1=82=D0=BE?= =?UTF-8?q?=D0=BC=D0=B0=D1=82=D0=B8=D0=B7=D0=B8=D1=80=D0=BE=D0=B2=D0=B0?= =?UTF-8?q?=D1=82=D1=8C=20=D0=BF=D1=80=D0=BE=D1=86=D0=B5=D0=B4=D1=83=D1=80?= =?UTF-8?q?=D1=83=20=D0=BE=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8?= =?UTF-8?q?=D1=8F=20docker=20=D0=B8=20compute=20=D0=BE=D0=B1=D1=80=D0=B0?= =?UTF-8?q?=D0=B7=D0=BE=D0=B2=20Vault+KMS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- yandex/scripts/common.sh | 78 ++++++++++++ .../push_compute/push_compute_image.sh | 30 +++++ .../push_compute/push_compute_sample.cfg | 22 ++++ .../scripts/push_docker/push_docker_image.sh | 36 ++++++ .../push_docker/push_docker_sample.cfg | 15 +++ .../push_helm_chart/helm_chart_sample.cfg | 15 +++ .../push_helm_chart/push_helm_chart.sh | 102 ++++++++++++++++ .../scripts/push_helm_chart/update_values.sh | 58 +++++++++ yandex/scripts/release/cleanup.sh | 9 ++ yandex/scripts/release/patch_version_base.sh | 11 ++ yandex/scripts/release/release.sh | 74 ++++++++++++ yandex/scripts/release/release_sample.cfg | 11 ++ yandex/scripts/release/release_wrapper.sh | 67 ++++++++++ .../script_inside_ssh.sh | 20 +++ .../test_docker_container.sh | 114 ++++++++++++++++++ .../test_docker_sample.cfg | 21 ++++ .../test_docker_container/vol/startup.sh | 34 ++++++ yandex/scripts/test_local/test_local.sh | 100 +++++++++++++++ .../scripts/test_local/test_local_sample.cfg | 16 +++ 19 files changed, 833 insertions(+) create mode 100644 yandex/scripts/common.sh create mode 100644 yandex/scripts/push_compute/push_compute_image.sh create mode 100644 yandex/scripts/push_compute/push_compute_sample.cfg create mode 100644 yandex/scripts/push_docker/push_docker_image.sh create mode 100644 yandex/scripts/push_docker/push_docker_sample.cfg create mode 100644 yandex/scripts/push_helm_chart/helm_chart_sample.cfg create mode 100644 yandex/scripts/push_helm_chart/push_helm_chart.sh create mode 100644 yandex/scripts/push_helm_chart/update_values.sh create mode 100755 yandex/scripts/release/cleanup.sh create mode 100755 yandex/scripts/release/patch_version_base.sh create mode 100755 yandex/scripts/release/release.sh create mode 100644 yandex/scripts/release/release_sample.cfg create mode 100755 yandex/scripts/release/release_wrapper.sh create mode 100644 yandex/scripts/test_docker_container/script_inside_ssh.sh create mode 100644 yandex/scripts/test_docker_container/test_docker_container.sh create mode 100644 yandex/scripts/test_docker_container/test_docker_sample.cfg create mode 100644 yandex/scripts/test_docker_container/vol/startup.sh create mode 100755 yandex/scripts/test_local/test_local.sh create mode 100644 yandex/scripts/test_local/test_local_sample.cfg diff --git a/yandex/scripts/common.sh b/yandex/scripts/common.sh new file mode 100644 index 000000000000..29fd302f6607 --- /dev/null +++ b/yandex/scripts/common.sh @@ -0,0 +1,78 @@ +init() { + START_DIR=$(pwd) + trap 'cd $START_DIR' EXIT + + SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") + cd $SCRIPT_DIR + SCRIPT_DIR=$(pwd) + + if [[ -n $WORK_DIR ]]; then + mkdir -p $WORK_DIR + cd $WORK_DIR + fi + WORK_DIR=$(pwd) + YCKMS_VERSION=${PATCH_BRANCH:-"$BASE_VERSION+yckms"} +} + +go_to_work_dir() { + cd $WORK_DIR +} + +cleanup() { + go_to_work_dir + rm -rf vault + rm -rf vault-kms-wrapper +} + +init_vault() { + go_to_work_dir + if [[ ! -d "vault" ]]; then + echo "Cloning vault" + git clone git@github.com:yandex-cloud/vault.git + cd vault + git remote add upstream git@github.com:hashicorp/vault.git + else + echo "Vault already cloned" + cd vault + git reset --hard + fi + + echo "Synchronizing vault with upstream" + git checkout main + git pull upstream main + echo "Fetching tags" + git fetch upstream --tags + + go_to_work_dir +} + +init_vault_kms_wrapper() { + go_to_work_dir + if [[ ! -d "vault-kms-wrapper" ]]; then + echo "Cloning vault-kms-wrapper" + git clone git@github.com:yandex-cloud/vault-kms-wrapper.git + cd vault-kms-wrapper + else + echo "Vault already cloned" + cd vault-kms-wrapper + fi + + echo "Refreshing main" + git reset --hard + git checkout main + git pull + git fetch -p + + go_to_work_dir +} + +get_kms_wrapper_version() { + go_to_work_dir + cd vault + CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + git checkout $BASE_VERSION + KMS_WRAPPER_VERSION=$(go list -m github.com/hashicorp/go-kms-wrapping/v2 | cut -f 2 -d " ") + echo "Current go-kms-wrapping version: $KMS_WRAPPER_VERSION" + git checkout $CURRENT_BRANCH + go_to_work_dir +} diff --git a/yandex/scripts/push_compute/push_compute_image.sh b/yandex/scripts/push_compute/push_compute_image.sh new file mode 100644 index 000000000000..798cbf22357b --- /dev/null +++ b/yandex/scripts/push_compute/push_compute_image.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e + +SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}") +. $SCRIPT_PATH/../common.sh +. $SCRIPT_PATH/push_compute.cfg + +init +init_vault +cd vault + +if ! git checkout $YCKMS_VERSION; then + echo >&2 "Cannot find branch $YCKMS_VERSION !" + exit 1 +else + echo "Checkout to branch '$YCKMS_VERSION' succeeded" +fi + +TOKEN=$(ycp --profile $PROFILE iam create-token) + +cd yandex/compute + +if [[ $REPLACE_DEFAULT_ENDPOINT == 1 ]]; then + echo "$(jq --arg ENDPOINT "$ENDPOINT" --arg ZONE "$ZONE" --arg PLATFORM_ID "$PLATFORM_ID" \ + '.builders[0] += {"endpoint":$ENDPOINT, "zone":$ZONE, "platform_id":$PLATFORM_ID}' vault.packer.json)" > vault.packer.json +fi + +echo "$(jq --arg SOURCE_IMAGE_FAMILY "ubuntu-2004-lts" '.builders[0].source_image_family = $SOURCE_IMAGE_FAMILY' vault.packer.json)" > vault.packer.json + +FOLDER_ID="$FOLDER_ID" TOKEN="$TOKEN" $SCRIPT_PATH/packer build vault.packer.json \ No newline at end of file diff --git a/yandex/scripts/push_compute/push_compute_sample.cfg b/yandex/scripts/push_compute/push_compute_sample.cfg new file mode 100644 index 000000000000..239b4d132243 --- /dev/null +++ b/yandex/scripts/push_compute/push_compute_sample.cfg @@ -0,0 +1,22 @@ +#Requirements: +# - yc and ycp +# - jq +# - packer (from https://wiki.yandex-team.ru/cloud/devel/assembly-workshop/#packer) +# packer should be placed inside directory with script +# Github already contains release version of $BASE_VERSION ($BASE_VERSION+$WRAPPER_SUFFIX) + +#Version of patched vault from which will be build image(from git branch $BASE_VERSION+yckms) +BASE_VERSION="v1.14.1" +#Directory inside which patched version will be pulled from git +WORK_DIR=./../tmp +#If need to replace default endpoint(default is cloud.yandex.ru) +REPLACE_DEFAULT_ENDPOINT=1 +ENDPOINT=api.il.nebius.cloud:443 +ZONE=il1-a +PLATFORM_ID=standard-v3 +#Folder inside which image will be pulled +FOLDER_ID=b48ans97du1s547k9e9m +PROFILE=israel +#Branch of git inside https://github.com/yandex-cloud/vault +#from which will be build image(if not set $BASE_VERSION+yckms will be used) +PATCH_BRANCH=test \ No newline at end of file diff --git a/yandex/scripts/push_docker/push_docker_image.sh b/yandex/scripts/push_docker/push_docker_image.sh new file mode 100644 index 000000000000..1dafeb5cff85 --- /dev/null +++ b/yandex/scripts/push_docker/push_docker_image.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -e + +SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}") + . $SCRIPT_PATH/../common.sh + . $SCRIPT_PATH/push_docker.cfg + +init +init_vault +cd vault + +if ! git checkout $YCKMS_VERSION; then + echo >&2 "Cannot find branch $YCKMS_VERSION !" + exit 1 +else + echo "Checkout to branch '$YCKMS_VERSION' succeeded" +fi + +cd yandex +cd docker + +IAM_TOKEN=$(ycp --profile="$PROFILE" iam create-token) +DOMAIN="cr.yandex" + +if [[ "$PROFILE" == "israel" ]]; then + DOMAIN="cr.cloudil.com" +fi + +if [[ "$PROFILE" == "preprod" ]]; then + DOMAIN="cr.cloud-preprod.yandex.net" +fi + +docker login --username iam --password $IAM_TOKEN $DOMAIN +docker build --platform linux/amd64 -t $DOMAIN/$REGISTRY_ID/vault:$BASE_VERSION-yckms -t $DOMAIN/$REGISTRY_ID/vault . +docker push $DOMAIN/$REGISTRY_ID/vault:$BASE_VERSION-yckms +docker logout \ No newline at end of file diff --git a/yandex/scripts/push_docker/push_docker_sample.cfg b/yandex/scripts/push_docker/push_docker_sample.cfg new file mode 100644 index 000000000000..1c9130f59aa8 --- /dev/null +++ b/yandex/scripts/push_docker/push_docker_sample.cfg @@ -0,0 +1,15 @@ +#Requirements: +# - yc and ycp +# - docker +# Github already contains release version of $BASE_VERSION ($BASE_VERSION+$WRAPPER_SUFFIX) + +#Version of patched vault from which will be build image(from git branch $BASE_VERSION+yckms) +BASE_VERSION="v1.14.1" +#Directory inside which patched version will be pulled from git +WORK_DIR=./../tmp +PROFILE=israel +#Registry to which docker image will be pushed +REGISTRY_ID=crls41le9juekjee2rv4 +#Branch of git inside https://github.com/yandex-cloud/vault +#from which will be build image(if not set $BASE_VERSION+yckms will be used) +PATCH_BRANCH=test \ No newline at end of file diff --git a/yandex/scripts/push_helm_chart/helm_chart_sample.cfg b/yandex/scripts/push_helm_chart/helm_chart_sample.cfg new file mode 100644 index 000000000000..af8e02f36ae1 --- /dev/null +++ b/yandex/scripts/push_helm_chart/helm_chart_sample.cfg @@ -0,0 +1,15 @@ +#Requirements: +# - yc and ycp +# - docker +# - helm +# - yq +# - diff, patch, perl (should be included in default distributive) +# - Build and pulled docker image of corresponding vault version(BASE_VERSION parameter) + +# Docker image version of patched version that will be used for helm (Version is $BASE_VERSION-yckms) +BASE_VERSION="v1.14.1" +#Diractory in will be pulled vault-csi-provider and vault-k8s +WORK_DIR=./../tmp +PROFILE=israel +#Registry inside which will be pushed helm image +REGISTRY_ID=crls41le9juekjee2rv4 \ No newline at end of file diff --git a/yandex/scripts/push_helm_chart/push_helm_chart.sh b/yandex/scripts/push_helm_chart/push_helm_chart.sh new file mode 100644 index 000000000000..6607a580fbc3 --- /dev/null +++ b/yandex/scripts/push_helm_chart/push_helm_chart.sh @@ -0,0 +1,102 @@ +#!/bin/bash +set -e + +SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}") + . $SCRIPT_PATH/helm_chart.cfg + +if [[ -n $WORK_DIR ]]; then + mkdir -p $WORK_DIR + cd $WORK_DIR +fi + +IAM_TOKEN=$(ycp --profile="$PROFILE" iam create-token) +DOMAIN="cr.yandex" + +if [[ "$PROFILE" == "israel" ]]; then + DOMAIN="cr.cloudil.com" +fi + +if [[ "$PROFILE" == "preprod" ]]; then + DOMAIN="cr.cloud-preprod.yandex.net" +fi + +docker login --username iam --password $IAM_TOKEN $DOMAIN + +git clone https://github.com/hashicorp/vault-helm.git ./vault-helm +git clone ssh://git@bb.yandexcloud.net/cloud/mk8s-marketplace-helm.git ./mk8s-marketplace-helm + +cd vault-helm + +TAGS=$(git tag --sort="-version:refname") +echo $TAGS + +VERSION=$(echo "$BASE_VERSION" | cut -c 2-) +CUT_VERSION="${VERSION%.*}" +ACTUAL_TAG='' + +for TAG in $TAGS +do + git checkout tags/$TAG values.yaml + ACTUAL_VERSION=$(yq '.server.image.tag' values.yaml) + ACTUAL_VERSION="${ACTUAL_VERSION%.*}" + echo "Actual version = $ACTUAL_VERSION and version = $CUT_VERSION" + if [ $CUT_VERSION = $ACTUAL_VERSION ] + then + ACTUAL_TAG=$TAG + echo "Found actual version $ACTUAL_VERSION" + break + fi +done + +git checkout tags/$ACTUAL_TAG + +sudo rm -r ./.git + +CSI_PROVIDER_ACTUAL_TAG=$(yq '.csi.image.tag' values.yaml) +K8S_ACTUAL_TAG=$(yq '.injector.image.tag' values.yaml) + +cd ../ + +docker pull hashicorp/vault-csi-provider:$CSI_PROVIDER_ACTUAL_TAG --platform amd64 +docker pull hashicorp/vault-k8s:$K8S_ACTUAL_TAG --platform amd64 + +CSI_PROVIDER_IMAGE_ID=$(docker images hashicorp/vault-csi-provider:$CSI_PROVIDER_ACTUAL_TAG -q) +K8S_IMAGE_ID=$(docker images hashicorp/vault-k8s:$K8S_ACTUAL_TAG -q) + +docker tag $K8S_IMAGE_ID $DOMAIN/$REGISTRY_ID/vault/vault-k8s:$K8S_ACTUAL_TAG +docker tag $K8S_IMAGE_ID $DOMAIN/$REGISTRY_ID/vault/vault-k8s:latest + +docker tag $CSI_PROVIDER_IMAGE_ID $DOMAIN/$REGISTRY_ID/vault/vault-csi-provider:$CSI_PROVIDER_ACTUAL_TAG +docker tag $CSI_PROVIDER_IMAGE_ID $DOMAIN/$REGISTRY_ID/vault/vault-csi-provider:latest + +docker push $DOMAIN/$REGISTRY_ID/vault/vault-k8s:$K8S_ACTUAL_TAG +docker push $DOMAIN/$REGISTRY_ID/vault/vault-k8s:latest + +docker push $DOMAIN/$REGISTRY_ID/vault/vault-csi-provider:$CSI_PROVIDER_ACTUAL_TAG +docker push $DOMAIN/$REGISTRY_ID/vault/vault-csi-provider:latest + +rm -r ./mk8s-marketplace-helm/products/hashicorp-vault/chart +cp -r ./vault-helm ./mk8s-marketplace-helm/products/hashicorp-vault/chart +rm -r ./mk8s-marketplace-helm/products/hashicorp-vault/chart/test + +CHART_VERSION="$(yq '.version' ./vault-helm/Chart.yaml)-1" +replacement=$CHART_VERSION yq -i '.version = strenv(replacement)' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/Chart.yaml + +cat >./mk8s-marketplace-helm/products/hashicorp-vault/chart/templates/kms-creds-secret.yaml < values.yaml.new + +DOMAIN="cr.yandex" +if [[ "$PROFILE" == "israel" ]]; then + DOMAIN="cr.cloudil.com" +fi +if [[ "$PROFILE" == "preprod" ]]; then + DOMAIN="cr.cloud-preprod.yandex.net" +fi +VAULT_VERSION="$(echo "$BASE_VERSION" | cut -c 2-)-yckms" + +replacement=$VAULT_VERSION yq -i '(.. | select(key == "repository" and . == "hashicorp/vault") | parent).tag |= strenv(replacement)' values.yaml.new +replacement="$DOMAIN/$REGISTRY_ID/vault/vault" yq -i '(.. | select(key == "repository" and . == "hashicorp/vault") | parent).repository |= strenv(replacement)' values.yaml.new +replacement=$CSI_PROVIDER_ACTUAL_TAG yq -i '(.. | select(key == "repository" and . == "hashicorp/vault-csi-provider") | parent).tag |= strenv(replacement)' values.yaml.new +replacement="$DOMAIN/$REGISTRY_ID/vault/vault-csi-provider" yq -i '(.. | select(key == "repository" and . == "hashicorp/vault-csi-provider") | parent).repository |= strenv(replacement)' values.yaml.new +replacement=$K8S_ACTUAL_TAG yq -i '(.. | select(key == "repository" and . == "hashicorp/vault-k8s") | parent).tag |= strenv(replacement)' values.yaml.new +replacement="$DOMAIN/$REGISTRY_ID/vault/vault-k8s" yq -i '(.. | select(key == "repository" and . == "hashicorp/vault-k8s") | parent).repository |= strenv(replacement)' values.yaml.new +yq -i '(.. | select(key == "extraVolumes")) |= {"type": "secret", "name": "kms-creds"} ' values.yaml.new + +yq '.' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml > values.yaml.noblanks +diff -B values.yaml.noblanks values.yaml.new > ./patch.file +patch ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml ./patch.file + +#patch works bad, so it's needed to patch twice +yq '.' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml > values.yaml.new + +read -rd '' replacement << EOF + +seal "yandexcloudkms" { + kms_key_id = "{{ .Values.yandexKmsKeyId}}" + service_account_key_file = "/vault/userconfig/kms-creds/credentials.json" +} + +EOF +replacement=$replacement yq -i '(.. | select(key == "standalone")).config |= . + strenv(replacement)' values.yaml.new + +yq '.' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml > values.yaml.noblanks +diff -B values.yaml.noblanks values.yaml.new > ./patch.file +patch ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml ./patch.file + +echo 'yandexKmsAuthJson: ""' >> ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml +echo 'yandexKmsKeyId: ""' >> ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml + +perl -pe 'chomp if eof' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml >tmp.file +mv tmp.file ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml + +rm patch.file +rm values.yaml.new +rm values.yaml.noblanks +rm ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml.orig \ No newline at end of file diff --git a/yandex/scripts/release/cleanup.sh b/yandex/scripts/release/cleanup.sh new file mode 100755 index 000000000000..05312c270c0a --- /dev/null +++ b/yandex/scripts/release/cleanup.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}") +. $SCRIPT_PATH/common.sh +. $SCRIPT_PATH/release_sample.cfg + +init +cleanup diff --git a/yandex/scripts/release/patch_version_base.sh b/yandex/scripts/release/patch_version_base.sh new file mode 100755 index 000000000000..ed67f5f99ea3 --- /dev/null +++ b/yandex/scripts/release/patch_version_base.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +START_DIR=$(pwd) +trap 'cd $START_DIR' EXIT + +SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}") +cd "$SCRIPT_PATH"/.. + +sed -i '' 's/.*VersionMetadata.*=.*""/VersionMetadata = "yckms"/' version/version_base.go +go fmt version/version_base.go diff --git a/yandex/scripts/release/release.sh b/yandex/scripts/release/release.sh new file mode 100755 index 000000000000..ef15f6574b96 --- /dev/null +++ b/yandex/scripts/release/release.sh @@ -0,0 +1,74 @@ +#!/bin/bash +set -e + +SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}") +. $SCRIPT_PATH/../common.sh +. $SCRIPT_PATH/release.cfg + +init +init_vault +get_kms_wrapper_version + +cd vault + +echo "Pushing synchronized main" +git push origin main + +echo "Updating yckms" +git checkout yckms +git rebase origin main +git push origin yckms + +if git checkout $YCKMS_VERSION; then + echo "Branch '$YCKMS_VERSION' already exists" +else + echo "Creating branch '$YCKMS_VERSION'" + git checkout -b $YCKMS_VERSION $BASE_VERSION +fi + +PATCH_LAST_COMMIT_MSG="YCKMS patch" + +while read -r line < <(git log $BASE_VERSION..$YCKMS_VERSION --oneline --reverse --pretty=format:"%B"); do + if [[ $line == "$PATCH_LAST_COMMIT_MSG" ]]; then + HAS_YCKMS_PATCH=true + break + fi +done + +if [[ "$HAS_YCKMS_PATCH" != true ]]; then + CUT_VERSION=$(echo $BASE_VERSION | cut -c 2-) + + echo "Applying patch from yckms branch" + git cherry-pick --no-commit $(git log main..yckms -1000 --oneline --reverse --pretty=format:"%h" | paste -sd' ' -) + # cherry-pick is more stable then merge-base + #git diff $(git merge-base --fork-point main yckms) yckms | git apply + sed -i '' 's/.*VersionMetadata.*=.*""/VersionMetadata = "yckms"/' version/version_base.go + go fmt version/version_base.go + git add version/version_base.go + sed -i '' "s/ARG BASE_VAULT_VERSION=.*/ARG BASE_VAULT_VERSION=$CUT_VERSION/" yandex/docker/Dockerfile + sed -i '' "s/BASE_VAULT_VERSION=.*/BASE_VAULT_VERSION=$CUT_VERSION/" yandex/compute/install.sh + git add yandex/docker/Dockerfile yandex/compute/install.sh + + echo "Adding github.com/yandex-cloud/vault-kms-wrapper/v2 dependency" + YCKMS_WRAPPER_VERSION="$KMS_WRAPPER_VERSION-yckms" + YCKMS_WRAPPER=github.com/yandex-cloud/vault-kms-wrapper/v2@"$YCKMS_WRAPPER_VERSION" + + if ! go list -m "$YCKMS_WRAPPER"; then + echo >&2 "Cannot find $YCKMS_WRAPPER, possible release required!" + exit 1 + fi + go mod edit -require="$YCKMS_WRAPPER" + go mod tidy + git add go.mod go.sum + + echo "Vendoring" + go mod vendor + git add vendor + + echo "Committing" + git commit -m "$PATCH_LAST_COMMIT_MSG" +else + echo "Patch is already applied" +fi + +git push -f origin $YCKMS_VERSION diff --git a/yandex/scripts/release/release_sample.cfg b/yandex/scripts/release/release_sample.cfg new file mode 100644 index 000000000000..9325c6bc7f64 --- /dev/null +++ b/yandex/scripts/release/release_sample.cfg @@ -0,0 +1,11 @@ +#Requirements +# - go +# - To have access to push to repository https://github.com/yandex-cloud/vault +# - for that it's needed to clone repository via ssh and to be invited + +#Version of original Vault build to patch +BASE_VERSION="v1.14.1" +#Directory inside which will be build patched version +WORK_DIR=./../tmp +#Name of a branch in which will be pushed patched version(if not set then $BASE_VERSION+yckms will be used) +PATCH_BRANCH=test \ No newline at end of file diff --git a/yandex/scripts/release/release_wrapper.sh b/yandex/scripts/release/release_wrapper.sh new file mode 100755 index 000000000000..37ef45f44ae8 --- /dev/null +++ b/yandex/scripts/release/release_wrapper.sh @@ -0,0 +1,67 @@ +#!/bin/bash +set -e + +SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}") +. $SCRIPT_PATH/common.sh +. $SCRIPT_PATH/release_sample.cfg + +init +init_vault +init_vault_kms_wrapper +get_kms_wrapper_version + +BRANCH=release/${KMS_WRAPPER_VERSION}+"yckms" +TAG=${KMS_WRAPPER_VERSION}-"yckms" + +cd vault-kms-wrapper + +if git ls-remote --exit-code origin $BRANCH; then + echo >&2 "Remote branch '$BRANCH' already exists!" + echo >&2 "Update suffix 'yckms' to release new version" + exit 1 +fi + +if git ls-remote --exit-code origin $TAG; then + echo >&2 "Remote tag '$TAG' already exists!" + echo >&2 "Update suffix 'yckms' to release new version" + exit 1 +fi + +echo "Getting github.com/hashicorp/go-kms-wrapping/v2@$KMS_WRAPPER_VERSION" +go get github.com/hashicorp/go-kms-wrapping/v2@$KMS_WRAPPER_VERSION +go mod tidy +echo "Testing" +go test +git add go.mod go.sum + +if ! git diff --cached --quiet --exit-code; then + echo "Committing" + git commit -m "Version updated $VERSION" +else + echo "Nothing to commit" +fi + +if git show-ref --quiet $BRANCH; then + git branch -D $BRANCH +fi + +echo "Creating branch $BRANCH" +git checkout -b $BRANCH + +go mod vendor +git add vendor + +if ! git diff --cached --quiet --exit-code; then + echo "Committing vendor" + git commit -m "Vendor" +else + echo "Nothing to commit" +fi + +git tag $TAG -f + +git push origin $BRANCH +git push origin $TAG + +git checkout main +git push origin main diff --git a/yandex/scripts/test_docker_container/script_inside_ssh.sh b/yandex/scripts/test_docker_container/script_inside_ssh.sh new file mode 100644 index 000000000000..2aa49c58cdfa --- /dev/null +++ b/yandex/scripts/test_docker_container/script_inside_ssh.sh @@ -0,0 +1,20 @@ +sudo apt-get update +sudo apt-get -y install docker.io + +curl -H Metadata-Flavor:Google 169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token | \ +cut -f1 -d',' | \ +cut -f2 -d':' | \ +tr -d '"' | \ +sudo docker login --username iam --password-stdin $DOMAIN + +curl -H Metadata-Flavor:Google 169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token | \ +cut -f1 -d',' | \ +cut -f2 -d':' | \ +tr -d '"' | \ +sudo docker login --username iam --password-stdin $DOCKER_ENDPOINT + +sudo docker pull $DOCKER_ENDPOINT/$REGISTRY_ID/vault:$BASE_VERSION-yckms + +sudo docker run -p8200:8200 --cap-add=IPC_LOCK -e "VAULT_LOCAL_CONFIG=$(< vault.hcl)" -e "VAULT_ADDR=http://127.0.0.1:8200" -v $(pwd)/startup.sh:/startup.sh -v $(pwd)/vault.hcl:/vault.hcl -v $(pwd)/key.json:/vault/config/key.json $DOMAIN/$REGISTRY_ID/vault:$BASE_VERSION-yckms /bin/sh "/startup.sh" + +exit \ No newline at end of file diff --git a/yandex/scripts/test_docker_container/test_docker_container.sh b/yandex/scripts/test_docker_container/test_docker_container.sh new file mode 100644 index 000000000000..b8c0d31e8f79 --- /dev/null +++ b/yandex/scripts/test_docker_container/test_docker_container.sh @@ -0,0 +1,114 @@ +#!/bin/bash +set -e + +SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}") + . $SCRIPT_PATH/../common.sh + . $SCRIPT_PATH/test_docker.cfg + +cleanup() { + yc --profile $PROFILE kms symmetric-key delete --folder-id $FOLDER_ID vault-test + yc --profile $PROFILE iam service-account delete --folder-id $FOLDER_ID --name "test-vault-sa" + yc --profile $PROFILE compute instance delete --folder-id $FOLDER_ID --name test-vault + yc --profile $PROFILE vpc subnet delete --folder-id $FOLDER_ID --name test-vault-subnet + yc --profile $PROFILE vpc network delete --folder-id $FOLDER_ID --name test-vault-network + exit +} + +trap "cleanup" EXIT; + +echo "Create service account" +yc --profile $PROFILE iam service-account create --folder-id $FOLDER_ID --name "test-vault-sa" + +echo "Create KMS key" +KMS_KEY=$(yc --profile $PROFILE kms symmetric-key create --folder-id $FOLDER_ID --name vault-test --default-algorithm aes-256 | yq '.id') + +echo "Create auth key for service account" +yc iam key create --folder-id $FOLDER_ID --service-account-name test-vault-sa --output ./vol/key.json + +echo "Give roles for service account" +yc --profile $PROFILE resource-manager folder add-access-binding $FOLDER_ID \ + --role kms.keys.encrypterDecrypter \ + --service-account-name test-vault-sa \ + --folder-id $FOLDER_ID + +yc --profile $PROFILE resource-manager folder add-access-binding $FOLDER_ID \ + --role container-registry.images.puller \ + --service-account-name test-vault-sa \ + --folder-id $FOLDER_ID + +echo "Create network" +ycp --profile $PROFILE vpc network create --name test-vault-network --folder-id $FOLDER_ID + +echo "Create subnet" +yc --profile $PROFILE vpc subnet create \ + --name test-vault-subnet \ + --description "test vault subnet" \ + --network-name test-vault-network \ + --zone il1-a \ + --folder-id $FOLDER_ID \ + --range 192.168.0.0/24 + +echo "Create instance" +yc --profile $PROFILE compute instance create test-vault \ + --zone il1-a \ + --platform standard-v3 \ + --create-boot-disk image-folder-id=standard-images,image-family=ubuntu-2004-lts \ + --cores 2 \ + --core-fraction 100 \ + --memory 2 \ + --ssh-key $PUBLIC_SSH \ + --folder-id $FOLDER_ID \ + --service-account-name test-vault-sa \ + --public-ip + +echo "Get machine's ip" +MACHINE_IP=$(yc --profile $PROFILE compute instance get --name test-vault --folder-id $FOLDER_ID | yq '.network_interfaces[0].primary_v4_address.one_to_one_nat.address') + +echo "Wait until status is RUNNING" +#status='' +#while [[ $status != "RUNNING" ]] +#do +#status=$(yc compute instance get --name test-vault --folder-id $FOLDER_ID | yq '.status') +#sleep 3 +#done + +sleep 20 + +DOMAIN="cr.yandex" + +if [[ "$PROFILE" == "israel" ]]; then + DOMAIN="cr.cloudil.com" +fi + +if [[ "$PROFILE" == "preprod" ]]; then + DOMAIN="cr.cloud-preprod.yandex.net" +fi + +cat >./vol/vault.hcl <&2 "Invalid key '$KEY' value! Expected $VAL, but was $ACTUAL_VAL" + exit 1 +fi +echo "Local test passed successfully" \ No newline at end of file diff --git a/yandex/scripts/test_local/test_local.sh b/yandex/scripts/test_local/test_local.sh new file mode 100755 index 000000000000..fc95fc38cfb4 --- /dev/null +++ b/yandex/scripts/test_local/test_local.sh @@ -0,0 +1,100 @@ +#!/bin/bash +set -e + +SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}") +. $SCRIPT_PATH/../common.sh +. $SCRIPT_PATH/test_local.cfg + +PID=0 +cleanup() { + yc --profile $PROFILE kms symmetric-key delete --folder-id $FOLDER_ID vault-test-local + yc --profile $PROFILE iam service-account delete --folder-id $FOLDER_ID --name "test-local-vault-sa" + kill $PID + exit +} + +trap "cleanup" EXIT; + +init +init_vault +cd vault + +echo "Testing $YCKMS_VERSION branch" +git checkout $YCKMS_VERSION + +echo "Create service account" +yc --profile $PROFILE iam service-account create --folder-id $FOLDER_ID --name "test-local-vault-sa" + +echo "Create KMS key" +KMS_KEY=$(yc --profile $PROFILE kms symmetric-key create --folder-id $FOLDER_ID --name vault-test-local --default-algorithm aes-256 | yq '.id') + +echo "Create auth key for service account" +yc --profile $PROFILE iam key create --folder-id $FOLDER_ID --service-account-name test-local-vault-sa --output ./auth_key.json + +echo "Give roles for service account" +yc --profile $PROFILE resource-manager folder add-access-binding $FOLDER_ID \ + --role kms.keys.encrypterDecrypter \ + --service-account-name test-local-vault-sa \ + --folder-id $FOLDER_ID + +echo "Building vault" +make bootstrap +make dev + +echo $KMS_ENDPOINT + +cat >vault.hcl <&2 "Invalid key '$KEY' value! Expected $VAL, but was $ACTUAL_VAL" + exit 1 +fi +echo "Local test passed successfully" diff --git a/yandex/scripts/test_local/test_local_sample.cfg b/yandex/scripts/test_local/test_local_sample.cfg new file mode 100644 index 000000000000..a66cb0eef781 --- /dev/null +++ b/yandex/scripts/test_local/test_local_sample.cfg @@ -0,0 +1,16 @@ +#Requirements +# - go +# - To have access to push to repository https://github.com/yandex-cloud/vault +# - for that it's needed to clone repository via ssh and to be invited + +#Version of original Vault build to patch +BASE_VERSION="v1.14.1" +#Directory inside which will be build patched version +WORK_DIR=./../tmp +#Folder in which kms key and service account will be created +FOLDER_ID=b48ans97du1s547k9e9m +PROFILE=israel +#Endpoint of kms service used for unseal process +KMS_ENDPOINT=api.il.nebius.cloud:443 +#Name of a branch in which will be pushed patched version(if not set then $BASE_VERSION+yckms will be used) +PATCH_BRANCH=test