From 66407c274aced5951a65f93bf05c6b48a6622962 Mon Sep 17 00:00:00 2001 From: Diwakar Sharma Date: Tue, 23 Jul 2024 07:56:03 +0000 Subject: [PATCH] ci: enhance pull_request workflow Signed-off-by: Diwakar Sharma --- .github/workflows/chart-lint-test.yml | 23 +- .github/workflows/pull_request.yml | 61 +++--- .github/workflows/release-charts.yml | 12 +- .gitignore | 1 + Makefile | 5 +- ci/ci-test.sh | 42 ++-- ct.yaml | 1 + deploy/helm/charts/Chart.yaml | 6 +- deploy/helm/charts/README.md | 4 +- deploy/helm/charts/charts/crds/Chart.yaml | 2 +- deploy/helm/charts/templates/_helpers.tpl | 2 +- .../helm/charts/templates/lvm-controller.yaml | 2 +- deploy/helm/charts/templates/lvm-node.yaml | 2 +- deploy/helm/charts/values.yaml | 4 +- deploy/lvm-operator.yaml | 10 +- deploy/yamls/lvm-driver.yaml | 4 +- deploy/yamls/namespace.yaml | 4 +- docs/faq.md | 2 +- docs/storageclasses.md | 2 +- legacy-upgrade/README.md | 4 +- nix/sources.json | 14 ++ nix/sources.nix | 198 ++++++++++++++++++ pkg/lvm/volume.go | 4 +- scripts/validate-chart-version.sh | 101 +++++++++ shell.nix | 12 ++ tests/suite_test.go | 10 +- tests/utils.go | 14 +- 27 files changed, 433 insertions(+), 113 deletions(-) create mode 100644 nix/sources.json create mode 100644 nix/sources.nix create mode 100755 scripts/validate-chart-version.sh create mode 100644 shell.nix diff --git a/.github/workflows/chart-lint-test.yml b/.github/workflows/chart-lint-test.yml index 645d8c29..fa686075 100644 --- a/.github/workflows/chart-lint-test.yml +++ b/.github/workflows/chart-lint-test.yml @@ -1,16 +1,17 @@ name: Chart Lint and Test -on: - push: - paths: - - 'deploy/helm/**' - branches: - - develop - pull_request: - paths: - - 'deploy/helm/**' - branches: - - develop +#on: +# push: +# paths: +# - 'deploy/helm/**' +# branches: +# - develop +# pull_request: +# paths: +# - 'deploy/helm/**' +# branches: +# - 'develop' +# - 'release/**' jobs: lint-test: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 586160bd..08d6382f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1,29 +1,14 @@ -# Copyright 2020 The OpenEBS Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: ci +name: Pull Request CI on: pull_request: branches: - 'develop' - - 'v*' + - 'release/**' paths-ignore: - '*.md' - 'BUILDMETA' - 'changelogs/**' - - 'deploy/helm/**' - 'docs/**' - 'design/**' - 'LICENSE' @@ -35,6 +20,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Go 1.19 uses: actions/setup-go@v5 @@ -62,7 +49,21 @@ jobs: path: '.' pattern: '*.sh' - unit-test: + - uses: cachix/install-nix-action@v22 + - uses: rrbutani/use-nix-shell-action@v1.1.0 + with: + file: shell.nix + + - name: Check if the chart follows a valid semver version + run: | + branch_name="${{ github.event.pull_request.base.ref }}" + ./scripts/validate-chart-version.sh --branch $branch_name + + - name: Run chart-testing lint + run: | + ct lint --config ct.yaml + + unit-tests: runs-on: ubuntu-latest steps: - name: Checkout @@ -85,8 +86,8 @@ jobs: name: coverage-$(date +%s) flags: unittests - bdd-test: - needs: ['unit-test'] + bdd-tests: + needs: ['unit-tests'] runs-on: ubuntu-latest strategy: fail-fast: true @@ -95,6 +96,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + + - uses: cachix/install-nix-action@v22 + - uses: rrbutani/use-nix-shell-action@v1.1.0 + with: + file: shell.nix - name: Set up Go 1.19 uses: actions/setup-go@v5 @@ -102,6 +108,9 @@ jobs: go-version: 1.19.9 cache: false + - name: Build images locally + run: make lvm-driver-image || exit 1; + - name: Setup Minikube-Kubernetes uses: medyagh/setup-minikube@latest with: @@ -112,8 +121,10 @@ jobs: cni: calico start-args: '--install-addons=false' - - name: Build images locally - run: make lvm-driver-image || exit 1; + - name: Setting environment variables + run: | + echo "KUBECONFIG=$HOME/.kube/config" >> $GITHUB_ENV + echo "OPENEBS_NAMESPACE=openebs" >> $GITHUB_ENV - name: bootstrap run: make bootstrap @@ -131,7 +142,7 @@ jobs: csi-driver: runs-on: ubuntu-latest - needs: ['lint', 'unit-test', 'bdd-test'] + needs: ['lint', 'unit-tests', 'bdd-tests'] steps: - name: Checkout uses: actions/checkout@v4 @@ -144,8 +155,6 @@ jobs: - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 - with: - version: v0.13.1 - name: Build uses: docker/build-push-action@v5 @@ -154,5 +163,3 @@ jobs: file: ./buildscripts/lvm-driver/Dockerfile.buildx push: false platforms: linux/amd64, linux/arm64 - tags: | - openebs/lvm-driver:ci diff --git a/.github/workflows/release-charts.yml b/.github/workflows/release-charts.yml index 95c9ba40..4638484b 100644 --- a/.github/workflows/release-charts.yml +++ b/.github/workflows/release-charts.yml @@ -1,11 +1,11 @@ name: Release Charts -on: - push: - paths: - - 'deploy/helm/**' - branches: - - develop +#on: +# push: +# paths: +# - 'deploy/helm/**' +# branches: +# - develop jobs: release: diff --git a/.gitignore b/.gitignore index af1799ba..e93a89e1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ tags *.swo *.swn *.idea +*.test diff --git a/Makefile b/Makefile index 0bb505dd..07e82ffc 100644 --- a/Makefile +++ b/Makefile @@ -52,9 +52,12 @@ ifeq (${DBUILD_SITE_URL}, ) export DBUILD_SITE_URL endif +# Set the path to the Chart.yaml file +ROOT_DIR:=$(dir $(realpath $(firstword $(MAKEFILE_LIST)))) +CHART_YAML:=${ROOT_DIR}/deploy/helm/charts/Chart.yaml ifeq (${IMAGE_TAG}, ) - IMAGE_TAG = ci + IMAGE_TAG := $(shell awk -F': ' '/^version:/ {print $$2}' $(CHART_YAML)) export IMAGE_TAG endif diff --git a/ci/ci-test.sh b/ci/ci-test.sh index 008f46bc..5b80a065 100755 --- a/ci/ci-test.sh +++ b/ci/ci-test.sh @@ -1,27 +1,10 @@ #!/usr/bin/env bash -# Copyright 2021 The OpenEBS Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - set -e -LVM_OPERATOR="$(realpath deploy/lvm-operator.yaml)" SNAP_CLASS="$(realpath deploy/sample/lvmsnapclass.yaml)" -export LVM_NAMESPACE="openebs" export TEST_DIR="tests" -export NAMESPACE="kube-system" # allow override if [ -z "${KUBECONFIG}" ] @@ -46,7 +29,7 @@ cleanup_foreign_lvmvg() { if [ -f /tmp/openebs_ci_foreign_disk.img ] then sudo vgremove foreign_lvmvg --config="${FOREIGN_LVM_CONFIG}" -y || true - rm /tmp/openebs_ci_foreign_disk.img + sudo rm /tmp/openebs_ci_foreign_disk.img fi cleanup_loopdev } @@ -60,8 +43,8 @@ cleanup() { kubectl delete pvc -n openebs lvmpv-pvc kubectl delete -f "${SNAP_CLASS}" - kubectl delete -f "${LVM_OPERATOR}" + helm uninstall lvm-localpv -n "$OPENEBS_NAMESPACE" || true # always return true return 0 } @@ -87,29 +70,30 @@ sudo sed -i '/^[^#]*thin_pool_autoextend_percent/ s/= .*/= 20/' /etc/lvm/lvm.con # Prepare env for running BDD tests # Minikube is already running -kubectl apply -f "${LVM_OPERATOR}" + +helm install lvm-localpv ./deploy/helm/charts -n "$OPENEBS_NAMESPACE" --create-namespace --set lvmPlugin.pullPolicy=Never --set analytics.enabled=false kubectl apply -f "${SNAP_CLASS}" dumpAgentLogs() { NR=$1 - AgentPOD=$(kubectl get pods -l app=openebs-lvm-node -o jsonpath='{.items[0].metadata.name}' -n "$NAMESPACE") - kubectl describe po "$AgentPOD" -n "$NAMESPACE" + AgentPOD=$(kubectl get pods -l app=openebs-lvm-node -o jsonpath='{.items[0].metadata.name}' -n "$OPENEBS_NAMESPACE") + kubectl describe po "$AgentPOD" -n "$OPENEBS_NAMESPACE" printf "\n\n" - kubectl logs --tail="${NR}" "$AgentPOD" -n "$NAMESPACE" -c openebs-lvm-plugin + kubectl logs --tail="${NR}" "$AgentPOD" -n "$OPENEBS_NAMESPACE" -c openebs-lvm-plugin printf "\n\n" } dumpControllerLogs() { NR=$1 - ControllerPOD=$(kubectl get pods -l app=openebs-lvm-controller -o jsonpath='{.items[0].metadata.name}' -n "$NAMESPACE") - kubectl describe po "$ControllerPOD" -n "$NAMESPACE" + ControllerPOD=$(kubectl get pods -l app=openebs-lvm-controller -o jsonpath='{.items[0].metadata.name}' -n "$OPENEBS_NAMESPACE") + kubectl describe po "$ControllerPOD" -n "$OPENEBS_NAMESPACE" printf "\n\n" - kubectl logs --tail="${NR}" "$ControllerPOD" -n "$NAMESPACE" -c openebs-lvm-plugin + kubectl logs --tail="${NR}" "$ControllerPOD" -n "$OPENEBS_NAMESPACE" -c openebs-lvm-plugin printf "\n\n" } isPodReady(){ - [ "$(kubectl get po "$1" -o 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' -n "$NAMESPACE")" = 'True' ] + [ "$(kubectl get po "$1" -o 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' -n "$OPENEBS_NAMESPACE")" = 'True' ] } isDriverReady(){ @@ -124,7 +108,7 @@ waitForLVMDriver() { i=0 while [ "$i" -le "$period" ]; do - lvmDriver="$(kubectl get pods -l role=openebs-lvm -o 'jsonpath={.items[*].metadata.name}' -n "$NAMESPACE")" + lvmDriver="$(kubectl get pods -l role=openebs-lvm -o 'jsonpath={.items[*].metadata.name}' -n "$OPENEBS_NAMESPACE")" if isDriverReady "$lvmDriver"; then return 0 fi @@ -143,7 +127,7 @@ waitForLVMDriver cd $TEST_DIR -kubectl get po -n "$NAMESPACE" +kubectl get po -n "$OPENEBS_NAMESPACE" set +e diff --git a/ct.yaml b/ct.yaml index 7959faf3..a646494f 100644 --- a/ct.yaml +++ b/ct.yaml @@ -5,3 +5,4 @@ chart-dirs: - deploy/helm helm-extra-args: --timeout=500s validate-maintainers: false +check-version-increment: false diff --git a/deploy/helm/charts/Chart.yaml b/deploy/helm/charts/Chart.yaml index 3cc8bcb3..ee72696a 100644 --- a/deploy/helm/charts/Chart.yaml +++ b/deploy/helm/charts/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: lvm-localpv description: CSI Driver for dynamic provisioning of LVM Persistent Local Volumes. -version: 1.6.0 -appVersion: 1.6.0 +version: 1.7.0-develop +appVersion: 1.7.0-develop icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/openebs/icon/color/openebs-icon-color.png home: https://openebs.io/ keywords: @@ -16,5 +16,5 @@ sources: - https://github.com/openebs/lvm-localpv dependencies: - name: crds - version: 1.6.0 + version: 1.7.0-develop condition: crds.enabled diff --git a/deploy/helm/charts/README.md b/deploy/helm/charts/README.md index e9385ec4..8b5c9b09 100644 --- a/deploy/helm/charts/README.md +++ b/deploy/helm/charts/README.md @@ -38,7 +38,7 @@ $ helm install [RELEASE_NAME] openebs-lvmlocalpv/lvm-localpv --namespace [NAMESP **Note:** If moving from the operator to helm -- Make sure the namespace provided in the helm install command is same as `LVM_NAMESPACE` (by default it is `openebs`) env in the controller deployment. +- Make sure the namespace provided in the helm install command is same as `OPENEBS_NAMESPACE` (by default it is `openebs`) env in the controller deployment. - Before installing, clean up the stale deployment and daemonset from `kube-system` namespace using the below commands ```sh kubectl delete deployment openebs-lvm-controller -n kube-system @@ -92,7 +92,7 @@ helm install openebs-lvmlocalpv openebs-lvmlocalpv/lvm-localpv --namespace opene | `lvmPlugin.image.registry` | Registry for openebs-lvm-plugin image | `""` | | `lvmPlugin.image.repository` | Image repository for openebs-lvm-plugin | `openebs/lvm-driver` | | `lvmPlugin.image.pullPolicy` | Image pull policy for openebs-lvm-plugin | `IfNotPresent` | -| `lvmPlugin.image.tag` | Image tag for openebs-lvm-plugin | `1.6.0` | +| `lvmPlugin.image.tag` | Image tag for openebs-lvm-plugin | `1.7.0-develop` | | `lvmPlugin.metricsPort` | The TCP port number used for exposing lvm-metrics | `9500` | | `lvmPlugin.allowedTopologies` | The comma seperated list of allowed node topologies | `kubernetes.io/hostname,` | | `lvmNode.driverRegistrar.image.registry` | Registry for csi-node-driver-registrar image | `registry.k8s.io/` | diff --git a/deploy/helm/charts/charts/crds/Chart.yaml b/deploy/helm/charts/charts/crds/Chart.yaml index bf745b7d..c5d90db1 100644 --- a/deploy/helm/charts/charts/crds/Chart.yaml +++ b/deploy/helm/charts/charts/crds/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v2 name: crds -version: 1.6.0 +version: 1.7.0-develop description: A Helm chart that collects CustomResourceDefinitions (CRDs) from lvm-localpv. diff --git a/deploy/helm/charts/templates/_helpers.tpl b/deploy/helm/charts/templates/_helpers.tpl index b43ec257..34580e9e 100644 --- a/deploy/helm/charts/templates/_helpers.tpl +++ b/deploy/helm/charts/templates/_helpers.tpl @@ -60,7 +60,7 @@ Define meta labels for openebs lvm-localpv components {{- define "lvmlocalpv.common.metaLabels" -}} chart: {{ template "lvmlocalpv.chart" . }} heritage: {{ .Release.Service }} -openebs.io/version: {{ .Values.release.version | quote }} +openebs.io/version: {{ .Chart.AppVersion | quote }} role: {{ .Values.role | quote }} {{- end -}} diff --git a/deploy/helm/charts/templates/lvm-controller.yaml b/deploy/helm/charts/templates/lvm-controller.yaml index 5ba84ed9..5ff5193b 100644 --- a/deploy/helm/charts/templates/lvm-controller.yaml +++ b/deploy/helm/charts/templates/lvm-controller.yaml @@ -109,7 +109,7 @@ spec: value: controller - name: OPENEBS_CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: LVM_NAMESPACE + - name: OPENEBS_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace diff --git a/deploy/helm/charts/templates/lvm-node.yaml b/deploy/helm/charts/templates/lvm-node.yaml index a72c83e8..209f5a00 100644 --- a/deploy/helm/charts/templates/lvm-node.yaml +++ b/deploy/helm/charts/templates/lvm-node.yaml @@ -91,7 +91,7 @@ spec: value: unix:///plugin/csi.sock - name: OPENEBS_NODE_DRIVER value: agent - - name: LVM_NAMESPACE + - name: OPENEBS_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace diff --git a/deploy/helm/charts/values.yaml b/deploy/helm/charts/values.yaml index c40f7405..aa4453b4 100644 --- a/deploy/helm/charts/values.yaml +++ b/deploy/helm/charts/values.yaml @@ -1,8 +1,6 @@ # Default values for openebs-lvmlocalpv. # This is a YAML-formatted file. # Declare variables to be passed into your templates. -release: - version: "1.6.0" imagePullSecrets: # - name: "image-pull-secret" @@ -151,7 +149,7 @@ lvmPlugin: repository: openebs/lvm-driver pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: 1.6.0 + tag: 1.7.0-develop ioLimits: enabled: false containerRuntime: containerd diff --git a/deploy/lvm-operator.yaml b/deploy/lvm-operator.yaml index 98743970..a873aaa5 100644 --- a/deploy/lvm-operator.yaml +++ b/deploy/lvm-operator.yaml @@ -8,12 +8,12 @@ # Create the OpenEBS namespace # This is the default namespace where the LVM driver will create all # its resources. If we want to change it to use a different namespace -# modify this to create the new namespace and also modify the LVM_NAMESPACE +# modify this to create the new namespace and also modify the OPENEBS_NAMESPACE # env for the LVM Driver's controller and agent deployments. # please note that this should be changed while initial setup, once LVM Driver # is deployed with a namespace, we should never modify it as old resources will # not be available under the new namespace and LVM Driver looks for all the resources -# in the LVM_NAMESPACE namespace passed as an env. +# in the OPENEBS_NAMESPACE namespace passed as an env. apiVersion: v1 kind: Namespace @@ -1476,7 +1476,7 @@ spec: value: controller - name: OPENEBS_CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: LVM_NAMESPACE + - name: OPENEBS_NAMESPACE value: openebs - name: OPENEBS_IO_INSTALLER_TYPE value: "lvm-operator" @@ -1627,7 +1627,7 @@ spec: labels: app: openebs-lvm-node role: openebs-lvm - openebs.io/component-name: openebs-lvm-node + openebs.io/component-name: openebs-lvm-localpv-node openebs.io/version: ci spec: priorityClassName: openebs-lvm-localpv-csi-node-critical @@ -1680,7 +1680,7 @@ spec: value: unix:///plugin/csi.sock - name: OPENEBS_NODE_DRIVER value: agent - - name: LVM_NAMESPACE + - name: OPENEBS_NAMESPACE value: openebs - name: METRICS_LISTEN_ADDRESS value: :9500 diff --git a/deploy/yamls/lvm-driver.yaml b/deploy/yamls/lvm-driver.yaml index 95979a42..3876bfdc 100644 --- a/deploy/yamls/lvm-driver.yaml +++ b/deploy/yamls/lvm-driver.yaml @@ -1040,7 +1040,7 @@ spec: value: controller - name: OPENEBS_CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: LVM_NAMESPACE + - name: OPENEBS_NAMESPACE value: openebs - name: OPENEBS_IO_INSTALLER_TYPE value: "lvm-operator" @@ -1244,7 +1244,7 @@ spec: value: unix:///plugin/csi.sock - name: OPENEBS_NODE_DRIVER value: agent - - name: LVM_NAMESPACE + - name: OPENEBS_NAMESPACE value: openebs - name: METRICS_LISTEN_ADDRESS value: :9500 diff --git a/deploy/yamls/namespace.yaml b/deploy/yamls/namespace.yaml index 920f0bbc..da9c4b18 100644 --- a/deploy/yamls/namespace.yaml +++ b/deploy/yamls/namespace.yaml @@ -1,12 +1,12 @@ # Create the OpenEBS namespace # This is the default namespace where the LVM driver will create all # its resources. If we want to change it to use a different namespace -# modify this to create the new namespace and also modify the LVM_NAMESPACE +# modify this to create the new namespace and also modify the OPENEBS_NAMESPACE # env for the LVM Driver's controller and agent deployments. # please note that this should be changed while initial setup, once LVM Driver # is deployed with a namespace, we should never modify it as old resources will # not be available under the new namespace and LVM Driver looks for all the resources -# in the LVM_NAMESPACE namespace passed as an env. +# in the OPENEBS_NAMESPACE namespace passed as an env. apiVersion: v1 kind: Namespace diff --git a/docs/faq.md b/docs/faq.md index d4cc4d74..3cbeec7b 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -27,7 +27,7 @@ env: value: unix:///plugin/csi.sock - name: OPENEBS_NODE_DRIVER value: agent - - name: LVM_NAMESPACE + - name: OPENEBS_NAMESPACE value: openebs - name: ALLOWED_TOPOLOGIES value: "openebs.io/rack" diff --git a/docs/storageclasses.md b/docs/storageclasses.md index 12c56512..f640c3de 100644 --- a/docs/storageclasses.md +++ b/docs/storageclasses.md @@ -333,7 +333,7 @@ env: value: unix:///plugin/csi.sock - name: OPENEBS_NODE_DRIVER value: agent - - name: LVM_NAMESPACE + - name: OPENEBS_NAMESPACE value: openebs - name: ALLOWED_TOPOLOGIES value: "test1,test2" diff --git a/legacy-upgrade/README.md b/legacy-upgrade/README.md index 99575e7d..1385bcb1 100644 --- a/legacy-upgrade/README.md +++ b/legacy-upgrade/README.md @@ -23,7 +23,7 @@ We can upgrade the lvm driver to the latest stable release version by apply the $ kubectl apply -f https://openebs.github.io/charts/lvm-operator.yaml ``` -Please note that if you were using the LVM_NAMESPACE env value other than `openebs` (default value) in which lvm-localpv CR's are created, don't forget to update that value in lvm-operator yaml file under LVM_NAMESPACE env. +Please note that if you were using the OPENEBS_NAMESPACE env value other than `openebs` (default value) in which lvm-localpv CR's are created, don't forget to update that value in lvm-operator yaml file under OPENEBS_NAMESPACE env. For upgrading the driver to any particular release, download the lvm-operator yaml from the desired branch and update the lvm-driver image tag to the corresponding tag. For e.g, to upgrade the lvm-driver to 0.7.0 version, follow these steps: @@ -44,7 +44,7 @@ one at `openebs-lvm-plugin` container image in lvm-controller deployment value: controller - name: OPENEBS_CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: LVM_NAMESPACE + - name: OPENEBS_NAMESPACE value: openebs ``` and other one at `openebs-lvm-plugin` container in lvm-node daemonset. diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 00000000..9faedce3 --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,14 @@ +{ + "nixpkgs": { + "branch": "release-23.05", + "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to", + "homepage": "https://github.com/NixOS/nixpkgs", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9a333eaa80901efe01df07eade2c16d183761fa3", + "sha256": "0xhqjli4m9wkzv7xhs6fr1iajdjbv7xnj0bwvwldq9s6arlwkhj3", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/9a333eaa80901efe01df07eade2c16d183761fa3.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 00000000..fe3dadf7 --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,198 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; name = name'; } + else + pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; + + fetch_tarball = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchTarball { name = name'; inherit (spec) url sha256; } + else + pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; + + fetch_git = name: spec: + let + ref = + spec.ref or ( + if spec ? branch then "refs/heads/${spec.branch}" else + if spec ? tag then "refs/tags/${spec.tag}" else + abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!" + ); + submodules = spec.submodules or false; + submoduleArg = + let + nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0; + emptyArgWithWarning = + if submodules + then + builtins.trace + ( + "The niv input \"${name}\" uses submodules " + + "but your nix's (${builtins.nixVersion}) builtins.fetchGit " + + "does not support them" + ) + { } + else { }; + in + if nixSupportsSubmodules + then { inherit submodules; } + else emptyArgWithWarning; + in + builtins.fetchGit + ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg); + + fetch_local = spec: spec.path; + + fetch_builtin-tarball = name: throw + ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; + + fetch_builtin-url = name: throw + ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; + + # + # Various helpers + # + + # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 + sanitizeName = name: + ( + concatMapStrings (s: if builtins.isList s then "-" else s) + ( + builtins.split "[^[:alnum:]+._?=-]+" + ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) + ) + ); + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: system: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import { } + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs name spec + else if spec.type == "tarball" then fetch_tarball pkgs name spec + else if spec.type == "git" then fetch_git name spec + else if spec.type == "local" then fetch_local spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball name + else if spec.type == "builtin-url" then fetch_builtin-url name + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # If the environment variable NIV_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + replace = name: drv: + let + saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name; + ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; + in + if ersatz == "" then drv else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 + optionalAttrs = cond: as: if cond then as else { }; + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs + ( + name: spec: + if builtins.hasAttr "outPath" spec + then + abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); } + ) + config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null + , sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile) + , system ? builtins.currentSystem + , pkgs ? mkPkgs sources system + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; + +in +mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/pkg/lvm/volume.go b/pkg/lvm/volume.go index c44ae0c5..71cb32e2 100644 --- a/pkg/lvm/volume.go +++ b/pkg/lvm/volume.go @@ -35,7 +35,7 @@ const ( // LvmNamespaceKey is the environment variable to get openebs namespace // // This environment variable is set via kubernetes downward API - LvmNamespaceKey string = "LVM_NAMESPACE" + LvmNamespaceKey string = "OPENEBS_NAMESPACE" // GoogleAnalyticsKey This environment variable is set via env GoogleAnalyticsKey string = "OPENEBS_IO_ENABLE_ANALYTICS" // LVMFinalizer for the LVMVolume CR @@ -75,7 +75,7 @@ func init() { LvmNamespace = os.Getenv(LvmNamespaceKey) if LvmNamespace == "" && os.Getenv("OPENEBS_NODE_DRIVER") != "" { - klog.Fatalf("LVM_NAMESPACE environment variable not set") + klog.Fatalf("OPENEBS_NAMESPACE environment variable not set") } NodeID = os.Getenv("OPENEBS_NODE_ID") if NodeID == "" && os.Getenv("OPENEBS_NODE_DRIVER") != "" { diff --git a/scripts/validate-chart-version.sh b/scripts/validate-chart-version.sh new file mode 100755 index 00000000..2b02c6ed --- /dev/null +++ b/scripts/validate-chart-version.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +# Write output to error output stream. +echo_stderr() { + echo -e "${1}" >&2 +} + +die() +{ + local _return="${2:-1}" + echo_stderr "$1" + exit "${_return}" +} + +set -euo pipefail + +# Set the path to the Chart.yaml file +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" +ROOT_DIR="$SCRIPT_DIR/.." +CHART_DIR="$ROOT_DIR/deploy/helm/charts" +CHART_YAML="$CHART_DIR/Chart.yaml" + +# Check if the Chart.yaml file exists +if [ ! -f "$CHART_YAML" ]; then + die "Chart.yaml file not found in $CHART_YAML" +fi + +# Extract the chart version and app version using yq +CHART_VERSION=$(yq e '.version' "$CHART_YAML") +APP_VERSION=$(yq e '.appVersion' "$CHART_YAML") + +# Check if extraction was successful +if [ -z "$CHART_VERSION" ] || [ -z "$APP_VERSION" ]; then + die "Failed to extract versions from Chart.yaml" +fi + +# Print the extracted versions +echo "Chart Version: $CHART_VERSION" +echo "App Version: $APP_VERSION" + +# Validate that the versions are valid semver +if [ "$(semver validate "$CHART_VERSION")" != "valid" ]; then + die "Invalid chart version: $CHART_VERSION" +fi + +if [ "$(semver validate "$APP_VERSION")" != "valid" ]; then + die "Invalid app version: $APP_VERSION" +fi + +help() { + cat < Name of the branch on which this workflow is running. +Examples: + $(basename "$0") --branch develop +EOF +} + +# Parse arguments +while [ "$#" -gt 0 ]; do + case $1 in + -b|--branch) + BRANCH_NAME=$2 + shift + ;; + -h|--help) + help + exit 0 + ;; + *) + help + die "Unknown option: $1" + ;; + esac + shift +done + +# Extract major and minor version from the branch name +extract_major_minor() { + echo "$1" | awk -F/ '{print $2}' +} + +if [[ "$BRANCH_NAME" == "develop" ]]; then + if [[ "$CHART_VERSION" != *"-develop" ]]; then + die "Chart version must include '-develop' for develop branch" + fi + if [[ "$APP_VERSION" != *"-develop" ]]; then + die "App version must include '-develop' for develop branch" + fi +elif [[ "$BRANCH_NAME" =~ ^(release/[0-9]+\.[0-9]+)$ ]]; then + RELEASE_VERSION=$(extract_major_minor "$BRANCH_NAME") + if [[ "$CHART_VERSION" != "$RELEASE_VERSION."*"-prerelease" ]]; then + die "Chart version must be in format $RELEASE_VERSION.X-prerelease for release branch" + fi + if [[ "$APP_VERSION" != "$RELEASE_VERSION."*"-prerelease" ]]; then + die "App version must be in format $RELEASE_VERSION.X-prerelease for release branch" + fi +else + die "Unknown branch name: $BRANCH_NAME" +fi \ No newline at end of file diff --git a/shell.nix b/shell.nix new file mode 100644 index 00000000..54f411c0 --- /dev/null +++ b/shell.nix @@ -0,0 +1,12 @@ +let + sources = import ./nix/sources.nix; + pkgs = import sources.nixpkgs {}; +in +pkgs.mkShell { + name = "scripts-shell"; + buildInputs = with pkgs; [ + semver-tool + yq-go + chart-testing + ]; +} \ No newline at end of file diff --git a/tests/suite_test.go b/tests/suite_test.go index 23d6e145..628d4cbb 100644 --- a/tests/suite_test.go +++ b/tests/suite_test.go @@ -65,9 +65,9 @@ var ( // default one fio app for most tests. appNames = []string{"fio-ci-0"} - nodeDaemonSet = "openebs-lvm-node" - controllerDeployment = "openebs-lvm-controller" - vgPattern = "lvmvg1.*" + nodeDaemonSet = "lvm-localpv-node" + controllerDeployment = "lvm-localpv-controller" + vgPattern = "lvmvg1.*" nsObj *corev1.Namespace scObj *storagev1.StorageClass @@ -84,9 +84,9 @@ var ( func init() { KubeConfigPath = os.Getenv("KUBECONFIG") - OpenEBSNamespace = os.Getenv("LVM_NAMESPACE") + OpenEBSNamespace = os.Getenv("OPENEBS_NAMESPACE") if OpenEBSNamespace == "" { - klog.Fatalf("LVM_NAMESPACE environment variable not set") + klog.Fatalf("OPENEBS_NAMESPACE environment variable not set") } var err error diff --git a/tests/utils.go b/tests/utils.go index dc16f02d..9184aa2a 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -744,7 +744,7 @@ func WaitForLVMVolumeReady() { func scaleControllerPlugin(num int32) int32 { ginkgo.By(fmt.Sprintf("scaling controller plugin deployment %v to size %v", controllerDeployment, num)) - scale, err := K8sClient.AppsV1().Deployments(metav1.NamespaceSystem). + scale, err := K8sClient.AppsV1().Deployments(OpenEBSNamespace). GetScale(context.Background(), controllerDeployment, metav1.GetOptions{}) gomega.Expect(err).To( gomega.BeNil(), @@ -755,14 +755,14 @@ func scaleControllerPlugin(num int32) int32 { return existingReplicas } scale.Spec.Replicas = num - scale, err = K8sClient.AppsV1().Deployments(metav1.NamespaceSystem). + scale, err = K8sClient.AppsV1().Deployments(OpenEBSNamespace). UpdateScale(context.Background(), controllerDeployment, scale, metav1.UpdateOptions{}) gomega.Expect(err).To( gomega.BeNil(), "update replicas of deployment %v to %v", controllerDeployment, num) scaled := gomega.Eventually(func() bool { - scale, err = K8sClient.AppsV1().Deployments(metav1.NamespaceSystem). + scale, err = K8sClient.AppsV1().Deployments(OpenEBSNamespace). GetScale(context.Background(), controllerDeployment, metav1.GetOptions{}) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) return scale.Spec.Replicas == num @@ -785,14 +785,14 @@ func deleteNodeDaemonSet() *appsv1.DaemonSet { ginkgo.By("deleting node plugin daemonset " + nodeDaemonSet) ds, err := K8sClient.AppsV1(). - DaemonSets(metav1.NamespaceSystem). + DaemonSets(OpenEBSNamespace). Get(context.Background(), nodeDaemonSet, metav1.GetOptions{}) gomega.Expect(err).To( gomega.BeNil(), "fetching node plugin daemonset %v", nodeDaemonSet) policy := metav1.DeletePropagationForeground err = K8sClient.AppsV1(). - DaemonSets(metav1.NamespaceSystem). + DaemonSets(OpenEBSNamespace). Delete(context.Background(), nodeDaemonSet, metav1.DeleteOptions{ PropagationPolicy: &policy, }) @@ -803,7 +803,7 @@ func deleteNodeDaemonSet() *appsv1.DaemonSet { ginkgo.By("waiting for deletion of node plugin pods") status := gomega.Eventually(func() bool { _, err = K8sClient.AppsV1(). - DaemonSets(metav1.NamespaceSystem). + DaemonSets(OpenEBSNamespace). Get(context.Background(), nodeDaemonSet, metav1.GetOptions{}) return k8serrors.IsNotFound(err) }, 120, 10).Should(gomega.BeTrue()) @@ -828,7 +828,7 @@ func createNodeDaemonSet(ds *appsv1.DaemonSet) { ds.SetResourceVersion("") // reset the resource version for creation. ginkgo.By("creating node plugin daemonset " + nodeDaemonSet) _, err := K8sClient.AppsV1(). - DaemonSets(metav1.NamespaceSystem). + DaemonSets(OpenEBSNamespace). Create(context.Background(), ds, metav1.CreateOptions{}) gomega.Expect(err).To( gomega.BeNil(),