From c7d41732b8d40bb92610bfdd217003580a299628 Mon Sep 17 00:00:00 2001 From: Diwakar Sharma Date: Tue, 23 Jul 2024 07:56:03 +0000 Subject: [PATCH] ci: enhance pull_request workflow Signed-off-by: Diwakar Sharma --- .github/workflows/pull_request.yml | 56 +++--- .github/workflows/release-charts.yml | 12 +- .gitignore | 1 + Makefile | 5 +- ci/ci-test.sh | 20 +-- ct.yaml | 1 + deploy/helm/charts/Chart.yaml | 6 +- deploy/helm/charts/README.md | 2 +- deploy/helm/charts/charts/crds/Chart.yaml | 2 +- deploy/helm/charts/templates/_helpers.tpl | 2 +- deploy/helm/charts/values.yaml | 4 +- nix/sources.json | 14 ++ nix/sources.nix | 198 ++++++++++++++++++++++ scripts/validate-chart-version.sh | 101 +++++++++++ shell.nix | 12 ++ 15 files changed, 374 insertions(+), 62 deletions(-) create mode 100644 nix/sources.json create mode 100644 nix/sources.nix create mode 100644 scripts/validate-chart-version.sh create mode 100644 shell.nix diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 586160bd..ebeb5f39 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1,29 +1,14 @@ -# Copyright 2020 The OpenEBS Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: ci +name: Pull Request CI on: pull_request: branches: - 'develop' - - 'v*' + - release/**' paths-ignore: - '*.md' - 'BUILDMETA' - 'changelogs/**' - - 'deploy/helm/**' - 'docs/**' - 'design/**' - 'LICENSE' @@ -35,6 +20,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Go 1.19 uses: actions/setup-go@v5 @@ -62,7 +49,19 @@ jobs: path: '.' pattern: '*.sh' - unit-test: + - uses: cachix/install-nix-action@v22 + - uses: rrbutani/use-nix-shell-action@v1.1.0 + with: + file: shell.nix + - name: Check if the chart follows a valid semver version + run: | + branch_name="${{ github.event.pull_request.base.ref }}" + ./scripts/validate-chart-version.sh --branch $branch_name + + - name: Run chart-testing lint + run: | + ct lint --config ct.yaml + unit-tests: runs-on: ubuntu-latest steps: - name: Checkout @@ -85,8 +84,8 @@ jobs: name: coverage-$(date +%s) flags: unittests - bdd-test: - needs: ['unit-test'] + bdd-tests: + needs: ['unit-tests'] runs-on: ubuntu-latest strategy: fail-fast: true @@ -95,6 +94,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + + - uses: cachix/install-nix-action@v22 + - uses: rrbutani/use-nix-shell-action@v1.1.0 + with: + file: shell.nix - name: Set up Go 1.19 uses: actions/setup-go@v5 @@ -102,6 +106,9 @@ jobs: go-version: 1.19.9 cache: false + - name: Build images locally + run: make lvm-driver-image || exit 1; + - name: Setup Minikube-Kubernetes uses: medyagh/setup-minikube@latest with: @@ -112,9 +119,6 @@ jobs: cni: calico start-args: '--install-addons=false' - - name: Build images locally - run: make lvm-driver-image || exit 1; - - name: bootstrap run: make bootstrap @@ -131,7 +135,7 @@ jobs: csi-driver: runs-on: ubuntu-latest - needs: ['lint', 'unit-test', 'bdd-test'] + needs: ['lint', 'unit-tests', 'bdd-tests'] steps: - name: Checkout uses: actions/checkout@v4 @@ -144,8 +148,6 @@ jobs: - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 - with: - version: v0.13.1 - name: Build uses: docker/build-push-action@v5 @@ -154,5 +156,3 @@ jobs: file: ./buildscripts/lvm-driver/Dockerfile.buildx push: false platforms: linux/amd64, linux/arm64 - tags: | - openebs/lvm-driver:ci diff --git a/.github/workflows/release-charts.yml b/.github/workflows/release-charts.yml index 95c9ba40..4638484b 100644 --- a/.github/workflows/release-charts.yml +++ b/.github/workflows/release-charts.yml @@ -1,11 +1,11 @@ name: Release Charts -on: - push: - paths: - - 'deploy/helm/**' - branches: - - develop +#on: +# push: +# paths: +# - 'deploy/helm/**' +# branches: +# - develop jobs: release: diff --git a/.gitignore b/.gitignore index af1799ba..e93a89e1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ tags *.swo *.swn *.idea +*.test diff --git a/Makefile b/Makefile index 0bb505dd..07e82ffc 100644 --- a/Makefile +++ b/Makefile @@ -52,9 +52,12 @@ ifeq (${DBUILD_SITE_URL}, ) export DBUILD_SITE_URL endif +# Set the path to the Chart.yaml file +ROOT_DIR:=$(dir $(realpath $(firstword $(MAKEFILE_LIST)))) +CHART_YAML:=${ROOT_DIR}/deploy/helm/charts/Chart.yaml ifeq (${IMAGE_TAG}, ) - IMAGE_TAG = ci + IMAGE_TAG := $(shell awk -F': ' '/^version:/ {print $$2}' $(CHART_YAML)) export IMAGE_TAG endif diff --git a/ci/ci-test.sh b/ci/ci-test.sh index 008f46bc..c329da9f 100755 --- a/ci/ci-test.sh +++ b/ci/ci-test.sh @@ -1,27 +1,12 @@ #!/usr/bin/env bash -# Copyright 2021 The OpenEBS Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - set -e -LVM_OPERATOR="$(realpath deploy/lvm-operator.yaml)" SNAP_CLASS="$(realpath deploy/sample/lvmsnapclass.yaml)" export LVM_NAMESPACE="openebs" export TEST_DIR="tests" -export NAMESPACE="kube-system" +export NAMESPACE="openebs" # allow override if [ -z "${KUBECONFIG}" ] @@ -60,7 +45,6 @@ cleanup() { kubectl delete pvc -n openebs lvmpv-pvc kubectl delete -f "${SNAP_CLASS}" - kubectl delete -f "${LVM_OPERATOR}" # always return true return 0 @@ -87,7 +71,7 @@ sudo sed -i '/^[^#]*thin_pool_autoextend_percent/ s/= .*/= 20/' /etc/lvm/lvm.con # Prepare env for running BDD tests # Minikube is already running -kubectl apply -f "${LVM_OPERATOR}" +helm install lvm-localpv ./deploy/helm/charts -n "$OPENEBS_NAMESPACE" --create-namespace --set lvmPlugin.pullPolicy=Never --set analytics.enabled=false kubectl apply -f "${SNAP_CLASS}" dumpAgentLogs() { diff --git a/ct.yaml b/ct.yaml index 7959faf3..a646494f 100644 --- a/ct.yaml +++ b/ct.yaml @@ -5,3 +5,4 @@ chart-dirs: - deploy/helm helm-extra-args: --timeout=500s validate-maintainers: false +check-version-increment: false diff --git a/deploy/helm/charts/Chart.yaml b/deploy/helm/charts/Chart.yaml index 3cc8bcb3..ee72696a 100644 --- a/deploy/helm/charts/Chart.yaml +++ b/deploy/helm/charts/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: lvm-localpv description: CSI Driver for dynamic provisioning of LVM Persistent Local Volumes. -version: 1.6.0 -appVersion: 1.6.0 +version: 1.7.0-develop +appVersion: 1.7.0-develop icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/openebs/icon/color/openebs-icon-color.png home: https://openebs.io/ keywords: @@ -16,5 +16,5 @@ sources: - https://github.com/openebs/lvm-localpv dependencies: - name: crds - version: 1.6.0 + version: 1.7.0-develop condition: crds.enabled diff --git a/deploy/helm/charts/README.md b/deploy/helm/charts/README.md index e9385ec4..133f5c7e 100644 --- a/deploy/helm/charts/README.md +++ b/deploy/helm/charts/README.md @@ -92,7 +92,7 @@ helm install openebs-lvmlocalpv openebs-lvmlocalpv/lvm-localpv --namespace opene | `lvmPlugin.image.registry` | Registry for openebs-lvm-plugin image | `""` | | `lvmPlugin.image.repository` | Image repository for openebs-lvm-plugin | `openebs/lvm-driver` | | `lvmPlugin.image.pullPolicy` | Image pull policy for openebs-lvm-plugin | `IfNotPresent` | -| `lvmPlugin.image.tag` | Image tag for openebs-lvm-plugin | `1.6.0` | +| `lvmPlugin.image.tag` | Image tag for openebs-lvm-plugin | `1.7.0-develop` | | `lvmPlugin.metricsPort` | The TCP port number used for exposing lvm-metrics | `9500` | | `lvmPlugin.allowedTopologies` | The comma seperated list of allowed node topologies | `kubernetes.io/hostname,` | | `lvmNode.driverRegistrar.image.registry` | Registry for csi-node-driver-registrar image | `registry.k8s.io/` | diff --git a/deploy/helm/charts/charts/crds/Chart.yaml b/deploy/helm/charts/charts/crds/Chart.yaml index bf745b7d..c5d90db1 100644 --- a/deploy/helm/charts/charts/crds/Chart.yaml +++ b/deploy/helm/charts/charts/crds/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v2 name: crds -version: 1.6.0 +version: 1.7.0-develop description: A Helm chart that collects CustomResourceDefinitions (CRDs) from lvm-localpv. diff --git a/deploy/helm/charts/templates/_helpers.tpl b/deploy/helm/charts/templates/_helpers.tpl index b43ec257..34580e9e 100644 --- a/deploy/helm/charts/templates/_helpers.tpl +++ b/deploy/helm/charts/templates/_helpers.tpl @@ -60,7 +60,7 @@ Define meta labels for openebs lvm-localpv components {{- define "lvmlocalpv.common.metaLabels" -}} chart: {{ template "lvmlocalpv.chart" . }} heritage: {{ .Release.Service }} -openebs.io/version: {{ .Values.release.version | quote }} +openebs.io/version: {{ .Chart.AppVersion | quote }} role: {{ .Values.role | quote }} {{- end -}} diff --git a/deploy/helm/charts/values.yaml b/deploy/helm/charts/values.yaml index c40f7405..aa4453b4 100644 --- a/deploy/helm/charts/values.yaml +++ b/deploy/helm/charts/values.yaml @@ -1,8 +1,6 @@ # Default values for openebs-lvmlocalpv. # This is a YAML-formatted file. # Declare variables to be passed into your templates. -release: - version: "1.6.0" imagePullSecrets: # - name: "image-pull-secret" @@ -151,7 +149,7 @@ lvmPlugin: repository: openebs/lvm-driver pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: 1.6.0 + tag: 1.7.0-develop ioLimits: enabled: false containerRuntime: containerd diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 00000000..9faedce3 --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,14 @@ +{ + "nixpkgs": { + "branch": "release-23.05", + "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to", + "homepage": "https://github.com/NixOS/nixpkgs", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9a333eaa80901efe01df07eade2c16d183761fa3", + "sha256": "0xhqjli4m9wkzv7xhs6fr1iajdjbv7xnj0bwvwldq9s6arlwkhj3", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/9a333eaa80901efe01df07eade2c16d183761fa3.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 00000000..fe3dadf7 --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,198 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; name = name'; } + else + pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; + + fetch_tarball = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchTarball { name = name'; inherit (spec) url sha256; } + else + pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; + + fetch_git = name: spec: + let + ref = + spec.ref or ( + if spec ? branch then "refs/heads/${spec.branch}" else + if spec ? tag then "refs/tags/${spec.tag}" else + abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!" + ); + submodules = spec.submodules or false; + submoduleArg = + let + nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0; + emptyArgWithWarning = + if submodules + then + builtins.trace + ( + "The niv input \"${name}\" uses submodules " + + "but your nix's (${builtins.nixVersion}) builtins.fetchGit " + + "does not support them" + ) + { } + else { }; + in + if nixSupportsSubmodules + then { inherit submodules; } + else emptyArgWithWarning; + in + builtins.fetchGit + ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg); + + fetch_local = spec: spec.path; + + fetch_builtin-tarball = name: throw + ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; + + fetch_builtin-url = name: throw + ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; + + # + # Various helpers + # + + # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 + sanitizeName = name: + ( + concatMapStrings (s: if builtins.isList s then "-" else s) + ( + builtins.split "[^[:alnum:]+._?=-]+" + ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) + ) + ); + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: system: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import { } + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs name spec + else if spec.type == "tarball" then fetch_tarball pkgs name spec + else if spec.type == "git" then fetch_git name spec + else if spec.type == "local" then fetch_local spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball name + else if spec.type == "builtin-url" then fetch_builtin-url name + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # If the environment variable NIV_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + replace = name: drv: + let + saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name; + ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; + in + if ersatz == "" then drv else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 + optionalAttrs = cond: as: if cond then as else { }; + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs + ( + name: spec: + if builtins.hasAttr "outPath" spec + then + abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); } + ) + config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null + , sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile) + , system ? builtins.currentSystem + , pkgs ? mkPkgs sources system + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; + +in +mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/scripts/validate-chart-version.sh b/scripts/validate-chart-version.sh new file mode 100644 index 00000000..2b02c6ed --- /dev/null +++ b/scripts/validate-chart-version.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +# Write output to error output stream. +echo_stderr() { + echo -e "${1}" >&2 +} + +die() +{ + local _return="${2:-1}" + echo_stderr "$1" + exit "${_return}" +} + +set -euo pipefail + +# Set the path to the Chart.yaml file +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" +ROOT_DIR="$SCRIPT_DIR/.." +CHART_DIR="$ROOT_DIR/deploy/helm/charts" +CHART_YAML="$CHART_DIR/Chart.yaml" + +# Check if the Chart.yaml file exists +if [ ! -f "$CHART_YAML" ]; then + die "Chart.yaml file not found in $CHART_YAML" +fi + +# Extract the chart version and app version using yq +CHART_VERSION=$(yq e '.version' "$CHART_YAML") +APP_VERSION=$(yq e '.appVersion' "$CHART_YAML") + +# Check if extraction was successful +if [ -z "$CHART_VERSION" ] || [ -z "$APP_VERSION" ]; then + die "Failed to extract versions from Chart.yaml" +fi + +# Print the extracted versions +echo "Chart Version: $CHART_VERSION" +echo "App Version: $APP_VERSION" + +# Validate that the versions are valid semver +if [ "$(semver validate "$CHART_VERSION")" != "valid" ]; then + die "Invalid chart version: $CHART_VERSION" +fi + +if [ "$(semver validate "$APP_VERSION")" != "valid" ]; then + die "Invalid app version: $APP_VERSION" +fi + +help() { + cat < Name of the branch on which this workflow is running. +Examples: + $(basename "$0") --branch develop +EOF +} + +# Parse arguments +while [ "$#" -gt 0 ]; do + case $1 in + -b|--branch) + BRANCH_NAME=$2 + shift + ;; + -h|--help) + help + exit 0 + ;; + *) + help + die "Unknown option: $1" + ;; + esac + shift +done + +# Extract major and minor version from the branch name +extract_major_minor() { + echo "$1" | awk -F/ '{print $2}' +} + +if [[ "$BRANCH_NAME" == "develop" ]]; then + if [[ "$CHART_VERSION" != *"-develop" ]]; then + die "Chart version must include '-develop' for develop branch" + fi + if [[ "$APP_VERSION" != *"-develop" ]]; then + die "App version must include '-develop' for develop branch" + fi +elif [[ "$BRANCH_NAME" =~ ^(release/[0-9]+\.[0-9]+)$ ]]; then + RELEASE_VERSION=$(extract_major_minor "$BRANCH_NAME") + if [[ "$CHART_VERSION" != "$RELEASE_VERSION."*"-prerelease" ]]; then + die "Chart version must be in format $RELEASE_VERSION.X-prerelease for release branch" + fi + if [[ "$APP_VERSION" != "$RELEASE_VERSION."*"-prerelease" ]]; then + die "App version must be in format $RELEASE_VERSION.X-prerelease for release branch" + fi +else + die "Unknown branch name: $BRANCH_NAME" +fi \ No newline at end of file diff --git a/shell.nix b/shell.nix new file mode 100644 index 00000000..54f411c0 --- /dev/null +++ b/shell.nix @@ -0,0 +1,12 @@ +let + sources = import ./nix/sources.nix; + pkgs = import sources.nixpkgs {}; +in +pkgs.mkShell { + name = "scripts-shell"; + buildInputs = with pkgs; [ + semver-tool + yq-go + chart-testing + ]; +} \ No newline at end of file