From 1ef83f379544a5416ed777395c3d8a0a919ae8a2 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 01:41:20 +0000
Subject: [PATCH 01/78] PMM-12899 Use module and build cache

---
 .github/workflows/main.yml                   |   8 +-
 build/docker/client/Dockerfile               |  18 +-
 build/docker/server/Dockerfile.el9           |  14 +-
 build/docker/server/create_users.sh          |  29 ---
 build/local/README.md                        |  34 ++++
 build/local/build.sh                         | 110 ++++++++++++
 build/local/update.sh                        | 101 +++++++++++
 build/packages/rpm/server/SPECS/grafana.spec |   4 +-
 build/scripts/build-client-binary            |  70 +++++---
 build/scripts/build-client-docker            |  14 +-
 build/scripts/build-client-packages          | 104 +++++------
 build/scripts/build-client-rpm               |  16 +-
 build/scripts/build-client-source            |  39 +++--
 build/scripts/build-client-srpm              |  17 +-
 build/scripts/build-server-rpm               | 175 ++++++++++---------
 build/scripts/build-submodules               |  52 +++---
 build/scripts/vars                           |   6 +-
 17 files changed, 519 insertions(+), 292 deletions(-)
 delete mode 100644 build/docker/server/create_users.sh
 create mode 100644 build/local/README.md
 create mode 100755 build/local/build.sh
 create mode 100755 build/local/update.sh

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 9cb7d3927a..019b6bd5d8 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -142,8 +142,8 @@ jobs:
     env:
       SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }}
       SLACK_CHANNEL: "pmm-ci"
-      SLACK_USERNAME: "PR pipelines bot"
-      SLACK_ICON_EMOJI: ":chestnut:"
+      SLACK_USERNAME: "Build bot"
+      SLACK_ICON_EMOJI: ":octocat:"
       SLACK_COLOR: "#00FF00"
       SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow"
       SLACK_MESSAGE: "${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}"
@@ -160,8 +160,8 @@ jobs:
     env:
       SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }}
       SLACK_CHANNEL: "pmm-ci"
-      SLACK_USERNAME: "PR pipelines bot"
-      SLACK_ICON_EMOJI: ":chestnut:"
+      SLACK_USERNAME: "Build bot"
+      SLACK_ICON_EMOJI: ":octocat:"
       SLACK_COLOR: "#FF0000"
       SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow"
       SLACK_MESSAGE: "Workflow failed: ${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}"
diff --git a/build/docker/client/Dockerfile b/build/docker/client/Dockerfile
index 2c03161268..6154ddfd08 100644
--- a/build/docker/client/Dockerfile
+++ b/build/docker/client/Dockerfile
@@ -5,19 +5,13 @@ RUN microdnf install shadow-utils jq -y
 RUN groupadd -g 1002 pmm-agent && \
     useradd -u 1002 -r -g pmm-agent -s /sbin/nologin \
             -d /usr/local/percona/pmm \
-            -c "PMM 2.X Client User" pmm-agent
+            -c "PMM 3.x Client User" pmm-agent
 
 FROM redhat/ubi9-micro
 
 ARG VERSION
 ARG BUILD_DATE
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses Apache-2.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management Client
-LABEL org.opencontainers.image.vendor Percona
-LABEL org.opencontainers.image.version ${VERSION}
-
 COPY LICENSE /licenses/
 COPY --from=users --chown=0:0 /etc/passwd /etc/passwd
 COPY --from=users --chown=0:0 /etc/group /etc/group
@@ -32,10 +26,18 @@ RUN install -d -o pmm-agent -g pmm-agent -m 0775 /usr/local/percona/pmm
 ADD pmm-client.tar.gz /tmp/
 
 RUN cd /tmp/pmm-client-* \
-    && env PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \
+    && PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \
     && cd /tmp \
     && rm -rf /tmp/pmm-client*
 
+LABEL org.opencontainers.image.created ${BUILD_DATE}
+LABEL org.opencontainers.image.licenses Apache-2.0
+LABEL org.opencontainers.image.title Percona Monitoring and Management Client
+LABEL org.opencontainers.image.vendor Percona
+LABEL org.opencontainers.image.version ${VERSION}
+LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/client/Dockerfile
+LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#docker
+
 USER pmm-agent
 WORKDIR /usr/local/percona/pmm/
 ENV PATH /usr/local/percona/pmm/bin/:$PATH
diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9
index 35dc8ac340..28dae9192e 100644
--- a/build/docker/server/Dockerfile.el9
+++ b/build/docker/server/Dockerfile.el9
@@ -8,12 +8,6 @@ ENV LC_ALL=en_US.utf8
 ENV GF_PLUGIN_DIR=/srv/grafana/plugins
 ENV PS1="[\u@\h \W] # "
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses AGPL-3.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management
-LABEL org.opencontainers.image.vendor Percona LLC
-LABEL org.opencontainers.image.version ${VERSION}
-
 EXPOSE 8080 8443
 
 WORKDIR /opt
@@ -40,6 +34,14 @@ RUN install -T -p -m 644 /opt/ansible/ansible.cfg /etc/ansible/ansible.cfg && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm-docker/update.yml && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm/post-build-actions.yml
 
+LABEL org.opencontainers.image.created ${BUILD_DATE}
+LABEL org.opencontainers.image.licenses AGPL-3.0
+LABEL org.opencontainers.image.title Percona Monitoring and Management
+LABEL org.opencontainers.image.vendor Percona LLC
+LABEL org.opencontainers.image.version ${VERSION}
+LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
+LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/server/docker.html
+
 USER pmm
 
 HEALTHCHECK --interval=3s --timeout=2s --start-period=10s --retries=3 CMD curl -sf http://127.0.0.1:8080/v1/readyz
diff --git a/build/docker/server/create_users.sh b/build/docker/server/create_users.sh
deleted file mode 100644
index e37eae3bf8..0000000000
--- a/build/docker/server/create_users.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-users=(
-  "pmm:1000:/bin/false:/home/pmm:pmm"
-  "nginx:1001:/sbin/nologin:/var/cache/nginx:nginx"
-)
-
-for user in "${users[@]}"; do
-  IFS=: read -r name uid shell home_dir group <<< "$user"
-  group_id="$uid"
-
-  # Check if user already exists
-  if id "$name" >/dev/null 2>&1; then
-    echo "User $name already exists"
-    continue
-  fi
-
-  # Create user with home directory if it doesn't exist
-  if [ ! -d "$home_dir" ]; then
-    mkdir -p "$home_dir"
-  fi
-
-  # Create user with specified UID, GID, and shell
-  groupadd -o -g "$group_id" "$group"
-  useradd -o -u "$uid" -g "$group" -G "$group" -s "$shell" -d "$home_dir" -c "$name" -m "$name"
-  chown "$uid:$group_id" "$home_dir"
-
-done
-
diff --git a/build/local/README.md b/build/local/README.md
new file mode 100644
index 0000000000..1517e9da3f
--- /dev/null
+++ b/build/local/README.md
@@ -0,0 +1,34 @@
+# Local builds
+
+This directory contains a set of scripts aimed at providing a simple way to build PMM locally.
+
+## Background
+
+Historically, PMM used to be built using Jenkins. This worked well for the team, but not for the community. The learning curve was, and still is, rather steep, and it is hard for folks, even internally, to contribute to.
+
+Therefore, we decided to make it possible to build PMM locally. This is a work in progress, but we are definitely committed to making it easier to build PMM locally.
+
+The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on to make the build process easy to understand and contribute to. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching.
+
+The build process is designed to be run on a Linux host. We believe it can be run on other flavors of Linux, including MacOS, with little to no modification (TBC).
+
+## Prerequisites
+
+Below is a list of prerequisites that are required to build PMM locally.
+
+- OS: Linux (tested on Oracle Linux 9.3, CentOS 7, Ubuntu 22.04.3 LTS)
+- Docker: 25.0.2+ (tested on 25.0.2)
+- Docker Compose Plugin: 2.24.7+ (tested on 2.24.7)
+- make: 4.3+ (tested on 4.3)
+- bash (GNU): 5.1+ (tested on 5.1)
+- git: 2.34.1+ (tested on 2.34.1)
+- curl: 7.81.0+ (tested on 7.81.0)
+- yq: 4.42.0+ (tested on 4.42.1)
+- jq: 1.6+ (tested on 1.6)
+
+## Build Steps
+
+1. Install the prerequisites
+2. Clone the PMM repository
+3. Change to the `build/local` directory
+4. Run the `build.sh` script
diff --git a/build/local/build.sh b/build/local/build.sh
new file mode 100755
index 0000000000..f27ec569a8
--- /dev/null
+++ b/build/local/build.sh
@@ -0,0 +1,110 @@
+#!/bin/bash -e
+
+# Update submodules and PR branches
+/bin/bash $(dirname $0)/update.sh
+
+get_branch_name() {
+  local module="${1:-}"
+  local branch_name
+  local path
+
+  path=$(git config -f .gitmodules submodule.${module}.path)
+  cd "$path" || exit 1
+  branch_name=$(git branch --show-current)
+  cd - > /dev/null
+  echo $branch_name
+}
+
+# Define global variables
+pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+echo $pmm_commit > apiCommitSha
+pmm_branch=$(get_branch_name pmm)
+echo $pmm_branch > apiBranch
+pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+echo $pmm_url > apiURL
+pmm_qa_branch=$(get_branch_name pmm-qa)
+echo $pmm_qa_branch > pmmQABranch
+pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+echo $pmm_qa_commit > pmmQACommitSha
+pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
+echo $pmm_ui_tests_branch > pmmUITestBranch
+pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
+echo $pmm_ui_tests_commit > pmmUITestsCommitSha
+fb_commit_sha=$(git rev-parse HEAD)
+echo $fb_commit_sha > fbCommitSha
+
+# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:ol9`.
+# Important: the docker container's user need to be able to write to these directories.
+# The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
+# that the directories we create on the host are owned by a user with same uid and gid.
+
+# Create cache directories. 
+test -d "${root_dir}/go-path" || mkdir -p "go-path"
+test -d "${root_dir}/go-build" || mkdir -p "go-build"
+
+PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9 
+
+# Local reference test environment
+# CPU: 4 cores
+# RAM: 16GB
+# OS: Ubuntu 22.04.1 LTS
+
+# Build client source: 4m39s from scratch, 0m27s using cache
+"$PATH_TO_SCRIPTS/build-client-source"
+
+# Build client binary: ??? from scratch, 0m20s using cache
+"$PATH_TO_SCRIPTS/build-client-binary"
+
+# Building client source rpm takes 13s (caching does not apply)
+"$PATH_TO_SCRIPTS/build-client-srpm"
+
+# Building client rpm takes 1m40s
+"$PATH_TO_SCRIPTS/build-client-rpm"
+
+# Building client docker image takes 17s
+GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
+export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
+"$PATH_TO_SCRIPTS/build-client-docker"
+
+# Building PMM CLient locally (non-CI, i.e. non-Jenkins)
+# total time: 6m26s - build from scratch, no initial cache
+# total time: 2m49s - subsequent build, using cache from prior builds
+
+
+# Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
+# total time: 8m45s - build from scratch, no initial cache
+# total time: ??? - subsequent build, using cache from prior builds
+
+export RPM_EPOCH=1
+export RPMBUILD_DIST="el9"
+"$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
+"$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
+"$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
+"$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
+"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-controller
+"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-tools
+"$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
+"$PATH_TO_SCRIPTS/build-server-rpm" grafana-db-migrator
+"$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+
+# 3rd-party
+"$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
+"$PATH_TO_SCRIPTS/build-server-rpm" alertmanager
+"$PATH_TO_SCRIPTS/build-server-rpm" grafana
+
+export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9
+export RPMBUILD_DIST=el9
+export DOCKERFILE=Dockerfile.el9.local
+${PATH_TO_SCRIPTS}/build-server-docker
+
+# Clean up temporary files
+rm -f apiBranch \
+  apiCommitSha \
+	apiURL \
+	fbCommitSha \
+	pmmQABranch \
+	pmmQACommitSha \
+	pmmUITestBranch \
+	pmmUITestsCommitSha
diff --git a/build/local/update.sh b/build/local/update.sh
new file mode 100755
index 0000000000..4922dd9bda
--- /dev/null
+++ b/build/local/update.sh
@@ -0,0 +1,101 @@
+#!/bin/bash -e
+
+needs-to-pull() {
+  local UPSTREAM=${1:-'@{u}'}
+  local LOCAL=$(git rev-parse @)
+  local BASE=$(git merge-base @ "$UPSTREAM")
+  local REMOTE=$(git rev-parse "$UPSTREAM")
+
+  if [ "$LOCAL" = "$REMOTE" ]; then
+    return 1 # false, we are up-to-date
+  fi
+
+  if [ "$LOCAL" = "$BASE" ]; then
+    return 0 # true, we are behind upstream
+  fi
+}
+
+rewind() {
+  local DIR="$1"
+  local BRANCH="$2"
+
+  cd "$DIR"
+  CURRENT=$(git branch --show-current)
+  git fetch
+
+  if [ "$CURRENT" != "$BRANCH" ]; then
+    echo "Currently on $CURRENT, checking out $BRANCH"
+    git checkout "$BRANCH"
+  fi
+
+  if needs-to-pull; then
+    git pull origin
+    echo "Submodule has pulled from upstream"
+    git logs -n 2
+    cd - >/dev/null
+    git add "$DIR"
+  else
+    cd - >/dev/null
+    echo "Submodule is up-to-date with upstream"
+  fi
+}
+
+check-files() {
+  local DIR="$1"
+
+  test -z "DIR" && exit 1
+
+  if [ -d "$DIR/sources" ] && [ -f "$DIR/ci-default.yml" ] && [ -f "$DIR/ci.yml" ]; then
+    return 0
+  fi
+
+  return 1
+}
+
+main() {
+  local DEPS=
+  local CURDIR="$PWD"
+  local DIR=pmm-submodules
+
+  # Thouroughly verify the presence of known files, otherwise bail out
+  if check-files "."; then # pwd is pmm-submodules
+    DIR="."
+  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
+    if ! check-files "$DIR"; then
+      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
+      exit 1
+    fi
+  else
+    echo "FATAL: could not locate known files in $PWD"
+    exit 1
+  fi
+
+  cd "$DIR"
+
+  # Join the dependencies from ci-default.yml and ci.yml
+  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
+
+  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'"
+
+  echo "$DEPS" | jq -c '.[]' | while read -r item; do
+    branch=$(echo "$item" | jq -r '.branch')
+    path=$(echo "$item" | jq -r '.path')
+    name=$(echo "$item" | jq -r '.name')
+    echo
+    echo "Rewinding submodule '$name' ..."
+    echo "path: ${path}, branch: ${branch}"
+
+    rewind "$path" "$branch"
+  done
+
+  echo
+  echo "Printing git status..."
+  git status --short
+  echo
+  echo "Printing git submodule status..."
+  git submodule status
+
+  cd "$CURDIR" > /dev/null
+}
+
+main
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index bd16ba3121..85976d18a1 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -17,8 +17,8 @@ Version:        %{grafana_version}
 Release:        %{rpm_release}
 Summary:        Grafana is an open source, feature rich metrics dashboard and graph editor
 License:        AGPLv3
-URL:            https://github.com/percona-platform/grafana
-Source0:        https://github.com/percona-platform/grafana/archive/%{commit}.tar.gz
+URL:            https://github.com/percona/grafana
+Source0:        https://github.com/percona/grafana/archive/%{commit}.tar.gz
 ExclusiveArch:  %{ix86} x86_64 %{arm}
 
 BuildRequires: fontconfig
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 82d88c645d..295a5a5b02 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -5,6 +5,9 @@ set -o xtrace
 
 . $(dirname $0)/vars
 
+# Important: `bin_dir` and `binary_dir` are different directories. `bin_dir` defines the directory for binary files 
+# inside the container, while `binary_dir` defines the directory for binary files outside the container.
+
 extract_source_tarball() {
     if [ ! -d "${source_dir}" ]; then
         mkdir -p $(dirname ${source_dir})
@@ -18,58 +21,71 @@ gobuild_component() {
     local extract_path=${3:-"github.com/percona/$parent_component"}
     local component_path=$4
     local result_file=${5:-bin/$(basename ${component_path:-$extract_path})}
+    local volume_mounts="-v ${tmp_dir}:/home/builder/tmp"
+    local src_dir=/home/builder/tmp/source/pmm-client-${pmm_version}
+    local bin_dir=/home/builder/tmp/binary/pmm-client-${pmm_version}
+    local archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz" | head -n1)
+    local build_command
+
+    if [ -d "${root_dir}/go-build" ]; then
+        volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
+    fi
+    if [ -d "${root_dir}/go-path" ]; then
+        volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
+    fi
 
     if [ ${component} = 'azure_metrics_exporter' ]; then
         result_file=${result_file/_metrics/}
     fi
 
     if [ -x "${binary_dir}/${result_file}" ]; then
-        echo skip build
+        echo Skipping the build...
         return
     fi
-    if [ ${component} = 'pmm-admin' -o ${component} = 'pmm-agent' ]; then
+    if [[ ${component} =~ pmm-admin|pmm-agent ]]; then
         build_command="cd /tmp/go/src/${extract_path}/${component_path} && make release"
     elif [ ${component} = 'mongodb_exporter' ]; then
         build_command="cd /tmp/go/src/${extract_path} && make build"
-    elif [ ${component} = 'node_exporter' ] || [ ${component} = 'mysqld_exporter' ] || \
-         [ ${component} = 'proxysql_exporter' ] || [ ${component} = 'rds_exporter' ] || \
-         [ ${component} = 'postgres_exporter' ] || [ ${component} = 'azure_metrics_exporter' ]; then
+    elif [[ ${component} =~ node_exporter|mysqld_exporter|proxysql_exporter|rds_exporter|postgres_exporter|azure_metrics_exporter ]]; then
         build_command="cd /tmp/go/src/${extract_path} && make release"
     elif [ ${component} = 'percona-toolkit' ]; then
-        build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o \${binary_dir}/${result_file} ."
+        build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     else
-        build_command="go build -o \${binary_dir}/${result_file} ./src/${extract_path}/${component_path}"
+        build_command="cd /tmp/go/src/${extract_path}/${component_path} go build -o ${bin_dir}/${result_file} ."
     fi
-    docker run --rm -v ${tmp_dir}:/home/builder/tmp ${rpmbuild_docker_image} sh -c "
+
+    docker run --rm \
+      ${volume_mounts} \
+      ${rpmbuild_docker_image} sh -c "
         set -o errexit
         set -o xtrace
         export GOOS=${GOOS:-linux}
         export GOARCH=${GOARCH:-amd64}
         export CGO_ENABLED=0
         export GO111MODULE=auto
+        export GOPATH=/tmp/go
+        export GOMODCACHE=/home/builder/go/pkg/mod
 
-        # for 'make release'
+        # Setting variable for 'make release'
         export PMM_RELEASE_VERSION=${full_pmm_version}
         export PMM_RELEASE_TIMESTAMP=$(date '+%s')
-        export PMM_RELEASE_FULLCOMMIT=$(cd sources/${parent_component}/src/${extract_path} && git rev-parse HEAD)
-        export COMPONENT_VERSION=$(cd sources/${parent_component}/src/${extract_path} && git describe --abbrev=0 --always)
-        export COMPONENT_BRANCH=$(cd sources/${parent_component}/src/${extract_path} && git describe --always --contains --all)
         export PMM_RELEASE_BRANCH=""
         sudo chown -R builder:builder /home/builder/tmp
 
-        source_dir=/home/builder/tmp/source/pmm-client-${pmm_version}
-        binary_dir=/home/builder/tmp/binary/pmm-client-${pmm_version}
-        export PMM_RELEASE_PATH=\${binary_dir}/bin
-
-        mkdir -p \$binary_dir/bin
+        export PMM_RELEASE_PATH=${bin_dir}/bin
+        if [ "${component}" = vmagent ]; then
+            export PMM_RELEASE_FULLCOMMIT=${vmagent_commit_hash}
+        else
+            export PMM_RELEASE_FULLCOMMIT=$(git -C sources/${parent_component}/src/${extract_path} rev-parse HEAD)
+        fi
 
+        mkdir -p ${bin_dir}/bin
         mkdir -p /tmp/go/src/${extract_path}
-        archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz")
-        tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf \${source_dir}/\$archive
-        pushd /tmp/go
-            export GOPATH=\$(pwd -P)
-            ${build_command}
-        popd
+
+        tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf ${src_dir}/${archive}
+
+        ${build_command}
+
         sudo chown -R $(id -u):$(id -g) /home/builder/tmp
     "
 }
@@ -83,8 +99,7 @@ copy_component() {
 
     if [ ! -d "${component_dir}" ]; then
         mkdir -p ${component_dir}
-        tar \
-            -C ${component_dir} \
+        tar -C ${component_dir} \
             --strip-components=1 \
             -zxpf "$archive"
     fi
@@ -104,7 +119,6 @@ main() {
     gobuild_component "rds_exporter"
     gobuild_component "azure_metrics_exporter"
     gobuild_component "clickhouse_exporter" "" "github.com/Percona-Lab/clickhouse_exporter"
-
     gobuild_component "pmm-admin" "pmm" "" "admin"
     gobuild_component "pmm-agent" "pmm" "" "agent"
 
@@ -121,16 +135,16 @@ main() {
     copy_component "postgres_exporter" "queries-postgres-uptime.yml"
     copy_component "postgres_exporter" "queries-mr.yaml"
     copy_component "postgres_exporter" "queries-lr.yaml"
-
     copy_component "percona-toolkit" "bin/pt-summary"
     copy_component "percona-toolkit" "bin/pt-mysql-summary"
+
     gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-mongodb-summary"
     gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-pg-summary"
 
     rm -rf ${binary_tarball}
     mkdir -p $(dirname ${binary_tarball}) || :
     
-    # Removes pmm binary in case this is not a PR build - such as RC/GA build
+    # Removes pmm binary (aka CLI) in case this is not a PR build - such as RC/GA build
     if [[ "$(basename ${root_dir})" != pmm-submodules_PR-* ]]; then
         rm -f "${binary_dir}/bin/pmm"
     fi
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index ea6287703f..119a47f7a3 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -20,13 +20,13 @@ fi
 
 CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
 
-docker build --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
-             --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
-             --squash \
-             --no-cache \
-             -f ${DOCKER_FILE_LOCATION}/${docker_file} \
-             -t ${DOCKER_CLIENT_TAG} \
-             ${DOCKER_FILE_LOCATION}
+docker buildx build \
+    --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
+    --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
+    --progress plain \
+    -f ${DOCKER_FILE_LOCATION}/${docker_file} \
+    -t ${DOCKER_CLIENT_TAG} \
+    ${DOCKER_FILE_LOCATION}
 
 if [ -n "${PUSH_DOCKER}" ]; then
     mkdir -p $(dirname ${docker_client_tag_file})
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index ae155999aa..97865555b3 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -17,7 +17,7 @@ Usage: $0 [OPTIONS]
         --build_source_deb  If it is 1 source deb package will be built
         --build_rpm         If it is 1 rpm will be built
         --build_deb         If it is 1 deb will be built
-        --install_deps      Install build dependencies(root previlages are required)
+        --install_deps      Install build dependencies(root privileges are required)
         --branch            Branch from which submodules should be taken(default v3)
         --help) usage ;;
 Example $0 --builddir=/tmp/PMM_CLIENT --get_sources=1 --build_src_rpm=1 --build_rpm=1
@@ -90,13 +90,12 @@ check_workdir(){
     if [ "x$WORKDIR" = "x$CURDIR" ]; then
         echo >&2 "Current directory cannot be used for building!"
         exit 1
-    else
-        if ! test -d "$WORKDIR"; then
-            echo >&2 "$WORKDIR is not a directory."
-            exit 1
-        fi
     fi
-    return
+    
+    if ! test -d "$WORKDIR"; then
+        echo >&2 "$WORKDIR is not a directory."
+        exit 1
+    fi
 }
 
 get_sources(){
@@ -108,11 +107,11 @@ get_sources(){
     git clone $REPO
     retval=$?
     if [ $retval != 0 ]; then
-        echo "There were some issues during repo cloning from github. Please retry one more time"
+        echo "There were some issues during repo cloning from github. Please retry."
         exit 1
     fi
     cd pmm-client
-    if [ ! -z $BRANCH ]; then
+    if [ -n $BRANCH ]; then
         git reset --hard
         git clean -xdf
         git checkout $BRANCH
@@ -122,7 +121,6 @@ get_sources(){
     #
     VERSION=$(pmm_version)
     mv Makefile build/
-    #cat VERSION > $VERSION_FILE
     echo "VERSION=${VERSION}" > $VERSION_FILE
     echo "REVISION=${REVISION}" >> $VERSION_FILE
     echo "RPM_RELEASE=${RPM_RELEASE}" >> $VERSION_FILE
@@ -133,7 +131,6 @@ get_sources(){
     echo "MongoExp_BRANCH_NAME=${MongoExp_BRANCH_NAME}" >> $VERSION_FILE
     echo "MysqlExp_BRANCH_NAME=${MysqlExp_BRANCH_NAME}" >> $VERSION_FILE
     echo "ProxysqlExp_BRANCH_NAME=${ProxysqlExp_BRANCH_NAME}" >> $VERSION_FILE
-    echo "QAN_BRANCH_NAME=${QAN_BRANCH_NAME}" >> $VERSION_FILE
     echo "TOOLKIT_REPO=${TOOLKIT_REPO}" >> $VERSION_FILE
     echo "TOOLKIT_BRANCH_NAME=${TOOLKIT_BRANCH_NAME}" >> $VERSION_FILE
     PRODUCT=pmm-client
@@ -146,10 +143,8 @@ get_sources(){
     echo "BUILD_ID=${BUILD_ID}" >> $VERSION_FILE
     echo "UPLOAD=UPLOAD/experimental/BUILDS/${PRODUCT_NAME}/${VERSION}/${BRANCH_NAME}/${REVISION}/${BUILD_ID}" >> $VERSION_FILE
     echo "MongoExp_REPO=${MongoExp_REPO}" >> $VERSION_FILE
-    echo "TOOLKIT_REPO=${TOOLKIT_REPO}" >> $VERSION_FILE
     echo "MysqlExp_REPO=${MysqlExp_REPO}" >> $VERSION_FILE
     echo "ProxysqlExp_REPO=${ProxysqlExp_REPO}" >> $VERSION_FILE
-    echo "QAN_REPO=${QAN_REPO}" >> $VERSION_FILE
     echo "NodeExp_REPO=${NodeExp_REPO}" >> $VERSION_FILE
     cd ../
     mv ${PRODUCT} ${PRODUCT}-${VERSION}
@@ -179,6 +174,7 @@ get_system(){
 }
 
 install_deps() {
+    local RUN_CMD
     if [ $INSTALL = 0 ]; then
         echo "Dependencies will not be installed"
         return;
@@ -188,7 +184,17 @@ install_deps() {
         exit 1
     fi
     if [ "$OS" = "rpm" ]; then
-        yum -y install git wget rpmdevtools bison yum-utils rpm-build
+        # yum -y install git curl wget rpmdevtools bison yum-utils rpm-build
+        command -v git || RUN_CMD+=" git"
+        command -v curl || RUN_CMD+=" curl"
+        command -v wget || RUN_CMD+=" wget"
+        command -v spectool || RUN_CMD+=" rpmdevtools"
+        command -v bison || RUN_CMD+=" bison"
+        command -v repoquery || RUN_CMD+=" yum-utils"
+        command -v rpmbuild || RUN_CMD+=" rpm-build"
+        if [ -n "$RUN_CMD" ]; then
+            yum -y install "$RUN_CMD"
+        fi
     else
         apt-get update
         apt-get -y install devscripts equivs debhelper
@@ -198,9 +204,9 @@ install_deps() {
 
 get_tar(){
     TARBALL=$1
-    TARFILE=$(basename $(find $WORKDIR/$TARBALL -name 'pmm-client*.tar.gz' | sort | tail -n1) || :)
+    TARFILE=$(basename $(find $WORKDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
     if [ -z $TARFILE ]; then
-        TARFILE=$(basename $(find $CURDIR/$TARBALL -name 'pmm-client*.tar.gz' | sort | tail -n1) || :)
+        TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
         if [ -z $TARFILE ]; then
             echo "There is no $TARBALL for build"
             exit 1
@@ -216,9 +222,9 @@ get_tar(){
 get_deb_sources(){
     param=$1
     echo $param
-    FILE=$(basename $(find $WORKDIR/source_deb -name "pmm-client*.$param" | sort | tail -n1) || :)
+    FILE=$(basename $(find $WORKDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :)
     if [ -z $FILE ]; then
-        FILE=$(basename $(find $CURDIR/source_deb -name "pmm-client*.$param" | sort | tail -n1) || :)
+        FILE=$(basename $(find $CURDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :)
         if [ -z $FILE ]; then
             echo "There is no sources for build"
             exit 1
@@ -232,7 +238,7 @@ get_deb_sources(){
 }
 
 build_srpm(){
-    if [ $SRPM = 0 ]; then
+    if [ "$SRPM" = 0 ]; then
         echo "SRC RPM will not be created"
         return;
     fi
@@ -242,28 +248,24 @@ build_srpm(){
     fi
     cd $WORKDIR
     get_tar "tarball"
-    
-    #
+
     rm -fr rpmbuild
     ls | grep -v tar.gz | xargs rm -rf
-    
-    #
-    TARFILE=$(basename $(find . -name 'pmm-client-*.tar.gz' | sort | tail -n1))
+
+    TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
     NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
     VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
     VERSION=${VERSION_TMP%.tar.gz}
-    
-    #
+
     mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
-    
-    #
+
     tar -C ${WORKDIR} -zxpf ${TARFILE} ${NAME}-${VERSION}/rpm
-    #
+
     cd ${WORKDIR}/rpmbuild/SPECS
     cp ${WORKDIR}/${NAME}-${VERSION}/rpm/*.spec .
     cp ${WORKDIR}/${TARFILE} ../SOURCES/
     cd ${WORKDIR}
-    rpmbuild -bs --define "_topdir ${WORKDIR}/rpmbuild" --define "version $VERSION" --define "release $RPM_RELEASE" --define "dist %{nil}" rpmbuild/SPECS/pmm-client.spec
+    rpmbuild -bs --define "_topdir ${WORKDIR}/rpmbuild" --define "version $VERSION" --define "release $RPM_RELEASE" --define "dist %{nil}" "rpmbuild/SPECS/${PACKAGE_NAME}.spec"
     mkdir -p ${WORKDIR}/srpm
     mkdir -p ${CURDIR}/srpm
     cp rpmbuild/SRPMS/*.src.rpm ${CURDIR}/srpm
@@ -271,7 +273,7 @@ build_srpm(){
 }
 
 build_rpm(){
-    if [ $RPM = 0 ]; then
+    if [ "$RPM" = 0 ]; then
         echo "RPM will not be created"
         return;
     fi
@@ -280,9 +282,9 @@ build_rpm(){
         exit 1
     fi
 
-    SRC_RPM=$(basename $(find $WORKDIR/srpm -name 'pmm-client*.src.rpm' | sort | tail -n1) || :)
+    SRC_RPM=$(basename $(find $WORKDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
     if [ -z $SRC_RPM ]; then
-        SRC_RPM=$(basename $(find $CURDIR/srpm -name 'pmm-client*.src.rpm' | sort | tail -n1) || :)
+        SRC_RPM=$(basename $(find $CURDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
         if [ -z $SRC_RPM ]; then
             echo "There is no src rpm for build"
             echo "You can create it using key --build_src_rpm=1"
@@ -298,6 +300,10 @@ build_rpm(){
     rm -fr rpmbuild
     mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
     cp $SRC_RPM rpmbuild/SRPMS/
+    if find "${CURDIR}/rpm" -name "${PACKAGE_NAME}-${pmm_version}-*.rpm"; then
+        # Remove older client rpm packages before building a new one
+        rm -f "${CURDIR}/rpm/${PACKAGE_NAME}-${pmm_version}-*.rpm"
+    fi
     rpmbuild --define "_topdir ${WORKDIR}/rpmbuild" \
              --define "version $pmm_version" \
              --define "release $RPM_RELEASE" \
@@ -305,17 +311,17 @@ build_rpm(){
              --rebuild rpmbuild/SRPMS/$SRC_RPM
 
     return_code=$?
-    if [ $return_code != 0 ]; then
+    if [ "$return_code" -ne 0 ]; then
+        echo "Error: rpmbuild exited with a non-zero return code '$return_code'\n"
         exit $return_code
     fi
-    mkdir -p ${WORKDIR}/rpm
+
     mkdir -p ${CURDIR}/rpm
-    cp rpmbuild/RPMS/*/*.rpm ${WORKDIR}/rpm
     cp rpmbuild/RPMS/*/*.rpm ${CURDIR}/rpm
 }
 
 build_source_deb(){
-    if [ $SDEB = 0 ]; then
+    if [ "$SDEB" = 0 ]; then
         echo "source deb package will not be created"
         return;
     fi
@@ -326,17 +332,16 @@ build_source_deb(){
     rm -rf pmm-client*
     get_tar "tarball"
     rm -f *.dsc *.orig.tar.gz *.debian.tar.gz *.changes
-    #
-    TARFILE=$(basename $(find . -name 'pmm-client-*.tar.gz' | sort | tail -n1))
+
+    TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
     NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
     VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
     VERSION=${VERSION_TMP%.tar.gz}
 
     rm -fr ${NAME}-${VERSION}
-    #
+
     NEWTAR=${NAME}_${VERSION}.orig.tar.gz
     mv ${TARFILE} ${NEWTAR}
-    #
 
     git clone $REPO ${NAME}-${VERSION}_all
     pushd ${NAME}-${VERSION}_all
@@ -363,7 +368,7 @@ build_source_deb(){
     mv queries-mr.yaml ../${NAME}-${VERSION}_all/distro/
     mv queries-lr.yaml ../${NAME}-${VERSION}_all/distro/
     mv debian ../${NAME}-${VERSION}_all/
-    mv config/pmm-agent.service ../${NAME}-${VERSION}_all/debian/pmm-client.pmm-agent.service
+    mv config/pmm-agent.service ../${NAME}-${VERSION}_all/debian/${PACKAGE_NAME}.pmm-agent.service
     cd ../
 
     rm -rf ${NAME}-${VERSION}
@@ -396,12 +401,12 @@ build_source_deb(){
 }
 
 build_deb(){
-    if [ $DEB = 0 ]; then
-        echo "source deb package will not be created"
+    if [ "$DEB" = 0 ]; then
+        echo "deb package will not be created"
         return;
     fi
     if [ "$OS" = "rpm" ]; then
-        echo "It is not possible to build source deb here"
+        echo "It is not possible to build deb here"
         exit 1
     fi
     for file in 'dsc' 'orig.tar.gz' 'changes' 'diff.gz'; do
@@ -432,6 +437,7 @@ build_deb(){
 #main
 
 CURDIR=$(pwd)
+PACKAGE_NAME=pmm3-client
 VERSION_FILE=$CURDIR/pmm-client.properties
 args=
 WORKDIR=
@@ -446,15 +452,15 @@ ARCH=
 OS=
 SUBMODULE_BRANCH="v3"
 INSTALL=0
-RPM_RELEASE=6
-DEB_RELEASE=6
+RPM_RELEASE=1
+DEB_RELEASE=1
+REVISION=0
 
 if [ -n "$pmm_release" ]; then
     RPM_RELEASE="$RPM_RELEASE.$pmm_release"
     DEB_RELEASE="$DEB_RELEASE.$pmm_release"
 fi
 
-REVISION=0
 parse_arguments PICK-ARGS-FROM-ARGV "$@"
 get_system
 install_deps
@@ -482,8 +488,6 @@ if [ "$TARBALL" = '1' -o "$SOURCE" = '1' ]; then
 fi
 
 check_workdir
-get_system
-install_deps
 get_sources
 build_srpm
 build_source_deb
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index 7c178670d0..ae5780ae24 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -5,9 +5,6 @@
 set -o errexit
 set -o xtrace
 
-bin_dir=$(cd $(dirname $0); pwd -P)
-tmp_dir=${root_dir}/tmp
-
 main() {
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
@@ -18,14 +15,11 @@ main() {
         export pmm_release=$pmm_release
         export sudo_path=\$(ls /usr/bin/sudo)
         mkdir -p /tmp/pmm
-        pushd /home/builder/results
-            \$sudo_path /home/builder/bin/build-client-packages \
-                --builddir=/tmp/pmm \
-                --install_deps=1
-            /home/builder/bin/build-client-packages \
-                --builddir=/tmp/pmm \
-                --build_rpm=1
-        popd
+        cd /home/builder/results
+        /home/builder/bin/build-client-packages \
+            --builddir=/tmp/pmm \
+            --build_rpm=1
+
         \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/rpm
     "
 }
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index b9867f764b..978f09a1b0 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -7,11 +7,12 @@ set -o xtrace
 
 prepare_tarball() {
     local repo_name=$1
+    local git_dir
     if [ $repo_name = 'pmm-submodules' ]; then
-        mkdir -p ${root_dir}/sources/${repo_name}
         git_dir="${root_dir}/sources/${repo_name}"
+        mkdir -p "${git_dir}"
     else
-        local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
+        git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
     fi
     local commit=$(git -C "${git_dir}" rev-parse --short HEAD)
     local tarball=${source_dir}/${repo_name}-${commit}.tar.gz
@@ -24,43 +25,43 @@ prepare_tarball() {
 
     if [ -f "${tarball}" ]; then
         echo ${tarball} already exists, skipping build
-    else
-        if [ ${repo_name} = 'pmm-submodules' ]; then
-            tar -C sources -zcf ${repo_name}-${commit}.tar.gz  ${repo_name}
-            mv ${repo_name}-${commit}.tar.gz  ${source_dir}/
-        else
-            git -C "${git_dir}" archive \
-                --format=tar.gz \
-                --prefix=${repo_name}-${commit}/ \
-                -o ${tarball} \
-                "${commit}"
-        fi
+        return
+    fi
+
+    if [ ${repo_name} = 'pmm-submodules' ]; then
+        tar -C sources -zcf ${repo_name}-${commit}.tar.gz  ${repo_name}
+        mv ${repo_name}-${commit}.tar.gz  ${source_dir}/
+        return
     fi
+
+    git -C "${git_dir}" archive \
+        --format=tar.gz \
+        --prefix=${repo_name}-${commit}/ \
+        -o ${tarball} \
+        "${commit}"
 }
 
 prepare_vmagent_tarball() {
     local repo_name=vmagent
     local commit_hash=${vmagent_commit_hash}
-    local commit_short=${commit_hash::6}
+    local commit_short=${commit_hash::7}
     local tarball=${source_dir}/${repo_name}-${commit_short}.tar.gz
     local repo_var=$(
-        echo "${repo_name}_commit" \
-            | sed -e 's:-:_:g' \
-            | tr '[[:lower:]]' '[[:upper:]]'
+        echo "${repo_name}_commit" | sed -e 's:-:_:g' | tr '[[:lower:]]' '[[:upper:]]'
     )
     echo "${repo_var}=$commit_short" >> ${client_properties}
 
     if [ -f "${tarball}" ]; then
         echo ${tarball} already exists, skipping download
     else
-        wget https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz -O ${tarball}
+        curl -o ${tarball} -fSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz
     fi
 }
 
 main() {
     rm -rf ${source_tarball}
     mkdir -p ${source_dir}/ $(dirname ${source_tarball}) || :
-    echo "PMM_SUBMODULES_COMMIT=$(git rev-parse --short HEAD)" > ${client_properties}
+
     prepare_vmagent_tarball
     prepare_tarball pmm-submodules
     prepare_tarball pmm
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index aff1082492..20747d0779 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -5,9 +5,6 @@
 set -o errexit
 set -o xtrace
 
-bin_dir=$(cd $(dirname $0); pwd -P)
-tmp_dir=${root_dir}/tmp
-
 main() {
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
@@ -15,17 +12,13 @@ main() {
         set -o xtrace
 
         export pmm_release=$pmm_release
-        export sudo_path=
-        if [ -e /usr/bin/sudo ]; then
-            export sudo_path=\$(ls /usr/bin/sudo)
-        fi
+        export sudo_path=\$(command -v sudo)
         [[ ${IMAGE} = ${rpmbuild_docker_image} ]] || \$sudo_path yum -y install git rpm-build
         mkdir -p /tmp/pmm
-        pushd /home/builder/results
-            /home/builder/bin/build-client-packages \
-                --builddir=/tmp/pmm \
-                --build_src_rpm=1
-        popd
+        cd /home/builder/results
+        /home/builder/bin/build-client-packages \
+            --builddir=/tmp/pmm \
+            --build_src_rpm=1
         \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/srpm
     "
 }
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 4c2cb97df6..9be0b05f3c 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -24,6 +24,7 @@ is_build_needed() {
     local spec_name=$1
     local rpm_version=$2
     local packages=
+    local cache_dir=RELEASE
 
     # Structure of S3 build cache
     # s3://pmm-build-cache/PR-BUILDS/7 - el7
@@ -31,18 +32,17 @@ is_build_needed() {
     # s3://pmm-build-cache/RELEASE/7 - el7
     # s3://pmm-build-cache/RELEASE/9 - el9
 
-    # RPM_EPOCH is set for Feature Builds
-    if [ -n "$RPM_EPOCH" ]; then
-        aws s3 sync \
-            --region us-east-2 \
-            --no-sign-request \
-            s3://pmm-build-cache/PR-BUILDS/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-            ${rpms_dir}/${spec_name}-${rpm_version}
-    else
+    [ "${FORCE_REBUILD}" = 1 ] && return 0
+
+    # RPM_EPOCH is set to 1 for Feature Builds
+    if [ "$RPM_EPOCH" = 1 ]; then
+        cache_dir=PR-BUILDS
+    fi
+    if command -v aws &> /dev/null; then
         aws s3 sync \
             --region us-east-2 \
             --no-sign-request \
-            s3://pmm-build-cache/RELEASE/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+            s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
             ${rpms_dir}/${spec_name}-${rpm_version}
     fi
 
@@ -58,103 +58,120 @@ prepare_specs() {
 
     sudo chown -R $(id -u):$(id -g) ${rpmbuild_dir}/SPECS ${rpmbuild_dir}/SOURCES
     cp ${rpmbuild_dir}/SPECS/${spec_name}.spec ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-    if [ -d "${root_dir}/sources/${repo_name}" ]; then
-        local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
-        local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
-        local short_commit=${full_commit:0:7}
-
-        sed -i -e "s/global commit.*/global commit ${full_commit}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-        if [[ ${spec_name} != "grafana" ]]; then
-            sed -i -e "s/Version:.*/Version: ${pmm_version}/"       ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-        fi
+    if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
+        return
+    fi
+    
+    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
+    local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
+    local short_commit=${full_commit:0:7}
+
+    sed -i -e "s/global commit.*/global commit ${full_commit}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    if [ "${spec_name}" != "grafana" ]; then
+        sed -i -e "s/Version:.*/Version: ${pmm_version}/"       ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    fi
 
-        if [ -z "${full_pmm_version}" ]; then
-            echo 'A full_pmm_version is not specified.'
-            exit 1
-        fi
-        sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    if [ -z "${full_pmm_version}" ]; then
+        echo 'The full_pmm_version is not specified.'
+        exit 1
+    fi
+    sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
 
-        if [ -n "$pmm_release" ]; then
-            sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-            grep -r 'define release' ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-        fi
+    if [ -n "$pmm_release" ]; then
+        sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+        grep -r 'define release' ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    fi
 
-        if [ -f "${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz" ]; then
-            echo SOURCES/${repo_name}-${short_commit}.tar.gz already exists, skip archiving...
-        else
-            git -C "${git_dir}" archive \
-                --format=tar.gz \
-                --prefix=${repo_name}-${full_commit}/ \
-                -o ${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz \
-                "${full_commit}"
-        fi
+    if [ -f "${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz" ]; then
+        echo SOURCES/${repo_name}-${short_commit}.tar.gz already exists, skip archiving...
+        return
     fi
+
+    git -C "${git_dir}" archive \
+        --format=tar.gz \
+        --prefix=${repo_name}-${full_commit}/ \
+        -o ${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz \
+        "${full_commit}"
 }
 
 build() {
     local spec_name=$1
     local repo_name=${2:-$1}
     prepare_specs "${spec_name}" "${repo_name}"
+
     local rpm_version=$(get_rpm_version "${spec_name}")
     local CH_VERSION=${rpm_version%-*}
     local CH_TAG=${rpm_version#*-}
+    local cache_dir=RELEASE
+    local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
+    local build_deps=("dbaas-tools.spec" "grafana.spec" "percona-dashboards.spec")
 
-    if  [[ -n "${FORCE_REBUILD}" ]] || is_build_needed "${spec_name}" "${rpm_version}"; then
-        echo "RPM Build Dir: ${rpmbuild_dir}, Docker image: ${rpmbuild_docker_image}"
-
-        docker run --rm -v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
-            set -o errexit
-            set -o xtrace
-            echo \"spec_name: ${spec_name}\"
-            echo \"repo_name: ${repo_name}\"
-            echo \"rpm_verison: ${rpm_version}\"
+    if ! is_build_needed "${spec_name}" "${rpm_version}"; then
+        return
+    fi
 
-            sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+    if [ -d "${root_dir}/go-build" ]; then
+        volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
+    fi
+    if [ -d "${root_dir}/go-path" ]; then
+        volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
+    fi
 
-            # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
-            if [ -n \"$RPM_EPOCH\" ]; then
-                sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
-            fi
+    echo "spec_name: ${spec_name}"
+    echo "repo_name: ${repo_name}"
+    echo "rpm_verison: ${rpm_version}"
 
-            rm -rf /home/builder/rpm/RPMS/${spec_name}-*
+    docker run --rm ${volume_mounts} ${rpmbuild_docker_image} sh -c "
+        set -o errexit
+        set -o xtrace
 
-            printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
-                | sudo tee /etc/yum.repos.d/local.repo
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
 
-            until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
-              echo waiting
-              sleep 1
-            done
+        # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
+        if [ \"$RPM_EPOCH\" = 1 ]; then
+            sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
+        fi
 
-            sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
+        rm -rf /home/builder/rpm/RPMS/${spec_name}-*
 
-            spectool -C SOURCES -g SOURCES/${spec_name}.spec
-            rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
-                      --define 'dist .${rpmbuild_dist}' \
-                      --define 'debug_package %{nil}' \
-                      -ba SOURCES/${spec_name}.spec
+        printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
+            | sudo tee /etc/yum.repos.d/local.repo
 
-            rm -f SOURCES/${spec_name}.spec*
-            sudo chown -R $(id -u):$(id -g) /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
-        "
+        until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
+          echo waiting
+          sleep 1
+        done
 
-        if [ -n "$RPM_EPOCH" ]; then
-            aws s3 sync \
-                --region us-east-2 \
-                ${rpms_dir}/${spec_name}-${rpm_version} \
-                s3://pmm-build-cache/PR-BUILDS/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-                || :
-        else
-            aws s3 sync \
-                --region us-east-2 \
-                ${rpms_dir}/${spec_name}-${rpm_version} \
-                s3://pmm-build-cache/RELEASE/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-                || :
+        if grep ${spec_name}.spec <<< ${build_deps[*]} &>/dev/null; then
+          sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
         fi
 
-        cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
+        spectool -C SOURCES -g SOURCES/${spec_name}.spec
+        rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
+                  --define 'dist .${rpmbuild_dist}' \
+                  --define 'debug_package %{nil}' \
+                  -ba SOURCES/${spec_name}.spec
+
+        rm -f SOURCES/${spec_name}.spec*
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+    "
+
+    # TODO: see if this is even needed
+    # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
+
+    if ! command -v aws &> /dev/null; then
+        echo "AWS CLI is not installed. Skipping upload to S3."
+        return
+    fi
 
+    if [ "$RPM_EPOCH" = 1 ]; then
+      cache_dir=PR-BUILDS
     fi
+    aws s3 sync \
+        --region us-east-2 \
+        ${rpms_dir}/${spec_name}-${rpm_version} \
+        s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+        || :
 }
 
 build "$1" "$2"
diff --git a/build/scripts/build-submodules b/build/scripts/build-submodules
index 34134e794b..e32f4b05a8 100755
--- a/build/scripts/build-submodules
+++ b/build/scripts/build-submodules
@@ -1,42 +1,30 @@
 #!/bin/bash
 set -o errexit
 
+git submodule update --init --jobs 10
+git submodule status
+
 if [ -s ci.yml ]; then
-    sudo rm -rf results tmp || :
     python3 ci.py
+    cat .git-sources
     . ./.git-sources
-    echo $pmm_commit > apiCommitSha
-    echo $pmm_branch > apiBranch
-    echo $pmm_url > apiURL
-    echo $pmm_qa_branch > pmmQABranch
-    echo $pmm_qa_commit > pmmQACommitSha
-    echo $pmm_ui_tests_branch > pmmUITestBranch
-    echo $pmm_ui_tests_commit > pmmUITestsCommitSha
 else
-    sudo rm -rf results tmp || :
-    git reset --hard
-    git clean -fdx
-    git submodule foreach --recursive git reset --hard
-    git submodule foreach --recursive git clean -fdx
-    git submodule status
-    export commit_sha=$(git submodule status | grep 'pmm-managed' | awk -F ' ' '{print $1}')
-    export api_tests_commit_sha=$(git submodule status | grep 'pmm' | awk -F ' ' '{print $1}')
-    export api_tests_branch=$(git config -f .gitmodules submodule.pmm.branch)
-    export api_tests_url=$(git config -f .gitmodules submodule.pmm.url)
-    echo $api_tests_commit_sha > apiCommitSha
-    echo $api_tests_branch > apiBranch
-    echo $api_tests_url > apiURL
-    cat apiBranch
-    cat apiURL
-    export pmm_qa_commit_sha=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
-    export pmm_qa_branch=$(git config -f .gitmodules submodule.pmm-qa.branch)
-    echo $pmm_qa_branch > pmmQABranch
-    echo $pmm_qa_commit_sha > pmmQACommitSha
-    export pmm_ui_tests_commit_sha=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
-    export pmm_ui_tests_branch=$(git config -f .gitmodules submodule.pmm-ui-tests.branch)
-    echo $pmm_ui_tests_branch > pmmUITestBranch
-    echo $pmm_ui_tests_commit_sha > pmmUITestsCommitSha
+    # Define variables
+    pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+    pmm_branch=$(git config -f .gitmodules submodule.pmm.branch)
+    pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+    pmm_qa_branch=$(git config -f .gitmodules submodule.pmm-qa.branch)
+    pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+    pmm_ui_tests_branch=$(git config -f .gitmodules submodule.pmm-ui-tests.branch)
+    pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
 fi
 
-export fb_commit_sha=$(git rev-parse HEAD)
+fb_commit_sha=$(git rev-parse HEAD)
 echo $fb_commit_sha > fbCommitSha
+echo $pmm_commit > apiCommitSha
+echo $pmm_branch > apiBranch
+echo $pmm_url > apiURL
+echo $pmm_qa_branch > pmmQABranch
+echo $pmm_qa_commit > pmmQACommitSha
+echo $pmm_ui_tests_branch > pmmUITestBranch
+echo $pmm_ui_tests_commit > pmmUITestsCommitSha
diff --git a/build/scripts/vars b/build/scripts/vars
index af854a8ffe..ea72294352 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -19,11 +19,7 @@ full_pmm_version=${pmm_base_version}-${pmm_branch}-$(git rev-parse --short HEAD)
 # Replace '/' with '-' to prevent sed from failing on dependabot-authored PRs
 full_pmm_version=${full_pmm_version//\//-}
 
-# TODO Maybe it makes sense to use variable from job here
-if [[ ${pmm_branch} =~ release-* || \
-      ${pmm_branch} =~ pmm-3.* || \
-      ${pmm_branch} == v3 || \
-      ${pmm_branch} == "main" ]]; then
+if [[ "${pmm_branch}" =~ ^pmm-3.|^v3$|^main$ ]]; then
     full_pmm_version=${pmm_base_version}
 fi
 

From 32c8fa8f90063b01efcda089c32d34b7b65eef4f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 01:41:46 +0000
Subject: [PATCH 02/78] PMM-12899 update grafana spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 85976d18a1..da293bbc1f 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -22,9 +22,6 @@ Source0:        https://github.com/percona/grafana/archive/%{commit}.tar.gz
 ExclusiveArch:  %{ix86} x86_64 %{arm}
 
 BuildRequires: fontconfig
-%if 0%{?rhel} < 9
-BuildRequires: nodejs-grunt-cli
-%endif
 
 %description
 Grafana is an open source, feature rich metrics dashboard and graph editor for
@@ -34,9 +31,7 @@ Graphite, InfluxDB & OpenTSDB.
 %setup -q -n grafana-%{commit}
 rm -rf Godeps
 sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go
-%if 0%{?rhel} >= 9
-    sudo npm install -g grunt-cli
-%endif
+sudo npm install -g grunt-cli
 
 %build
 mkdir -p _build/src
@@ -82,6 +77,9 @@ getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it
 exit 0
 
 %changelog
+* Tue Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-2
+- PMM-12899 Use module and build cache
+
 * Tue Mar 12 2024 Matej Kubinec <matej.kubinec@ext.percona.com> - 10.4.0-1
 - PMM-12991 Grafana 10.4.0
 

From 37e005a07ee7c730bbb6f3e42c55620f6522a184 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 02:11:03 +0000
Subject: [PATCH 03/78] PMM-12899 fix the build_command

---
 build/scripts/build-client-binary | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 295a5a5b02..f580e3f5d8 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -51,7 +51,7 @@ gobuild_component() {
     elif [ ${component} = 'percona-toolkit' ]; then
         build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     else
-        build_command="cd /tmp/go/src/${extract_path}/${component_path} go build -o ${bin_dir}/${result_file} ."
+        build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     fi
 
     docker run --rm \

From c863f50812f8c198d15610526f9f24c9026113ab Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 03:47:06 +0000
Subject: [PATCH 04/78] PMM-12899 fix builddeps

---
 build/local/build.sh                         |  3 ---
 build/packages/rpm/server/SPECS/grafana.spec |  2 +-
 build/scripts/build-client-packages          |  2 +-
 build/scripts/build-server-rpm               | 12 ++++++------
 4 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index f27ec569a8..938d12f720 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -82,15 +82,12 @@ export RPMBUILD_DIST="el9"
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
 "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-controller
-"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-tools
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
 "$PATH_TO_SCRIPTS/build-server-rpm" grafana-db-migrator
 "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
 # 3rd-party
 "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-"$PATH_TO_SCRIPTS/build-server-rpm" alertmanager
 "$PATH_TO_SCRIPTS/build-server-rpm" grafana
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index da293bbc1f..95d5b956f4 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -77,7 +77,7 @@ getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it
 exit 0
 
 %changelog
-* Tue Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-2
+* Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-2
 - PMM-12899 Use module and build cache
 
 * Tue Mar 12 2024 Matej Kubinec <matej.kubinec@ext.percona.com> - 10.4.0-1
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index 97865555b3..e7205ae7bb 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -437,7 +437,7 @@ build_deb(){
 #main
 
 CURDIR=$(pwd)
-PACKAGE_NAME=pmm3-client
+PACKAGE_NAME=pmm-client
 VERSION_FILE=$CURDIR/pmm-client.properties
 args=
 WORKDIR=
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 9be0b05f3c..295924c54f 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -104,7 +104,7 @@ build() {
     local CH_TAG=${rpm_version#*-}
     local cache_dir=RELEASE
     local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
-    local build_deps=("dbaas-tools.spec" "grafana.spec" "percona-dashboards.spec")
+    # local build_deps=("grafana.spec" "percona-dashboards.spec")
 
     if ! is_build_needed "${spec_name}" "${rpm_version}"; then
         return
@@ -128,7 +128,7 @@ build() {
         sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
-        if [ \"$RPM_EPOCH\" = 1 ]; then
+        if [ ${RPM_EPOCH} = 1 ]; then
             sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
         fi
 
@@ -138,12 +138,12 @@ build() {
             | sudo tee /etc/yum.repos.d/local.repo
 
         until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
-          echo waiting
-          sleep 1
+            echo waiting
+            sleep 1
         done
 
-        if grep ${spec_name}.spec <<< ${build_deps[*]} &>/dev/null; then
-          sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
+        if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
+            sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
         fi
 
         spectool -C SOURCES -g SOURCES/${spec_name}.spec

From 853be6ed9076267f039ed860dce8ef1c30168c42 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 04:07:25 +0000
Subject: [PATCH 05/78] PMM-12899 use rpmbuild:3

---
 build/local/build.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 938d12f720..03da4e51eb 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -43,7 +43,7 @@ test -d "${root_dir}/go-path" || mkdir -p "go-path"
 test -d "${root_dir}/go-build" || mkdir -p "go-build"
 
 PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
-export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9 
+export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
 # Local reference test environment
 # CPU: 4 cores
@@ -83,7 +83,6 @@ export RPMBUILD_DIST="el9"
 "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-"$PATH_TO_SCRIPTS/build-server-rpm" grafana-db-migrator
 "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
 # 3rd-party
@@ -91,7 +90,6 @@ export RPMBUILD_DIST="el9"
 "$PATH_TO_SCRIPTS/build-server-rpm" grafana
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
-export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9
 export RPMBUILD_DIST=el9
 export DOCKERFILE=Dockerfile.el9.local
 ${PATH_TO_SCRIPTS}/build-server-docker

From d57ec55633f133d0fc74f543eca0a67e708e93d8 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 26 Mar 2024 09:39:53 +0300
Subject: [PATCH 06/78] PMM-12641 Fix the dashboard upgrade (#2926)

* PMM-12641 Fix the dashboard upgrade

This fixes a bug where, due to a wrong ansible formula, PMM would upgrade the dashboards no matter what :)

* PMM-12641 follow up on review
---
 .../roles/initialization/tasks/main.yml       | 95 ++++++++++---------
 1 file changed, 50 insertions(+), 45 deletions(-)

diff --git a/build/ansible/roles/initialization/tasks/main.yml b/build/ansible/roles/initialization/tasks/main.yml
index be42baf2e8..4cb85312f7 100644
--- a/build/ansible/roles/initialization/tasks/main.yml
+++ b/build/ansible/roles/initialization/tasks/main.yml
@@ -27,56 +27,61 @@
 
 - name: Set need_upgrade fact
   set_fact:
-    need_upgrade: not pmm_current_version is version(pmm_image_version, '>=')
+    need_upgrade: "{{ pmm_current_version is version(pmm_image_version, '<') }}"
 
 - name: Print current PMM and image versions
   debug:
     msg: "Current version: {{ pmm_current_version }} Image Version: {{ pmm_image_version }}"
 
-- name: Enable maintenance mode before upgrade
-  copy:
-    src: maintenance.html
-    dest: /usr/share/pmm-server/maintenance/
-    owner: pmm
-    group: pmm
-    mode: 0644
-
-- name: Upgrade dashboards
-  include_role:
-    name: dashboards
-  when: need_upgrade
-
-- name: Copy file with image version
-  copy:
-    src: /usr/share/percona-dashboards/VERSION
-    dest: /srv/grafana/PERCONA_DASHBOARDS_VERSION
-    owner: pmm
-    group: pmm
-    mode: 0644
-    remote_src: yes
-  when: need_upgrade
-
-- name: Create a backup directory
-  file:
-    path: /srv/backup
-    state: directory
-    owner: pmm
-    group: pmm
-    mode: 0775
+- name: Print need_upgrade fact
+  debug:
+    msg: "Need upgrade: {{ need_upgrade }}"
 
-# Note: we want to leave this for some time until we achieve stable builds
-- name: Output pmm-managed logs
-  shell: sleep 10 && tail -n 300 /srv/logs/pmm-managed.log
+- name: Perform upgrade tasks
+  block:
+    - name: Enable maintenance mode before upgrade
+      copy:
+        src: maintenance.html
+        dest: /usr/share/pmm-server/maintenance/
+        owner: pmm
+        group: pmm
+        mode: 0644
+    
+    - name: Upgrade dashboards
+      include_role:
+        name: dashboards
+    
+    - name: Copy file with image version
+      copy:
+        src: /usr/share/percona-dashboards/VERSION
+        dest: /srv/grafana/PERCONA_DASHBOARDS_VERSION
+        owner: pmm
+        group: pmm
+        mode: 0644
+        remote_src: yes
+    
+    - name: Create a backup directory
+      file:
+        path: /srv/backup
+        state: directory
+        owner: pmm
+        group: pmm
+        mode: 0775
 
-- name: Wait for PMM to be ready
-  ansible.builtin.uri:
-    url: "http://127.0.0.1:7772/v1/readyz"
-    status_code: 200
-    method: GET
-  retries: 20
-  delay: 5
+    # Note: we want to leave this for some time until we achieve stable builds
+    - name: Output pmm-managed logs
+      shell: sleep 10 && tail -n 300 /srv/logs/pmm-managed.log
 
-- name: Disable maintenance mode
-  file:
-    state: absent
-    path: /usr/share/pmm-server/maintenance/maintenance.html
+    - name: Wait for PMM to be ready
+      ansible.builtin.uri:
+        url: "http://127.0.0.1:7772/v1/readyz"
+        status_code: 200
+        method: GET
+      retries: 20
+      delay: 5
+    
+    - name: Disable maintenance mode
+      file:
+        state: absent
+        path: /usr/share/pmm-server/maintenance/maintenance.html
+  when: need_upgrade

From 5c7821fc83520d31455ef6ad20a405f162a7669c Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 31 Mar 2024 23:03:42 +0000
Subject: [PATCH 07/78] PMM-12899 prevent grafana from using own GOPATH

---
 build/packages/rpm/server/SPECS/grafana.spec |  4 ++--
 build/scripts/build-server-rpm               | 15 +++++++++------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 95d5b956f4..a535cf69b1 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -34,8 +34,8 @@ sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go
 sudo npm install -g grunt-cli
 
 %build
-mkdir -p _build/src
-export GOPATH="$(pwd)/_build"
+# mkdir -p _build/src
+# export GOPATH="$(pwd)/_build"
 
 make build-go
 
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 295924c54f..6aa60e4388 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -104,7 +104,6 @@ build() {
     local CH_TAG=${rpm_version#*-}
     local cache_dir=RELEASE
     local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
-    # local build_deps=("grafana.spec" "percona-dashboards.spec")
 
     if ! is_build_needed "${spec_name}" "${rpm_version}"; then
         return
@@ -117,6 +116,7 @@ build() {
         volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
     fi
 
+    echo "Start building Server RPMs..."
     echo "spec_name: ${spec_name}"
     echo "repo_name: ${repo_name}"
     echo "rpm_verison: ${rpm_version}"
@@ -125,7 +125,7 @@ build() {
         set -o errexit
         set -o xtrace
 
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
         if [ ${RPM_EPOCH} = 1 ]; then
@@ -142,8 +142,10 @@ build() {
             sleep 1
         done
 
+        # Only these two specs have build dependencies
         if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
-            sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
+            sleep 5s
+            sudo yum-builddep -y SOURCES/${spec_name}.spec
         fi
 
         spectool -C SOURCES -g SOURCES/${spec_name}.spec
@@ -160,7 +162,7 @@ build() {
     # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
 
     if ! command -v aws &> /dev/null; then
-        echo "AWS CLI is not installed. Skipping upload to S3."
+        echo "AWS CLI is not installed. Upload to S3 will be skipped."
         return
     fi
 
@@ -172,9 +174,10 @@ build() {
         ${rpms_dir}/${spec_name}-${rpm_version} \
         s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
         || :
+
+    echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
 }
 
-build "$1" "$2"
-echo "build-server-rpm finished, spec_name: $1, repo_name: ${2:-$1}"
+build "$@"
 
 # vim: expandtab shiftwidth=4 tabstop=4

From 42a02a9217d204fac5d490e808b39ae9878c8cbc Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 31 Mar 2024 23:52:37 +0000
Subject: [PATCH 08/78] PMM-12899 use a dedicated folder for yarn cache

---
 build/local/build.sh           | 1 +
 build/scripts/build-server-rpm | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/build/local/build.sh b/build/local/build.sh
index 03da4e51eb..cba72f27df 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -41,6 +41,7 @@ echo $fb_commit_sha > fbCommitSha
 # Create cache directories. 
 test -d "${root_dir}/go-path" || mkdir -p "go-path"
 test -d "${root_dir}/go-build" || mkdir -p "go-build"
+test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
 
 PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
 export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 6aa60e4388..06ae02326e 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -112,6 +112,9 @@ build() {
     if [ -d "${root_dir}/go-build" ]; then
         volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
     fi
+    if [ -d "${root_dir}/yarn-cache" ]; then
+        volume_mounts+=" -v ${root_dir}/yarn-cache:/home/builder/.yarn"
+    fi
     if [ -d "${root_dir}/go-path" ]; then
         volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
     fi

From 6d0b02bdcab1d18ef4a91591a31683c8270ebc00 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 01:19:42 +0000
Subject: [PATCH 09/78] PMM-12899 extract server rpm script

---
 build/local/build.sh                  | 82 ++++++++++++++++++++-------
 build/scripts/build-rpmbuild-docker   | 11 ----
 build/scripts/build-server-docker     | 18 +++---
 build/scripts/build-server-rpm        | 12 ++--
 build/scripts/build-server-rpm-script | 40 +++++++++++++
 5 files changed, 115 insertions(+), 48 deletions(-)
 delete mode 100755 build/scripts/build-rpmbuild-docker
 create mode 100755 build/scripts/build-server-rpm-script

diff --git a/build/local/build.sh b/build/local/build.sh
index cba72f27df..80caaa0d2d 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -1,7 +1,38 @@
 #!/bin/bash -e
 
-# Update submodules and PR branches
-/bin/bash $(dirname $0)/update.sh
+# Define global variables
+NO_UPDATE=0
+NO_CLIENT=0
+NO_CLIENT_DOCKER=0
+NO_SERVER_RPM=0
+
+while test "$#" -gt 0; do
+  case "$1" in
+    --no-update)
+      NO_UPDATE=1
+      ;;
+    --no-client)
+      NO_CLIENT=1
+      ;;
+    --no-client-docker)
+      NO_CLIENT_DOCKER=1
+      ;;
+    --no-server-rpm)
+      NO_SERVER_RPM=1
+      ;;
+    *)
+      echo "Unknown argument: $1"
+      echo "Usage: $0 [--no-update] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+if [ "$NO_UPDATE" -eq 0 ]; then
+  # Update submodules and PR branches
+  /bin/bash $(dirname $0)/update.sh
+fi
 
 get_branch_name() {
   local module="${1:-}"
@@ -33,7 +64,7 @@ echo $pmm_ui_tests_commit > pmmUITestsCommitSha
 fb_commit_sha=$(git rev-parse HEAD)
 echo $fb_commit_sha > fbCommitSha
 
-# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:ol9`.
+# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
 # Important: the docker container's user need to be able to write to these directories.
 # The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
 # that the directories we create on the host are owned by a user with same uid and gid.
@@ -51,22 +82,26 @@ export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 # RAM: 16GB
 # OS: Ubuntu 22.04.1 LTS
 
-# Build client source: 4m39s from scratch, 0m27s using cache
-"$PATH_TO_SCRIPTS/build-client-source"
+if [ "$NO_CLIENT" -eq 0 ]; then
+  # Build client source: 4m39s from scratch, 0m27s using cache
+  "$PATH_TO_SCRIPTS/build-client-source"
 
-# Build client binary: ??? from scratch, 0m20s using cache
-"$PATH_TO_SCRIPTS/build-client-binary"
+  # Build client binary: ??? from scratch, 0m20s using cache
+  "$PATH_TO_SCRIPTS/build-client-binary"
 
-# Building client source rpm takes 13s (caching does not apply)
-"$PATH_TO_SCRIPTS/build-client-srpm"
+  # Building client source rpm takes 13s (caching does not apply)
+  "$PATH_TO_SCRIPTS/build-client-srpm"
 
-# Building client rpm takes 1m40s
-"$PATH_TO_SCRIPTS/build-client-rpm"
+  # Building client rpm takes 1m40s
+  "$PATH_TO_SCRIPTS/build-client-rpm"
+fi
 
 # Building client docker image takes 17s
 GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
 export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
-"$PATH_TO_SCRIPTS/build-client-docker"
+if [ "$NO_CLIENT_DOCKER" -eq 0 ] || [ "$NO_CLIENT" -eq 0 ]; then
+  "$PATH_TO_SCRIPTS/build-client-docker"
+fi
 
 # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
 # total time: 6m26s - build from scratch, no initial cache
@@ -79,20 +114,23 @@ export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
 
 export RPM_EPOCH=1
 export RPMBUILD_DIST="el9"
-"$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
-"$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-"$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
-# 3rd-party
-"$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-"$PATH_TO_SCRIPTS/build-server-rpm" grafana
+if [ "$NO_SERVER_RPM" -eq 0 ]; then
+  "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
+  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
+  "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
+  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
+  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
+  "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+
+  # 3rd-party
+  "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
+  "$PATH_TO_SCRIPTS/build-server-rpm" grafana
+fi
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
 export RPMBUILD_DIST=el9
-export DOCKERFILE=Dockerfile.el9.local
+export DOCKERFILE=Dockerfile.el9
 ${PATH_TO_SCRIPTS}/build-server-docker
 
 # Clean up temporary files
diff --git a/build/scripts/build-rpmbuild-docker b/build/scripts/build-rpmbuild-docker
deleted file mode 100755
index 77706f3621..0000000000
--- a/build/scripts/build-rpmbuild-docker
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o xtrace
-
-. $(dirname $0)/vars
-
-root_dir=$(cd $(dirname $0)/../..; pwd -P)
-docker build --squash -f ${docker_file} -t rpmbuild:local ${root_dir}/build/docker/rpmbuild/
-
-# vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 421a4dd688..8cb2f70bfc 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -32,15 +32,15 @@ if [ -z "${DOCKER_TAG}" ]; then
     DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
 fi
 
-IMAGE_VERSION=`echo $DOCKER_TAG | cut -d ':' -f2`
-
-docker build --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
-             --build-arg VERSION="$IMAGE_VERSION" \
-             --squash \
-             --no-cache \
-             -f ${docker_root}/${docker_file} \
-             -t ${DOCKER_TAG} \
-             ${docker_root}/
+IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+
+docker buildx build \
+    --build-arg BUILD_DATE=$(date --rfc-3339=seconds) \
+    --build-arg VERSION="$IMAGE_VERSION" \
+    --progress plain \
+    -f ${docker_root}/${docker_file} \
+    -t ${DOCKER_TAG} \
+    ${docker_root}
 
 if [ -n "${PUSH_DOCKER}" ]; then
     mkdir -p $(dirname ${docker_tag_file})
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 06ae02326e..031aa7707e 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -24,7 +24,7 @@ is_build_needed() {
     local spec_name=$1
     local rpm_version=$2
     local packages=
-    local cache_dir=RELEASE
+    local s3_cache_dir=RELEASE
 
     # Structure of S3 build cache
     # s3://pmm-build-cache/PR-BUILDS/7 - el7
@@ -36,13 +36,13 @@ is_build_needed() {
 
     # RPM_EPOCH is set to 1 for Feature Builds
     if [ "$RPM_EPOCH" = 1 ]; then
-        cache_dir=PR-BUILDS
+        s3_cache_dir=PR-BUILDS
     fi
     if command -v aws &> /dev/null; then
         aws s3 sync \
             --region us-east-2 \
             --no-sign-request \
-            s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+            s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
             ${rpms_dir}/${spec_name}-${rpm_version}
     fi
 
@@ -102,7 +102,7 @@ build() {
     local rpm_version=$(get_rpm_version "${spec_name}")
     local CH_VERSION=${rpm_version%-*}
     local CH_TAG=${rpm_version#*-}
-    local cache_dir=RELEASE
+    local s3_cache_dir=RELEASE
     local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
 
     if ! is_build_needed "${spec_name}" "${rpm_version}"; then
@@ -170,12 +170,12 @@ build() {
     fi
 
     if [ "$RPM_EPOCH" = 1 ]; then
-      cache_dir=PR-BUILDS
+      s3_cache_dir=PR-BUILDS
     fi
     aws s3 sync \
         --region us-east-2 \
         ${rpms_dir}/${spec_name}-${rpm_version} \
-        s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+        s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
         || :
 
     echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
diff --git a/build/scripts/build-server-rpm-script b/build/scripts/build-server-rpm-script
new file mode 100755
index 0000000000..cece634cdc
--- /dev/null
+++ b/build/scripts/build-server-rpm-script
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -o errexit
+set -o xtrace
+
+. $(dirname $0)/vars
+
+sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache
+
+# Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
+if [ ${RPM_EPOCH} = 1 ]; then
+    sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
+fi
+
+rm -rf /home/builder/rpm/RPMS/${spec_name}-*
+
+printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
+    | sudo tee /etc/yum.repos.d/local.repo
+
+until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
+    echo waiting
+    sleep 1
+done
+
+# Only these two specs have build dependencies
+if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
+    sleep 5s
+    sudo yum-builddep -y SOURCES/${spec_name}.spec
+fi
+
+spectool -C SOURCES -g SOURCES/${spec_name}.spec
+rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
+          --define 'dist .${rpmbuild_dist}' \
+          --define 'debug_package %{nil}' \
+          -ba SOURCES/${spec_name}.spec
+
+rm -f SOURCES/${spec_name}.spec*
+sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+
+# vim: expandtab shiftwidth=4 tabstop=4

From 098e894628c160cd4805ac5f5feb7bdf743ac024 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 01:31:54 +0000
Subject: [PATCH 10/78] PMM-12899 fix server param evaluation

---
 build/scripts/build-server-docker | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 8cb2f70bfc..16027c3beb 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -35,7 +35,7 @@ fi
 IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
 
 docker buildx build \
-    --build-arg BUILD_DATE=$(date --rfc-3339=seconds) \
+    --build-arg BUILD_DATE="$(date --rfc-3339=seconds)" \
     --build-arg VERSION="$IMAGE_VERSION" \
     --progress plain \
     -f ${docker_root}/${docker_file} \

From aa5729fb413c226baaa908cb05ac7ab19dd69e52 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 20:08:11 +0000
Subject: [PATCH 11/78] PMM-12899 clean up server RPM specs

---
 build/local/build.sh                          |  6 +--
 build/packages/rpm/server/SPECS/grafana.spec  | 15 +++----
 .../rpm/server/SPECS/percona-dashboards.spec  | 44 ++++++++++--------
 .../rpm/server/SPECS/percona-qan-api2.spec    | 41 +++++++++--------
 build/packages/rpm/server/SPECS/pmm-dump.spec | 16 ++++---
 .../rpm/server/SPECS/pmm-managed.spec         | 45 ++++++++++---------
 .../packages/rpm/server/SPECS/pmm-update.spec | 32 ++++++-------
 .../rpm/server/SPECS/victoriametrics.spec     | 20 ++++-----
 build/packages/rpm/server/SPECS/vmproxy.spec  | 44 +++++++++---------
 build/scripts/vars                            |  2 +-
 10 files changed, 139 insertions(+), 126 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 80caaa0d2d..bd174fb965 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -12,7 +12,7 @@ while test "$#" -gt 0; do
       NO_UPDATE=1
       ;;
     --no-client)
-      NO_CLIENT=1
+      NO_CLIENT=1; NO_CLIENT_DOCKER=1
       ;;
     --no-client-docker)
       NO_CLIENT_DOCKER=1
@@ -99,7 +99,7 @@ fi
 # Building client docker image takes 17s
 GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
 export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
-if [ "$NO_CLIENT_DOCKER" -eq 0 ] || [ "$NO_CLIENT" -eq 0 ]; then
+if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
   "$PATH_TO_SCRIPTS/build-client-docker"
 fi
 
@@ -113,7 +113,6 @@ fi
 # total time: ??? - subsequent build, using cache from prior builds
 
 export RPM_EPOCH=1
-export RPMBUILD_DIST="el9"
 
 if [ "$NO_SERVER_RPM" -eq 0 ]; then
   "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
@@ -129,7 +128,6 @@ if [ "$NO_SERVER_RPM" -eq 0 ]; then
 fi
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
-export RPMBUILD_DIST=el9
 export DOCKERFILE=Dockerfile.el9
 ${PATH_TO_SCRIPTS}/build-server-docker
 
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index a535cf69b1..11e7b246a0 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -1,15 +1,13 @@
-%global debug_package   %{nil}
 %global commit          f283b87d773da4ad61ca65660e72c084ac8f8716
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         104
+%define release         105
 %define grafana_version 10.4.0
 %define full_pmm_version 2.0.0
 %define full_version    v%{grafana_version}-%{full_pmm_version}
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 %if ! 0%{?gobuild:1}
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom|od -An -tx1|tr -d ' \\n')" -a -v -x %{?**};
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
 %endif
 
 Name:           percona-grafana
@@ -24,8 +22,9 @@ ExclusiveArch:  %{ix86} x86_64 %{arm}
 BuildRequires: fontconfig
 
 %description
-Grafana is an open source, feature rich metrics dashboard and graph editor for
-Graphite, InfluxDB & OpenTSDB.
+Grafana is an open source observability and data visualization platform.
+Visualize metrics, logs, and traces from multiple sources like
+Prometheus, Loki, Elasticsearch, InfluxDB, Postgres and many more.
 
 %prep
 %setup -q -n grafana-%{commit}
@@ -77,7 +76,7 @@ getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it
 exit 0
 
 %changelog
-* Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-2
+* Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-105
 - PMM-12899 Use module and build cache
 
 * Tue Mar 12 2024 Matej Kubinec <matej.kubinec@ext.percona.com> - 10.4.0-1
diff --git a/build/packages/rpm/server/SPECS/percona-dashboards.spec b/build/packages/rpm/server/SPECS/percona-dashboards.spec
index 5a1a895ed3..e356038179 100644
--- a/build/packages/rpm/server/SPECS/percona-dashboards.spec
+++ b/build/packages/rpm/server/SPECS/percona-dashboards.spec
@@ -1,27 +1,30 @@
-%global debug_package   %{nil}
 %global __strip         /bin/true
 
-%global repo		grafana-dashboards
-%global provider	github.com/percona/%{repo}
-%global import_path	%{provider}
-%global commit		ad4af6808bcd361284e8eb8cd1f36b1e98e32bce
-%global shortcommit	%(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         21
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
-
-Name:		percona-dashboards
-Version:	%{version}
-Release:	%{rpm_release}
-Summary:	Grafana dashboards for monitoring
-
-License:	AGPLv3
-URL:		https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+%global repo            grafana-dashboards
+%global provider        github.com/percona/%{repo}
+%global import_path     %{provider}
+%global commit          ad4af6808bcd361284e8eb8cd1f36b1e98e32bce
+%global shortcommit     %(c=%{commit}; echo ${c:0:7})
+%define release         22
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
+
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
+
+Name:     percona-dashboards
+Version:  %{version}
+Release:  %{rpm_release}
+Summary:  Grafana dashboards for monitoring
+
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 BuildRequires:	nodejs
 Requires:	percona-grafana
-Provides:	percona-grafana-dashboards = %{version}-%{release}
 
 %description
 This is a set of Grafana dashboards for database and system monitoring
@@ -57,6 +60,9 @@ echo %{version} > %{buildroot}%{_datadir}/%{name}/VERSION
 
 
 %changelog
+* Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-22
+- PMM-12899 Use module and build cache
+
 * Wed Nov 29 2023 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-21
 - PMM-12693 Run Grafana as non-root user
 
diff --git a/build/packages/rpm/server/SPECS/percona-qan-api2.spec b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
index 14e89677b5..ae33484d71 100644
--- a/build/packages/rpm/server/SPECS/percona-qan-api2.spec
+++ b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
@@ -1,8 +1,3 @@
-# Go build id is not supported for now.
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://bugzilla.redhat.com/show_bug.cgi?id=1295951
-%undefine _missing_build_ids_terminate_build
-
 %global repo            pmm
 %global provider        github.com/percona/%{repo}
 %global import_path     %{provider}
@@ -10,17 +5,22 @@
 # see: https://github.com/percona/pmm/blob/main/build/scripts/build-server-rpm#L58
 %global commit          0000000000000000000000000000000000000000
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         17
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         18
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:           percona-qan-api2
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
+
+Name:           pmm-qan
 Version:        %{version}
 Release:        %{rpm_release}
-Summary:        Query Analytics API v2 for PMM
+Summary:        Query Analytics API for PMM
 
 License:        AGPLv3
 URL:            https://%{provider}
@@ -32,10 +32,7 @@ See PMM docs for more information - https://docs.percona.com/percona-monitoring-
 
 
 %prep
-%setup -T -c -n %{repo}-%{version}
-%setup -q -c -a 0 -n %{repo}-%{version}
-mkdir -p src/github.com/percona
-mv %{repo}-%{commit} src/%{provider}
+%setup -q -n %{repo}-%{commit}
 
 
 %build
@@ -43,29 +40,31 @@ export PMM_RELEASE_VERSION=%{full_pmm_version}
 export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
-cd src/%{provider}/qan-api2
-make release
+make -C qan-api2 release
 
 
 %install
 
 install -d -p %{buildroot}%{_sbindir}
-install -p -m 0755 src/%{provider}/bin/qan-api2 %{buildroot}%{_sbindir}/%{name}
+install -p -m 0755 ./bin/qan-api2 %{buildroot}%{_sbindir}/%{name}
 
 
 %files
 %attr(0755, root, root) %{_sbindir}/%{name}
-%license src/%{provider}/qan-api2/LICENSE
-%doc src/%{provider}/qan-api2/README.md
+%license qan-api2/LICENSE
+%doc qan-api2/README.md
 
 %changelog
-* Mon Nov  7 2022 Alexander Tymchuk <alexander.tymchuk@percona.com> - 2.0.0-17
+* Wed Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-18
+- PMM-12899 Use module and build cache
+
+* Mon Nov 7 2022 Alexander Tymchuk <alexander.tymchuk@percona.com> - 2.0.0-17
 - PMM-10117 migrate QAN API to monorepo
 
 * Mon May 16 2022 Nikita Beletskii <nikita.beletskii@percona.com> - 2.0.0-16
 - PMM-10027 remove useless packages
 
-* Thu Jul  2 2020 Mykyta Solomko <mykyta.solomko@percona.com> - 2.0.0-15
+* Thu Jul 2 2020 Mykyta Solomko <mykyta.solomko@percona.com> - 2.0.0-15
 - PMM-5645 built using Golang 1.14
 
 * Tue Mar 19 2019 Vadim Yalovets <vadim.yalovets@percona.com> - 2.0.0-4
diff --git a/build/packages/rpm/server/SPECS/pmm-dump.spec b/build/packages/rpm/server/SPECS/pmm-dump.spec
index c8882aa25b..e5ddbee367 100644
--- a/build/packages/rpm/server/SPECS/pmm-dump.spec
+++ b/build/packages/rpm/server/SPECS/pmm-dump.spec
@@ -1,12 +1,15 @@
-%undefine _missing_build_ids_terminate_build
-
 %global repo            pmm-dump
 %global provider        github.com/percona/%{repo}
 %global commit          0d49b27729506dc62950f9fa59147d63df194db2
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         1
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         2
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
+
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
 
 Name:		pmm-dump
 Version:	0.7.0
@@ -37,6 +40,9 @@ install -p -m 0755 pmm-dump %{buildroot}%{_sbindir}/pmm-dump
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 0.7.0-2
+- PMM-12899 Use module and build cache
+
 * Tue Nov 23 2023 Artem Gavrilov <artem.gavrilov@percona.com> - 0.7.0-ga
 - PMM-12460 Update pmm-dump to v0.7.0-ga version
 
diff --git a/build/packages/rpm/server/SPECS/pmm-managed.spec b/build/packages/rpm/server/SPECS/pmm-managed.spec
index b313be8ba8..d9cc53019c 100644
--- a/build/packages/rpm/server/SPECS/pmm-managed.spec
+++ b/build/packages/rpm/server/SPECS/pmm-managed.spec
@@ -1,25 +1,29 @@
-%undefine _missing_build_ids_terminate_build
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
 %global provider        github.com/percona/%{repo}
 %global commit          8f3d007617941033867aea6a134c48b39142427f
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         20
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         21
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:		pmm-managed
-Version:	%{version}
-Release:	%{rpm_release}
-Summary:	Percona Monitoring and Management management daemon
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
 
-License:	AGPLv3
-URL:		  https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+Name:     pmm-managed
+Version:  %{version}
+Release:  %{rpm_release}
+Summary:  Percona Monitoring and Management management daemon
+
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 %description
 pmm-managed manages configuration of PMM server components (VictoriaMetrics,
@@ -28,19 +32,14 @@ See PMM docs for more information.
 
 
 %prep
-%setup -q -n pmm-%{commit}
-mkdir -p src/github.com/percona
-ln -s $(pwd) src/%{provider}
-
+%setup -q -n %{repo}-%{commit}
 
 %build
-
 export PMM_RELEASE_VERSION=%{full_pmm_version}
 export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
-cd src/github.com/percona/pmm/managed
-make release
+make -C managed release
 
 
 %install
@@ -51,13 +50,12 @@ install -p -m 0755 bin/pmm-managed %{buildroot}%{_sbindir}/pmm-managed
 install -p -m 0755 bin/pmm-managed-init %{buildroot}%{_sbindir}/pmm-managed-init
 install -p -m 0755 bin/pmm-managed-starlark %{buildroot}%{_sbindir}/pmm-managed-starlark
 
-cd src/github.com/percona/pmm
-cp -pa ./api/swagger %{buildroot}%{_datadir}/%{name}
+cp -pa api/swagger %{buildroot}%{_datadir}/%{name}
 
 
 %files
-%license src/%{provider}/LICENSE
-%doc src/%{provider}/README.md
+%license managed/LICENSE
+%doc managed/README.md
 %{_sbindir}/pmm-managed
 %{_sbindir}/pmm-managed-init
 %{_sbindir}/pmm-managed-starlark
@@ -65,6 +63,9 @@ cp -pa ./api/swagger %{buildroot}%{_datadir}/%{name}
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-21
+- PMM-12899 Use module and build cache
+
 * Thu Jul 28 2022 Alex Tymchuk <alexander.tymchuk@percona.com> - 2.30.0-1
 - PMM-10036 migrate to monorepo
 
diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index e68fc9f55d..9a9a5eac21 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -1,5 +1,3 @@
-%undefine _missing_build_ids_terminate_build
-
 # TODO: remove it as soon as we remove all noarch pmm-update rpms
 # from 'pmm3-components/yum/laboratory'
 %define _binaries_in_noarch_packages_terminate_build   0
@@ -10,20 +8,25 @@
 %global import_path     %{provider}
 %global commit	        592eddf656bce32a11bd958af0a32c62bd5ea34c
 %global shortcommit	    %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         67
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         68
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:		pmm-update
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
+
+Name:		  pmm-update
 Version:	%{full_pmm_version}
 Release:	%{rpm_release}
 Summary:	Tool for updating packages and OS configuration for PMM Server
 
 License:	AGPLv3
-URL:		https://%{provider}
+URL:		  https://%{provider}
 Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 BuildArch:	noarch
@@ -34,16 +37,13 @@ BuildArch:	noarch
 
 %prep
 %setup -q -n %{repo}-%{commit}
-mkdir -p src/github.com/percona
-ln -s $(pwd) src/%{provider}
 
 %build
 export PMM_RELEASE_VERSION=%{full_pmm_version}
 export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
-cd src/github.com/percona/pmm/update
-make release
+make -C update release
 
 
 %install
@@ -51,18 +51,20 @@ install -d %{buildroot}%{_datadir}/%{name}
 cp -pav ./update/ansible %{buildroot}%{_datadir}/%{name}
 
 install -d %{buildroot}%{_sbindir}
-cd src/github.com/percona/pmm/update
-install -p -m 0755 bin/pmm-update %{buildroot}%{_sbindir}/
+install -p -m 0755 update/bin/pmm-update %{buildroot}%{_sbindir}/
 
 
 %files
-%license LICENSE
-%doc README.md
+%license update/LICENSE
+%doc update/README.md
 %{_sbindir}/pmm-update
 %{_datadir}/%{name}
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-68
+- PMM-12899 Use module and build cache
+
 * Thu Dec 8 2022 Michal Kralik <michal.kralik@percona.com> - 2.34.0-67
 - PMM-11207 Migrate pmm-update to monorepo
 
diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec
index ec55d239d1..585a35954a 100644
--- a/build/packages/rpm/server/SPECS/victoriametrics.spec
+++ b/build/packages/rpm/server/SPECS/victoriametrics.spec
@@ -1,19 +1,16 @@
-%undefine _missing_build_ids_terminate_build
-
-%define copying() \
-%if 0%{?fedora} >= 21 || 0%{?rhel} >= 7 \
-%license %{*} \
-%else \
-%doc %{*} \
-%endif
-
 %global repo            VictoriaMetrics
 %global provider        github.com/VictoriaMetrics/%{repo}
 %global commit          pmm-6401-v1.93.4
 
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
+
 Name:           percona-victoriametrics
 Version:        1.93.4
-Release:        1%{?dist}
+Release:        2%{?dist}
 Summary:        VictoriaMetrics monitoring solution and time series database
 License:        Apache-2.0
 URL:            https://%{provider}
@@ -50,6 +47,9 @@ install -D -p -m 0755 ./bin/vmalert-pure %{buildroot}%{_sbindir}/vmalert
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 1.93.4-2
+- PMM-12899 Use module and build cache
+
 * Thu Sep 14 2023 Alex Tymchuk <alexander.tymchuk@percona.com> - 1.93.4-1
 - upgrade victoriametrics to 1.93.4 release
 
diff --git a/build/packages/rpm/server/SPECS/vmproxy.spec b/build/packages/rpm/server/SPECS/vmproxy.spec
index be5cdca4ca..fe4100e2d8 100644
--- a/build/packages/rpm/server/SPECS/vmproxy.spec
+++ b/build/packages/rpm/server/SPECS/vmproxy.spec
@@ -1,25 +1,29 @@
-%undefine _missing_build_ids_terminate_build
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
 %global provider        github.com/percona/%{repo}
 %global commit          8f74cea10d85e441ee88ef4b12bc47bc05165ba9
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         1
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         2
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:		vmproxy
-Version:	%{full_pmm_version}
-Release:	%{rpm_release}
-Summary:	Percona VMProxy stateless reverse proxy for VictoriaMetrics
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
 
-License:	AGPLv3
-URL:		https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+Name:     vmproxy
+Version:  %{full_pmm_version}
+Release:  %{rpm_release}
+Summary:  Percona VMProxy stateless reverse proxy for VictoriaMetrics
+
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 %description
 VMProxy is a stateless reverse proxy which proxies requests to VictoriaMetrics and
@@ -27,33 +31,31 @@ optionally adds `extra_filters` query based on the provided configuration.
 
 
 %prep
-%setup -q -n pmm-%{commit}
-mkdir -p src/github.com/percona
-ln -s $(pwd) src/%{provider}
+%setup -q -n %{repo}-%{commit}
 
 
 %build
-
 export PMM_RELEASE_VERSION=%{full_pmm_version}
 export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
-cd src/github.com/percona/pmm/vmproxy
-make release
+make -C vmproxy release
 
 
 %install
-install -d -p %{buildroot}%{_bindir}
 install -d -p %{buildroot}%{_sbindir}
-install -p -m 0755 bin/vmproxy %{buildroot}%{_sbindir}/vmproxy
+install -p -m 0755 vmproxy/bin/vmproxy %{buildroot}%{_sbindir}/vmproxy
 
 
 %files
-%license src/%{provider}/vmproxy/LICENSE
-%doc src/%{provider}/vmproxy/README.md
+%license vmproxy/LICENSE
+%doc vmproxy/README.md
 %{_sbindir}/vmproxy
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-2
+- PMM-12899 Use module and build cache
+
 * Mon Dec 5 2022 Michal Kralik <michal.kralik@percona.com> - 2.34.0-1
 - Initial release of VMProxy
diff --git a/build/scripts/vars b/build/scripts/vars
index ea72294352..de57bfb8a4 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -41,7 +41,7 @@ echo -e "\n\n\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_versio
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:2}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
 rpmbuild_dir=${root_dir}/sources/pmm/src/github.com/percona/pmm/build/packages/rpm/server
-rpmbuild_dist=${RPMBUILD_DIST:-"el7"}
+rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
 source_dir=${root_dir}/tmp/source/pmm-client-${pmm_version}
 binary_dir=${root_dir}/tmp/binary/pmm-client-${pmm_version}
 client_properties=${root_dir}/results/pmm-client.properties

From aa5e8b82cd3cfa621be875d28df3ee6ab05d5df7 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 20:16:41 +0000
Subject: [PATCH 12/78] PMM-12899 fix directory permissions

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 031aa7707e..ff5ab06553 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -128,7 +128,7 @@ build() {
         set -o errexit
         set -o xtrace
 
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache /home/builder/go
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
         if [ ${RPM_EPOCH} = 1 ]; then

From ee45fd6aadb70d8bc59d4aa5b9eee695b4723572 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 21:49:29 +0000
Subject: [PATCH 13/78] PMM-12899 remove unnecessary macro

---
 build/packages/rpm/server/SPECS/grafana.spec  |  2 ++
 .../rpm/server/SPECS/percona-dashboards.spec  |  6 -----
 .../rpm/server/SPECS/percona-qan-api2.spec    |  6 -----
 build/packages/rpm/server/SPECS/pmm-dump.spec | 24 ++++++++-----------
 .../rpm/server/SPECS/pmm-managed.spec         |  7 +-----
 .../packages/rpm/server/SPECS/pmm-update.spec |  7 +-----
 .../rpm/server/SPECS/victoriametrics.spec     |  8 ++-----
 build/packages/rpm/server/SPECS/vmproxy.spec  |  9 ++-----
 8 files changed, 18 insertions(+), 51 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 11e7b246a0..e4c3e1558d 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -7,6 +7,8 @@
 %define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 %if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
 %define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
 %endif
 
diff --git a/build/packages/rpm/server/SPECS/percona-dashboards.spec b/build/packages/rpm/server/SPECS/percona-dashboards.spec
index e356038179..d52adfdddf 100644
--- a/build/packages/rpm/server/SPECS/percona-dashboards.spec
+++ b/build/packages/rpm/server/SPECS/percona-dashboards.spec
@@ -8,12 +8,6 @@
 %define release         22
 %define rpm_release     %{release}.%{shortcommit}%{?dist}
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:     percona-dashboards
 Version:  %{version}
 Release:  %{rpm_release}
diff --git a/build/packages/rpm/server/SPECS/percona-qan-api2.spec b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
index ae33484d71..46995118b1 100644
--- a/build/packages/rpm/server/SPECS/percona-qan-api2.spec
+++ b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
@@ -11,12 +11,6 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:           pmm-qan
 Version:        %{version}
 Release:        %{rpm_release}
diff --git a/build/packages/rpm/server/SPECS/pmm-dump.spec b/build/packages/rpm/server/SPECS/pmm-dump.spec
index e5ddbee367..b5b4edad32 100644
--- a/build/packages/rpm/server/SPECS/pmm-dump.spec
+++ b/build/packages/rpm/server/SPECS/pmm-dump.spec
@@ -1,3 +1,5 @@
+%undefine _missing_build_ids_terminate_build
+
 %global repo            pmm-dump
 %global provider        github.com/percona/%{repo}
 %global commit          0d49b27729506dc62950f9fa59147d63df194db2
@@ -5,20 +7,14 @@
 %define release         2
 %define rpm_release     %{release}.%{shortcommit}%{?dist}
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
-Name:		pmm-dump
-Version:	0.7.0
-Release:	%{rpm_release}
-Summary:	Percona PMM Dump allows to export and import monitoring metrics and query analytics.
+Name:     pmm-dump
+Version:  0.7.0
+Release:  %{rpm_release}
+Summary:  Percona PMM Dump allows to export and import monitoring metrics and query analytics.
 
-License:	AGPLv3
-URL:		https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}.tar.gz
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}.tar.gz
 
 %description
 %{summary}
@@ -43,7 +39,7 @@ install -p -m 0755 pmm-dump %{buildroot}%{_sbindir}/pmm-dump
 * Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 0.7.0-2
 - PMM-12899 Use module and build cache
 
-* Tue Nov 23 2023 Artem Gavrilov <artem.gavrilov@percona.com> - 0.7.0-ga
+* Thu Nov 23 2023 Artem Gavrilov <artem.gavrilov@percona.com> - 0.7.0-ga
 - PMM-12460 Update pmm-dump to v0.7.0-ga version
 
 * Tue Mar 29 2022 Alex Tymchuk <alexander.tymchuk@percona.com> - 0.6.0-1
diff --git a/build/packages/rpm/server/SPECS/pmm-managed.spec b/build/packages/rpm/server/SPECS/pmm-managed.spec
index d9cc53019c..02f1578043 100644
--- a/build/packages/rpm/server/SPECS/pmm-managed.spec
+++ b/build/packages/rpm/server/SPECS/pmm-managed.spec
@@ -1,3 +1,4 @@
+%undefine _missing_build_ids_terminate_build
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
@@ -10,12 +11,6 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:     pmm-managed
 Version:  %{version}
 Release:  %{rpm_release}
diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index 9a9a5eac21..cb048070c3 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -1,3 +1,4 @@
+%undefine _missing_build_ids_terminate_build
 # TODO: remove it as soon as we remove all noarch pmm-update rpms
 # from 'pmm3-components/yum/laboratory'
 %define _binaries_in_noarch_packages_terminate_build   0
@@ -14,12 +15,6 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:		  pmm-update
 Version:	%{full_pmm_version}
 Release:	%{rpm_release}
diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec
index 585a35954a..97e2b39a78 100644
--- a/build/packages/rpm/server/SPECS/victoriametrics.spec
+++ b/build/packages/rpm/server/SPECS/victoriametrics.spec
@@ -1,13 +1,9 @@
+%undefine _missing_build_ids_terminate_build
+
 %global repo            VictoriaMetrics
 %global provider        github.com/VictoriaMetrics/%{repo}
 %global commit          pmm-6401-v1.93.4
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:           percona-victoriametrics
 Version:        1.93.4
 Release:        2%{?dist}
diff --git a/build/packages/rpm/server/SPECS/vmproxy.spec b/build/packages/rpm/server/SPECS/vmproxy.spec
index fe4100e2d8..11ca103785 100644
--- a/build/packages/rpm/server/SPECS/vmproxy.spec
+++ b/build/packages/rpm/server/SPECS/vmproxy.spec
@@ -1,3 +1,4 @@
+%undefine _missing_build_ids_terminate_build
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
@@ -10,12 +11,6 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:     vmproxy
 Version:  %{full_pmm_version}
 Release:  %{rpm_release}
@@ -44,7 +39,7 @@ make -C vmproxy release
 
 %install
 install -d -p %{buildroot}%{_sbindir}
-install -p -m 0755 vmproxy/bin/vmproxy %{buildroot}%{_sbindir}/vmproxy
+install -p -m 0755 ./bin/vmproxy %{buildroot}%{_sbindir}/vmproxy
 
 
 %files

From afde65ad8151a097eec42d8a80f4f1671c58686f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 22:24:15 +0000
Subject: [PATCH 14/78] PMM-12899 revert QAN package name

---
 build/packages/rpm/server/SPECS/percona-qan-api2.spec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/packages/rpm/server/SPECS/percona-qan-api2.spec b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
index 46995118b1..3f851cb36f 100644
--- a/build/packages/rpm/server/SPECS/percona-qan-api2.spec
+++ b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
@@ -11,7 +11,7 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:           pmm-qan
+Name:           percona-qan-api2
 Version:        %{version}
 Release:        %{rpm_release}
 Summary:        Query Analytics API for PMM

From f28cbccca37377a86d6abff3b6f08a9993955551 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 11:03:47 +0000
Subject: [PATCH 15/78] PMM-12899 merge update.sh to build.sh

---
 build/local/build.sh              | 316 ++++++++++++++++++++++--------
 build/local/update.sh             | 101 ----------
 build/scripts/build-client-binary |   4 +
 build/scripts/build-client-docker |   6 +-
 build/scripts/build-client-rpm    |   6 +-
 build/scripts/build-client-source |   4 +
 build/scripts/build-client-srpm   |   4 +
 build/scripts/build-server-docker |   9 +-
 build/scripts/build-server-rpm    |   6 +-
 9 files changed, 269 insertions(+), 187 deletions(-)
 delete mode 100755 build/local/update.sh

diff --git a/build/local/build.sh b/build/local/build.sh
index bd174fb965..6e9985e706 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -2,12 +2,116 @@
 
 # Define global variables
 NO_UPDATE=0
+UPDATE_ONLY=0
 NO_CLIENT=0
 NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
+OUTPUT_LOG_FILE="/tmp/build.log"
+
+needs-to-pull() {
+  local UPSTREAM=${1:-'@{u}'}
+  local LOCAL=$(git rev-parse @)
+  local BASE=$(git merge-base @ "$UPSTREAM")
+  local REMOTE=$(git rev-parse "$UPSTREAM")
+
+  if [ "$LOCAL" = "$REMOTE" ]; then
+    return 1 # false, we are up-to-date
+  fi
+
+  if [ "$LOCAL" = "$BASE" ]; then
+    return 0 # true, we are behind upstream
+  fi
+}
+
+rewind() {
+  local DIR="$1"
+  local BRANCH="$2"
+
+  cd "$DIR"
+  CURRENT=$(git branch --show-current)
+  git fetch
+
+  if [ "$CURRENT" != "$BRANCH" ]; then
+    echo "Currently on $CURRENT, checking out $BRANCH"
+    git checkout "$BRANCH"
+  fi
+
+  if needs-to-pull; then
+    git pull origin
+    echo "Submodule has pulled from upstream"
+    git logs -n 2
+    cd - >/dev/null
+    git add "$DIR"
+  else
+    cd - >/dev/null
+    echo "Submodule is up-to-date with upstream"
+  fi
+}
+
+check-files() {
+  local DIR="$1"
+
+  test -z "DIR" && exit 1
+
+  if [ -d "$DIR/sources" ] && [ -f "$DIR/ci-default.yml" ] && [ -f "$DIR/ci.yml" ]; then
+    return 0
+  fi
+
+  return 1
+}
+
+update() {
+  local DEPS=
+  local CURDIR="$PWD"
+  local DIR=pmm-submodules
+
+  # Thouroughly verify the presence of known files, otherwise bail out
+  if check-files "."; then # pwd is pmm-submodules
+    DIR="."
+  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
+    if ! check-files "$DIR"; then
+      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
+      exit 1
+    fi
+  else
+    echo "FATAL: could not locate known files in $PWD"
+    exit 1
+  fi
+
+  cd "$DIR"
+
+  # Join the dependencies from ci-default.yml and ci.yml
+  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
+
+  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'"
+
+  echo "$DEPS" | jq -c '.[]' | while read -r item; do
+    branch=$(echo "$item" | jq -r '.branch')
+    path=$(echo "$item" | jq -r '.path')
+    name=$(echo "$item" | jq -r '.name')
+    echo
+    echo "Rewinding submodule '$name' ..."
+    echo "path: ${path}, branch: ${branch}"
+
+    rewind "$path" "$branch"
+  done
+
+  echo
+  echo "Printing git status..."
+  git status --short
+  echo
+  echo "Printing git submodule status..."
+  git submodule status
+
+  cd "$CURDIR" > /dev/null
+}
+
 
 while test "$#" -gt 0; do
   case "$1" in
+    --update-only)
+      UPDATE_ONLY=1; NO_UPDATE=0
+      ;;
     --no-update)
       NO_UPDATE=1
       ;;
@@ -22,7 +126,7 @@ while test "$#" -gt 0; do
       ;;
     *)
       echo "Unknown argument: $1"
-      echo "Usage: $0 [--no-update] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm]"
       exit 1
       ;;
   esac
@@ -30,8 +134,18 @@ while test "$#" -gt 0; do
 done
 
 if [ "$NO_UPDATE" -eq 0 ]; then
+  MD5SUM=$(md5sum $(dirname $0)/build.sh)
+  
   # Update submodules and PR branches
-  /bin/bash $(dirname $0)/update.sh
+  update
+
+  test "$UPDATE_ONLY" -eq 1 && return
+
+  if [ "$MD5SUM" != "$(md5sum $(dirname $0)/build.sh)" ]; then
+    echo "The updated version of this script has been fetched from the repository, exiting..."
+    echo "Please run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+    return
+  fi
 fi
 
 get_branch_name() {
@@ -46,97 +160,141 @@ get_branch_name() {
   echo $branch_name
 }
 
-# Define global variables
-pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
-echo $pmm_commit > apiCommitSha
-pmm_branch=$(get_branch_name pmm)
-echo $pmm_branch > apiBranch
-pmm_url=$(git config -f .gitmodules submodule.pmm.url)
-echo $pmm_url > apiURL
-pmm_qa_branch=$(get_branch_name pmm-qa)
-echo $pmm_qa_branch > pmmQABranch
-pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
-echo $pmm_qa_commit > pmmQACommitSha
-pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
-echo $pmm_ui_tests_branch > pmmUITestBranch
-pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
-echo $pmm_ui_tests_commit > pmmUITestsCommitSha
-fb_commit_sha=$(git rev-parse HEAD)
-echo $fb_commit_sha > fbCommitSha
-
-# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
-# Important: the docker container's user need to be able to write to these directories.
-# The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
-# that the directories we create on the host are owned by a user with same uid and gid.
-
-# Create cache directories. 
-test -d "${root_dir}/go-path" || mkdir -p "go-path"
-test -d "${root_dir}/go-build" || mkdir -p "go-build"
-test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
-
-PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
-export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+build_with_logs() {
+  local script="$1"
+  local start_time
+  local end_time
+
+  if [ ! -f "$script" ]; then
+    echo "Fatal: script $script does not exist"
+    exit 1
+  fi
+
+  start_time=$(date +%s)
+  if [ "$#" -gt 1 ]; then
+    shift
+    $script "$@" | tee -a $OUTPUT_LOG_FILE
+  else
+    $script | tee -a $OUTPUT_LOG_FILE
+  fi
+  end_time=$(date +%s)
+
+  echo "Execution time for $script: $((end_time - start_time)) seconds"
+}
+
+init() {
+  # Remove the temp directory
+  if [ -d tmp ]; then
+    echo "Removing tmp directory..."
+    rm -rf tmp
+  fi
+  if [ -f "$OUTPUT_LOG_FILE" ]; then
+    echo "Removing the log file..."
+    rm -f $OUTPUT_LOG_FILE
+  fi
+
+  # Define global variables
+  pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+  echo $pmm_commit > apiCommitSha
+  pmm_branch=$(get_branch_name pmm)
+  echo $pmm_branch > apiBranch
+  pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+  echo $pmm_url > apiURL
+  pmm_qa_branch=$(get_branch_name pmm-qa)
+  echo $pmm_qa_branch > pmmQABranch
+  pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+  echo $pmm_qa_commit > pmmQACommitSha
+  pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
+  echo $pmm_ui_tests_branch > pmmUITestBranch
+  pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
+  echo $pmm_ui_tests_commit > pmmUITestsCommitSha
+  fb_commit_sha=$(git rev-parse HEAD)
+  echo $fb_commit_sha > fbCommitSha
+
+  PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+  export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+
+  # We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
+  # Important: the docker container's user needs to be able to write to these directories.
+  # The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
+  # that the directories we create on the host are owned by a user with the same uid and gid.
+
+  # Create cache directories.
+  test -d "${root_dir}/go-path" || mkdir -p "go-path"
+  test -d "${root_dir}/go-build" || mkdir -p "go-build"
+  test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
+  test -d "${root_dir}/yum-cache" || mkdir -p "yum-cache"
+
+}
+
+cleanup() {
+  # Clean up temporary files
+  rm -f apiBranch \
+    apiCommitSha \
+    apiURL \
+    fbCommitSha \
+    pmmQABranch \
+    pmmQACommitSha \
+    pmmUITestBranch \
+    pmmUITestsCommitSha
+}
 
 # Local reference test environment
 # CPU: 4 cores
 # RAM: 16GB
 # OS: Ubuntu 22.04.1 LTS
 
-if [ "$NO_CLIENT" -eq 0 ]; then
-  # Build client source: 4m39s from scratch, 0m27s using cache
-  "$PATH_TO_SCRIPTS/build-client-source"
+main() {
 
-  # Build client binary: ??? from scratch, 0m20s using cache
-  "$PATH_TO_SCRIPTS/build-client-binary"
+  if [ "$NO_CLIENT" -eq 0 ]; then
+    # Build client source: 4m39s from scratch, 0m27s using cache
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-source"
 
-  # Building client source rpm takes 13s (caching does not apply)
-  "$PATH_TO_SCRIPTS/build-client-srpm"
+    # Build client binary: ??? from scratch, 0m20s using cache
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-binary"
 
-  # Building client rpm takes 1m40s
-  "$PATH_TO_SCRIPTS/build-client-rpm"
-fi
+    # Building client source rpm takes 13s (caching does not apply)
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-srpm"
 
-# Building client docker image takes 17s
-GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
-export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
-if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
-  "$PATH_TO_SCRIPTS/build-client-docker"
-fi
+    # Building client rpm takes 1m40s
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-rpm"
+  fi
 
-# Building PMM CLient locally (non-CI, i.e. non-Jenkins)
-# total time: 6m26s - build from scratch, no initial cache
-# total time: 2m49s - subsequent build, using cache from prior builds
+  # Building client docker image takes 17s
+  GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
+  export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
+  if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-docker"
+  fi
 
+  # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
+  # total time: 6m26s - build from scratch, no initial cache
+  # total time: 2m49s - subsequent build, using cache from prior builds
 
-# Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
-# total time: 8m45s - build from scratch, no initial cache
-# total time: ??? - subsequent build, using cache from prior builds
 
-export RPM_EPOCH=1
+  # Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
+  # total time: 8m45s - build from scratch, no initial cache
+  # total time: ??? - subsequent build, using cache from prior builds
 
-if [ "$NO_SERVER_RPM" -eq 0 ]; then
-  "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
-  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
-  "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
-  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-  "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+  export RPM_EPOCH=1
+  if [ "$NO_SERVER_RPM" -eq 0 ]; then
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
-  # 3rd-party
-  "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-  "$PATH_TO_SCRIPTS/build-server-rpm" grafana
-fi
+    # 3rd-party
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" grafana
+  fi
+
+  export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+  export DOCKERFILE=Dockerfile.el9
+  build_with_logs "${PATH_TO_SCRIPTS}/build-server-docker"
+}
 
-export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
-export DOCKERFILE=Dockerfile.el9
-${PATH_TO_SCRIPTS}/build-server-docker
-
-# Clean up temporary files
-rm -f apiBranch \
-  apiCommitSha \
-	apiURL \
-	fbCommitSha \
-	pmmQABranch \
-	pmmQACommitSha \
-	pmmUITestBranch \
-	pmmUITestsCommitSha
+init
+main
+cleanup
diff --git a/build/local/update.sh b/build/local/update.sh
deleted file mode 100755
index 4922dd9bda..0000000000
--- a/build/local/update.sh
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash -e
-
-needs-to-pull() {
-  local UPSTREAM=${1:-'@{u}'}
-  local LOCAL=$(git rev-parse @)
-  local BASE=$(git merge-base @ "$UPSTREAM")
-  local REMOTE=$(git rev-parse "$UPSTREAM")
-
-  if [ "$LOCAL" = "$REMOTE" ]; then
-    return 1 # false, we are up-to-date
-  fi
-
-  if [ "$LOCAL" = "$BASE" ]; then
-    return 0 # true, we are behind upstream
-  fi
-}
-
-rewind() {
-  local DIR="$1"
-  local BRANCH="$2"
-
-  cd "$DIR"
-  CURRENT=$(git branch --show-current)
-  git fetch
-
-  if [ "$CURRENT" != "$BRANCH" ]; then
-    echo "Currently on $CURRENT, checking out $BRANCH"
-    git checkout "$BRANCH"
-  fi
-
-  if needs-to-pull; then
-    git pull origin
-    echo "Submodule has pulled from upstream"
-    git logs -n 2
-    cd - >/dev/null
-    git add "$DIR"
-  else
-    cd - >/dev/null
-    echo "Submodule is up-to-date with upstream"
-  fi
-}
-
-check-files() {
-  local DIR="$1"
-
-  test -z "DIR" && exit 1
-
-  if [ -d "$DIR/sources" ] && [ -f "$DIR/ci-default.yml" ] && [ -f "$DIR/ci.yml" ]; then
-    return 0
-  fi
-
-  return 1
-}
-
-main() {
-  local DEPS=
-  local CURDIR="$PWD"
-  local DIR=pmm-submodules
-
-  # Thouroughly verify the presence of known files, otherwise bail out
-  if check-files "."; then # pwd is pmm-submodules
-    DIR="."
-  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
-    if ! check-files "$DIR"; then
-      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
-      exit 1
-    fi
-  else
-    echo "FATAL: could not locate known files in $PWD"
-    exit 1
-  fi
-
-  cd "$DIR"
-
-  # Join the dependencies from ci-default.yml and ci.yml
-  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
-
-  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'"
-
-  echo "$DEPS" | jq -c '.[]' | while read -r item; do
-    branch=$(echo "$item" | jq -r '.branch')
-    path=$(echo "$item" | jq -r '.path')
-    name=$(echo "$item" | jq -r '.name')
-    echo
-    echo "Rewinding submodule '$name' ..."
-    echo "path: ${path}, branch: ${branch}"
-
-    rewind "$path" "$branch"
-  done
-
-  echo
-  echo "Printing git status..."
-  git status --short
-  echo
-  echo "Printing git submodule status..."
-  git submodule status
-
-  cd "$CURDIR" > /dev/null
-}
-
-main
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index f580e3f5d8..6b9460baf4 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -108,6 +108,10 @@ copy_component() {
 }
 
 main() {
+    echo -----------------------------------------
+    echo "Building PMM Client binary files..."
+    echo -----------------------------------------
+
     extract_source_tarball
 
     gobuild_component "vmagent" "" "github.com/VictoriaMetrics/VictoriaMetrics" "app/vmagent"
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index 119a47f7a3..af6cbae9b7 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -6,10 +6,14 @@ set -o xtrace
 . $(dirname $0)/vars
 
 if [ -f "${docker_client_tarball}" ]; then
-    echo skip docker build
+    echo Docker client tarball found, skipping docker build...
     exit 0
 fi
 
+echo -----------------------------------------
+echo "Building docker image for PMM Client..."
+echo -----------------------------------------
+
 DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
 cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
 
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index ae5780ae24..e11e691ae9 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -6,6 +6,10 @@ set -o errexit
 set -o xtrace
 
 main() {
+    echo -----------------------------------------
+    echo "Building PMM Client RPM files..."
+    echo -----------------------------------------
+
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
         set -o errexit
@@ -13,7 +17,7 @@ main() {
 
         export pmm_version=$pmm_version
         export pmm_release=$pmm_release
-        export sudo_path=\$(ls /usr/bin/sudo)
+        export sudo_path=\$$(command -v sudo)
         mkdir -p /tmp/pmm
         cd /home/builder/results
         /home/builder/bin/build-client-packages \
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index 978f09a1b0..a1112413ad 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -59,6 +59,10 @@ prepare_vmagent_tarball() {
 }
 
 main() {
+    echo -----------------------------------------
+    echo "Building PMM Client source files..."
+    echo -----------------------------------------
+
     rm -rf ${source_tarball}
     mkdir -p ${source_dir}/ $(dirname ${source_tarball}) || :
 
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index 20747d0779..8e4fe8e315 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -6,6 +6,10 @@ set -o errexit
 set -o xtrace
 
 main() {
+    echo -----------------------------------------
+    echo "Building PMM Client source RPM files..."
+    echo -----------------------------------------
+
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
         set -o errexit
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 16027c3beb..e6bffbaa20 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -6,15 +6,16 @@ set -o xtrace
 . $(dirname $0)/vars
 
 if [ -f "${docker_tarball}" ]; then
-    echo skip docker build
+    echo Docker tarball found, skipping docker build...
     exit 0
 fi
 
+echo -----------------------------------------
+echo "Building docker image for PMM Server..."
+echo -----------------------------------------
+
 docker_root=$(realpath ${rpms_dir}/..)
 cp -r ${root_dir}/tmp/source/pmm/build/ansible ${docker_root}/ansible
-ls ${docker_root}/ansible
-ls ${docker_root}/ansible/roles/pmm-images
-ls ${docker_root}/ansible/roles/pmm-images/tasks
 cp ${root_dir}/tmp/source/pmm/build/docker/server/* ${docker_root}/
 cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
 
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index ff5ab06553..4ada6b52d1 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -43,7 +43,7 @@ is_build_needed() {
             --region us-east-2 \
             --no-sign-request \
             s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-            ${rpms_dir}/${spec_name}-${rpm_version}
+            ${rpms_dir}/${spec_name}-${rpm_version} || :
     fi
 
     packages=$(find ${rpms_dir}/${spec_name}-${rpm_version} -name "*.${rpmbuild_dist}.*.rpm" | wc -l)
@@ -109,6 +109,10 @@ build() {
         return
     fi
 
+    echo -------------------------------------------------------
+    echo "Building PMM Server RPM for ${spec_name} component..."
+    echo -------------------------------------------------------
+
     if [ -d "${root_dir}/go-build" ]; then
         volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
     fi

From 093179afc763d58df0a9c7445e6fab5ebbbbd6ec Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 11:37:36 +0000
Subject: [PATCH 16/78] PMM-12899 reorganize the script structure

---
 build/local/build.sh | 93 +++++++++++++++++++++-----------------------
 1 file changed, 45 insertions(+), 48 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 6e9985e706..e59c9d8174 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -8,6 +8,32 @@ NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
 OUTPUT_LOG_FILE="/tmp/build.log"
 
+while test "$#" -gt 0; do
+  case "$1" in
+    --update-only)
+      UPDATE_ONLY=1; NO_UPDATE=0
+      ;;
+    --no-update)
+      NO_UPDATE=1
+      ;;
+    --no-client)
+      NO_CLIENT=1; NO_CLIENT_DOCKER=1
+      ;;
+    --no-client-docker)
+      NO_CLIENT_DOCKER=1
+      ;;
+    --no-server-rpm)
+      NO_SERVER_RPM=1
+      ;;
+    *)
+      echo "Unknown argument: $1"
+      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      exit 1
+      ;;
+  esac
+  shift
+done
+
 needs-to-pull() {
   local UPSTREAM=${1:-'@{u}'}
   local LOCAL=$(git rev-parse @)
@@ -106,48 +132,6 @@ update() {
   cd "$CURDIR" > /dev/null
 }
 
-
-while test "$#" -gt 0; do
-  case "$1" in
-    --update-only)
-      UPDATE_ONLY=1; NO_UPDATE=0
-      ;;
-    --no-update)
-      NO_UPDATE=1
-      ;;
-    --no-client)
-      NO_CLIENT=1; NO_CLIENT_DOCKER=1
-      ;;
-    --no-client-docker)
-      NO_CLIENT_DOCKER=1
-      ;;
-    --no-server-rpm)
-      NO_SERVER_RPM=1
-      ;;
-    *)
-      echo "Unknown argument: $1"
-      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm]"
-      exit 1
-      ;;
-  esac
-  shift
-done
-
-if [ "$NO_UPDATE" -eq 0 ]; then
-  MD5SUM=$(md5sum $(dirname $0)/build.sh)
-  
-  # Update submodules and PR branches
-  update
-
-  test "$UPDATE_ONLY" -eq 1 && return
-
-  if [ "$MD5SUM" != "$(md5sum $(dirname $0)/build.sh)" ]; then
-    echo "The updated version of this script has been fetched from the repository, exiting..."
-    echo "Please run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
-    return
-  fi
-fi
-
 get_branch_name() {
   local module="${1:-}"
   local branch_name
@@ -224,7 +208,6 @@ init() {
   test -d "${root_dir}/go-build" || mkdir -p "go-build"
   test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
   test -d "${root_dir}/yum-cache" || mkdir -p "yum-cache"
-
 }
 
 cleanup() {
@@ -239,12 +222,21 @@ cleanup() {
     pmmUITestsCommitSha
 }
 
-# Local reference test environment
-# CPU: 4 cores
-# RAM: 16GB
-# OS: Ubuntu 22.04.1 LTS
-
 main() {
+  if [ "$NO_UPDATE" -eq 0 ]; then
+    MD5SUM=$(md5sum $(dirname $0)/build.sh)
+    
+    # Update submodules and PR branches
+    update
+
+    test "$UPDATE_ONLY" -eq 1 && return
+
+    if [ "$MD5SUM" != "$(md5sum $(dirname $0)/build.sh)" ]; then
+      echo "The updated version of this script has been fetched from the repository, exiting..."
+      echo "Please run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+      return
+    fi
+  fi
 
   if [ "$NO_CLIENT" -eq 0 ]; then
     # Build client source: 4m39s from scratch, 0m27s using cache
@@ -295,6 +287,11 @@ main() {
   build_with_logs "${PATH_TO_SCRIPTS}/build-server-docker"
 }
 
+# Local reference test environment
+# CPU: 4 cores
+# RAM: 16GB
+# OS: Ubuntu 22.04.1 LTS
+
 init
 main
 cleanup

From 12fd453a88a2898ea97728209be8e7f06b11409c Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 13:20:14 +0000
Subject: [PATCH 17/78] PMM-12899 get rid of `exit 0` in callee scripts

---
 build/local/build.sh                         | 23 +++--
 build/packages/rpm/server/SPECS/grafana.spec |  2 +-
 build/scripts/build-client-binary            |  1 -
 build/scripts/build-client-deb               |  1 -
 build/scripts/build-client-docker            | 76 ++++++++-------
 build/scripts/build-client-rpm               |  1 -
 build/scripts/build-client-sdeb              |  1 -
 build/scripts/build-client-source            |  1 -
 build/scripts/build-client-srpm              |  1 -
 build/scripts/build-server-docker            | 98 ++++++++++----------
 build/scripts/vars                           |  4 +-
 11 files changed, 110 insertions(+), 99 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index e59c9d8174..32c8caa52f 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -6,7 +6,7 @@ UPDATE_ONLY=0
 NO_CLIENT=0
 NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
-OUTPUT_LOG_FILE="/tmp/build.log"
+LOG_FILE="/tmp/build.log"
 
 while test "$#" -gt 0; do
   case "$1" in
@@ -25,9 +25,18 @@ while test "$#" -gt 0; do
     --no-server-rpm)
       NO_SERVER_RPM=1
       ;;
+    --log-file)
+      shift
+      if [ -z "$1" ]; then
+        echo "Missing argument for --log-file"
+        exit 1
+      fi
+      LOG_FILE="$1"
+      ;;
     *)
       echo "Unknown argument: $1"
-      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>]"
+      echo
       exit 1
       ;;
   esac
@@ -150,16 +159,16 @@ build_with_logs() {
   local end_time
 
   if [ ! -f "$script" ]; then
-    echo "Fatal: script $script does not exist"
+    echo "FATAL: script $script does not exist"
     exit 1
   fi
 
   start_time=$(date +%s)
   if [ "$#" -gt 1 ]; then
     shift
-    $script "$@" | tee -a $OUTPUT_LOG_FILE
+    $script "$@" | tee -a $LOG_FILE
   else
-    $script | tee -a $OUTPUT_LOG_FILE
+    $script | tee -a $LOG_FILE
   fi
   end_time=$(date +%s)
 
@@ -172,9 +181,9 @@ init() {
     echo "Removing tmp directory..."
     rm -rf tmp
   fi
-  if [ -f "$OUTPUT_LOG_FILE" ]; then
+  if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file..."
-    rm -f $OUTPUT_LOG_FILE
+    rm -f $LOG_FILE
   fi
 
   # Define global variables
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index e4c3e1558d..94e993bd3c 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -75,7 +75,7 @@ install -d -p %{buildroot}%{_sharedstatedir}/grafana
 %pre
 getent group pmm >/dev/null || echo "Group pmm does not exist. Please create it manually."
 getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it manually."
-exit 0
+
 
 %changelog
 * Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-105
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 6b9460baf4..c26344bb6c 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -157,6 +157,5 @@ main() {
 }
 
 main
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-deb b/build/scripts/build-client-deb
index 2a94002557..ccedf8e497 100755
--- a/build/scripts/build-client-deb
+++ b/build/scripts/build-client-deb
@@ -35,6 +35,5 @@ main() {
 }
 
 main $*
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index af6cbae9b7..9494d41f5d 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -5,41 +5,45 @@ set -o xtrace
 
 . $(dirname $0)/vars
 
-if [ -f "${docker_client_tarball}" ]; then
-    echo Docker client tarball found, skipping docker build...
-    exit 0
-fi
-
-echo -----------------------------------------
-echo "Building docker image for PMM Client..."
-echo -----------------------------------------
-
-DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
-cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
-
-if [ -z "${DOCKER_CLIENT_TAG}" ]; then
-    DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version}
-fi
-
-
-CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
-
-docker buildx build \
-    --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
-    --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
-    --progress plain \
-    -f ${DOCKER_FILE_LOCATION}/${docker_file} \
-    -t ${DOCKER_CLIENT_TAG} \
-    ${DOCKER_FILE_LOCATION}
-
-if [ -n "${PUSH_DOCKER}" ]; then
-    mkdir -p $(dirname ${docker_client_tag_file})
-    echo ${DOCKER_CLIENT_TAG} > ${docker_client_tag_file}
-    docker push ${DOCKER_CLIENT_TAG}
-fi
-if [ -n "${SAVE_DOCKER}" ]; then
-    mkdir -p $(dirname ${docker_client_tarball})
-    docker save ${DOCKER_CLIENT_TAG} | xz > ${docker_client_tarball}
-fi
+main() {
+    echo -----------------------------------------
+    echo "Building docker image for PMM Client..."
+    echo -----------------------------------------
+
+    if [ -f "${docker_client_tarball}" ]; then
+        echo Docker client tarball found, skipping docker build...
+        return
+    fi
+
+    DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
+    cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
+
+    if [ -z "${DOCKER_CLIENT_TAG}" ]; then
+        DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version}
+    fi
+
+
+    CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
+
+    docker buildx build \
+        --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
+        --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
+        --progress plain \
+        -f ${DOCKER_FILE_LOCATION}/${docker_file} \
+        -t ${DOCKER_CLIENT_TAG} \
+        ${DOCKER_FILE_LOCATION}
+
+    if [ -n "${PUSH_DOCKER}" ]; then
+        mkdir -p $(dirname ${docker_client_tag_file})
+        echo ${DOCKER_CLIENT_TAG} > ${docker_client_tag_file}
+        docker push ${DOCKER_CLIENT_TAG}
+    fi
+    if [ -n "${SAVE_DOCKER}" ]; then
+        mkdir -p $(dirname ${docker_client_tarball})
+        docker save ${DOCKER_CLIENT_TAG} | xz > ${docker_client_tarball}
+    fi
+}
+
+main
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index e11e691ae9..6cf2291207 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -29,6 +29,5 @@ main() {
 }
 
 main $*
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-sdeb b/build/scripts/build-client-sdeb
index f52cff6577..fb6bb50a43 100755
--- a/build/scripts/build-client-sdeb
+++ b/build/scripts/build-client-sdeb
@@ -32,6 +32,5 @@ main() {
 }
 
 main $*
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index a1112413ad..7b0b62fb7a 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -84,6 +84,5 @@ main() {
 }
 
 main
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index 8e4fe8e315..ff4671659b 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -28,6 +28,5 @@ main() {
 }
 
 main $*
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index e6bffbaa20..005fbee466 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -5,52 +5,56 @@ set -o xtrace
 
 . $(dirname $0)/vars
 
-if [ -f "${docker_tarball}" ]; then
-    echo Docker tarball found, skipping docker build...
-    exit 0
-fi
-
-echo -----------------------------------------
-echo "Building docker image for PMM Server..."
-echo -----------------------------------------
-
-docker_root=$(realpath ${rpms_dir}/..)
-cp -r ${root_dir}/tmp/source/pmm/build/ansible ${docker_root}/ansible
-cp ${root_dir}/tmp/source/pmm/build/docker/server/* ${docker_root}/
-cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
-
-git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
-ls -la ${rpms_dir}
-docker run --rm -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
-    sudo chown -R builder /home/builder/rpm/RPMS
-    until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
-        echo "waiting"
-        sleep 1
-    done
-"
-
-if [ -z "${DOCKER_TAG}" ]; then
-    DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
-fi
-
-IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
-
-docker buildx build \
-    --build-arg BUILD_DATE="$(date --rfc-3339=seconds)" \
-    --build-arg VERSION="$IMAGE_VERSION" \
-    --progress plain \
-    -f ${docker_root}/${docker_file} \
-    -t ${DOCKER_TAG} \
-    ${docker_root}
-
-if [ -n "${PUSH_DOCKER}" ]; then
-    mkdir -p $(dirname ${docker_tag_file})
-    echo ${DOCKER_TAG} > ${docker_tag_file}
-    docker push ${DOCKER_TAG}
-fi
-if [ -n "${SAVE_DOCKER}" ]; then
-    mkdir -p $(dirname ${docker_tarball})
-    docker save ${DOCKER_TAG} > ${docker_tarball}
-fi
+main() {
+  echo -----------------------------------------
+  echo "Building docker image for PMM Server..."
+  echo -----------------------------------------
+
+  if [ -f "${docker_tarball}" ]; then
+      echo Docker tarball found, skipping docker build...
+      return
+  fi
+
+  docker_root=$(realpath ${rpms_dir}/..)
+  cp -r ${root_dir}/tmp/source/pmm/build/ansible ${docker_root}/ansible
+  cp ${root_dir}/tmp/source/pmm/build/docker/server/* ${docker_root}/
+  cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
+
+  git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
+  ls -la ${rpms_dir}
+  docker run --rm -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
+      sudo chown -R builder /home/builder/rpm/RPMS
+      until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
+          echo "waiting"
+          sleep 1
+      done
+  "
+
+  if [ -z "${DOCKER_TAG}" ]; then
+      DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
+  fi
+
+  IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+
+  docker buildx build \
+      --build-arg BUILD_DATE="$(date --rfc-3339=seconds)" \
+      --build-arg VERSION="$IMAGE_VERSION" \
+      --progress plain \
+      -f ${docker_root}/${docker_file} \
+      -t ${DOCKER_TAG} \
+      ${docker_root}
+
+  if [ -n "${PUSH_DOCKER}" ]; then
+      mkdir -p $(dirname ${docker_tag_file})
+      echo ${DOCKER_TAG} > ${docker_tag_file}
+      docker push ${DOCKER_TAG}
+  fi
+  if [ -n "${SAVE_DOCKER}" ]; then
+      mkdir -p $(dirname ${docker_tarball})
+      docker save ${DOCKER_TAG} > ${docker_tarball}
+  fi
+}
+
+main
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/vars b/build/scripts/vars
index de57bfb8a4..338e6512e2 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -2,8 +2,8 @@ bin_dir=$(cd $(dirname $0); pwd -P)
 # TODO: refactor to pass ${WORKSPACE} as ROOT_DIR from either Jenkins or, if relevant, from GH actions
 # NOTE: in most cases, this evaluates to Jenkins ${WORKSPACE} directory when called from pipelines
 # Example: /home/ec2-user/workspace/ol9-build-server/, see how it's used in ol9-build-server.groovy
-root_dir_tmp=$(cd $(dirname $0)/../../../../../../../..; pwd -P)
-root_dir=${ROOT_DIR:-$root_dir_tmp}
+root_dir=$(cd $(dirname $0)/../../../../../../../..; pwd -P)
+root_dir=${ROOT_DIR:-$root_dir}
 tmp_dir=${root_dir}/tmp
 
 # In VERSION file we can have numeric value like '2.0.0' as well as

From 9587f21bcf70678c278c53f85c6b3b352ab11506 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 13:39:18 +0000
Subject: [PATCH 18/78] PMM-12899 clean up pmm-update.spec

---
 build/packages/rpm/server/SPECS/pmm-update.spec | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index cb048070c3..5e93dea47a 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -42,9 +42,6 @@ make -C update release
 
 
 %install
-install -d %{buildroot}%{_datadir}/%{name}
-cp -pav ./update/ansible %{buildroot}%{_datadir}/%{name}
-
 install -d %{buildroot}%{_sbindir}
 install -p -m 0755 update/bin/pmm-update %{buildroot}%{_sbindir}/
 
@@ -53,7 +50,6 @@ install -p -m 0755 update/bin/pmm-update %{buildroot}%{_sbindir}/
 %license update/LICENSE
 %doc update/README.md
 %{_sbindir}/pmm-update
-%{_datadir}/%{name}
 
 
 %changelog

From 13b4f6517e670e24d1dd906922ec50f586808099 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 14:23:50 +0000
Subject: [PATCH 19/78] PMM-12899 send exec time to the log file

---
 build/local/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 32c8caa52f..7f46190b8e 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -172,7 +172,7 @@ build_with_logs() {
   fi
   end_time=$(date +%s)
 
-  echo "Execution time for $script: $((end_time - start_time)) seconds"
+  echo "Execution time for $script: $((end_time - start_time)) seconds" | tee -a $LOG_FILE
 }
 
 init() {

From 6f9eac8ea978c432ad15f16a7f9a0525d3f0373b Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 17:07:58 +0000
Subject: [PATCH 20/78] PMM-12899 remove a redundant directory

---
 build/local/build.sh    | 4 +++-
 update/ansible/.gitkeep | 0
 2 files changed, 3 insertions(+), 1 deletion(-)
 delete mode 100644 update/ansible/.gitkeep

diff --git a/build/local/build.sh b/build/local/build.sh
index 7f46190b8e..9dcef31aec 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -157,6 +157,7 @@ build_with_logs() {
   local script="$1"
   local start_time
   local end_time
+  local script_name=$(basename $script)
 
   if [ ! -f "$script" ]; then
     echo "FATAL: script $script does not exist"
@@ -166,13 +167,14 @@ build_with_logs() {
   start_time=$(date +%s)
   if [ "$#" -gt 1 ]; then
     shift
+    script_name="${script_name}:($1)"
     $script "$@" | tee -a $LOG_FILE
   else
     $script | tee -a $LOG_FILE
   fi
   end_time=$(date +%s)
 
-  echo "Execution time for $script: $((end_time - start_time)) seconds" | tee -a $LOG_FILE
+  echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee 
 }
 
 init() {
diff --git a/update/ansible/.gitkeep b/update/ansible/.gitkeep
deleted file mode 100644
index e69de29bb2..0000000000

From 3eb50f8b04721030c5be7778da3dd595cff496d4 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 17:42:18 +0000
Subject: [PATCH 21/78] PMM-12899 add total exec time

---
 build/local/build.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/build/local/build.sh b/build/local/build.sh
index 9dcef31aec..3c156e9349 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -6,6 +6,7 @@ UPDATE_ONLY=0
 NO_CLIENT=0
 NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
+START_TIME=$(date +%s)
 LOG_FILE="/tmp/build.log"
 
 while test "$#" -gt 0; do
@@ -296,6 +297,9 @@ main() {
   export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
   export DOCKERFILE=Dockerfile.el9
   build_with_logs "${PATH_TO_SCRIPTS}/build-server-docker"
+
+  echo "Done building PMM artifacts."
+  echo "Total execution time, sec: $(($(date +%s) - $START_TIME))" | tee -a $LOG_FILE
 }
 
 # Local reference test environment

From 36869b7a500ec7190e5af805468d57e50ec12f5f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 23:23:06 +0000
Subject: [PATCH 22/78] PMM-12899 reorg some variables

---
 build/local/build.sh                | 58 +++++++++++++++++------------
 build/scripts/build-client-packages | 24 ++++++------
 build/scripts/build-client-source   |  2 +-
 3 files changed, 48 insertions(+), 36 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 3c156e9349..ca1ac4a9dd 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -106,11 +106,11 @@ update() {
     DIR="."
   elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
     if ! check-files "$DIR"; then
-      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
+      echo "Fatal: could not locate known files in ${PWD}/${DIR}"
       exit 1
     fi
   else
-    echo "FATAL: could not locate known files in $PWD"
+    echo "Fatal: could not locate known files in $PWD"
     exit 1
   fi
 
@@ -155,13 +155,13 @@ get_branch_name() {
 }
 
 build_with_logs() {
-  local script="$1"
+  local script="$PATH_TO_SCRIPTS/$1"
   local start_time
   local end_time
-  local script_name=$(basename $script)
+  local script_name="$1"
 
   if [ ! -f "$script" ]; then
-    echo "FATAL: script $script does not exist"
+    echo "Fatal: script $script does not exist"
     exit 1
   fi
 
@@ -175,14 +175,23 @@ build_with_logs() {
   fi
   end_time=$(date +%s)
 
-  echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee 
+  echo ---
+  echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee -a $LOG_FILE
+  echo ---
 }
 
 init() {
-  # Remove the temp directory
+  local tmp_files
+  # Remove stale files and directories
   if [ -d tmp ]; then
-    echo "Removing tmp directory..."
-    rm -rf tmp
+    echo "Removing stale files and directories..."
+    if [ -d "tmp/pmm-server" ]; then
+      tmp_files=$(find tmp/pmm-server | grep -v "RPMS")
+      tmp_files=($tmp_files)
+      for f in "${tmp_files[@]}"; do
+        rm -rf "$f"
+      done
+    fi
   fi
   if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file..."
@@ -252,23 +261,23 @@ main() {
 
   if [ "$NO_CLIENT" -eq 0 ]; then
     # Build client source: 4m39s from scratch, 0m27s using cache
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-source"
+    build_with_logs build-client-source
 
     # Build client binary: ??? from scratch, 0m20s using cache
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-binary"
+    build_with_logs build-client-binary
 
     # Building client source rpm takes 13s (caching does not apply)
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-srpm"
+    build_with_logs build-client-srpm
 
     # Building client rpm takes 1m40s
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-rpm"
+    build_with_logs build-client-rpm
   fi
 
   # Building client docker image takes 17s
   GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
   export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-docker"
+    build_with_logs build-client-docker
   fi
 
   # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
@@ -282,24 +291,27 @@ main() {
 
   export RPM_EPOCH=1
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+    build_with_logs build-server-rpm percona-dashboards grafana-dashboards
+    build_with_logs build-server-rpm pmm-managed pmm
+    build_with_logs build-server-rpm percona-qan-api2 pmm
+    build_with_logs build-server-rpm pmm-update pmm
+    build_with_logs build-server-rpm pmm-dump
+    build_with_logs build-server-rpm vmproxy pmm
 
     # 3rd-party
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" grafana
+    build_with_logs build-server-rpm victoriametrics
+    build_with_logs build-server-rpm grafana
   fi
 
   export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
   export DOCKERFILE=Dockerfile.el9
-  build_with_logs "${PATH_TO_SCRIPTS}/build-server-docker"
+  build_with_logs build-server-docker
 
+  echo
   echo "Done building PMM artifacts."
+  echo ---
   echo "Total execution time, sec: $(($(date +%s) - $START_TIME))" | tee -a $LOG_FILE
+  echo ---
 }
 
 # Local reference test environment
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index e7205ae7bb..6f80554bc0 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -174,26 +174,26 @@ get_system(){
 }
 
 install_deps() {
-    local RUN_CMD
+    local PACKAGES
     if [ $INSTALL = 0 ]; then
-        echo "Dependencies will not be installed"
+        echo "Dependencies will not be installed."
         return;
     fi
     if [ ! $( id -u ) -eq 0 ]; then
-        echo "It is not possible to instal dependencies. Please run as root"
+        echo "It is not possible to install dependencies. Please run as root."
         exit 1
     fi
     if [ "$OS" = "rpm" ]; then
         # yum -y install git curl wget rpmdevtools bison yum-utils rpm-build
-        command -v git || RUN_CMD+=" git"
-        command -v curl || RUN_CMD+=" curl"
-        command -v wget || RUN_CMD+=" wget"
-        command -v spectool || RUN_CMD+=" rpmdevtools"
-        command -v bison || RUN_CMD+=" bison"
-        command -v repoquery || RUN_CMD+=" yum-utils"
-        command -v rpmbuild || RUN_CMD+=" rpm-build"
-        if [ -n "$RUN_CMD" ]; then
-            yum -y install "$RUN_CMD"
+        command -v git || PACKAGES+=" git"
+        command -v curl || PACKAGES+=" curl"
+        command -v wget || PACKAGES+=" wget"
+        command -v spectool || PACKAGES+=" rpmdevtools"
+        command -v bison || PACKAGES+=" bison"
+        command -v repoquery || PACKAGES+=" yum-utils"
+        command -v rpmbuild || PACKAGES+=" rpm-build"
+        if [ -n "$PACKAGES" ]; then
+            yum -y install "$PACKAGES"
         fi
     else
         apt-get update
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index 7b0b62fb7a..0786be9cd1 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -54,7 +54,7 @@ prepare_vmagent_tarball() {
     if [ -f "${tarball}" ]; then
         echo ${tarball} already exists, skipping download
     else
-        curl -o ${tarball} -fSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz
+        curl -o ${tarball} -fsSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz
     fi
 }
 

From bf640caa27e5eb9fb0a0670b643704232d382511 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 3 Apr 2024 00:02:13 +0000
Subject: [PATCH 23/78] PMM-12899 fix the regexp

---
 build/local/build.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index ca1ac4a9dd..999691aade 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -186,12 +186,15 @@ init() {
   if [ -d tmp ]; then
     echo "Removing stale files and directories..."
     if [ -d "tmp/pmm-server" ]; then
-      tmp_files=$(find tmp/pmm-server | grep -v "RPMS")
+      tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$")
       tmp_files=($tmp_files)
       for f in "${tmp_files[@]}"; do
         rm -rf "$f"
       done
     fi
+    if [ -d "tmp/source/pmm" ]; then
+      rm -rf tmp/source/pmm
+    fi
   fi
   if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file..."

From f28202aad1f4df24d3cff9ed50f6dfd65d6433d5 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 3 Apr 2024 07:50:16 +0000
Subject: [PATCH 24/78] PMM-12899 structure the code

---
 build/local/build.sh | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 999691aade..0779cbf53c 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -180,7 +180,7 @@ build_with_logs() {
   echo ---
 }
 
-init() {
+purge_files() {
   local tmp_files
   # Remove stale files and directories
   if [ -d tmp ]; then
@@ -200,7 +200,9 @@ init() {
     echo "Removing the log file..."
     rm -f $LOG_FILE
   fi
+}
 
+init() {
   # Define global variables
   pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
   echo $pmm_commit > apiCommitSha
@@ -262,6 +264,9 @@ main() {
     fi
   fi
 
+  init
+  purge_files
+
   if [ "$NO_CLIENT" -eq 0 ]; then
     # Build client source: 4m39s from scratch, 0m27s using cache
     build_with_logs build-client-source
@@ -315,6 +320,8 @@ main() {
   echo ---
   echo "Total execution time, sec: $(($(date +%s) - $START_TIME))" | tee -a $LOG_FILE
   echo ---
+
+  cleanup
 }
 
 # Local reference test environment
@@ -322,6 +329,4 @@ main() {
 # RAM: 16GB
 # OS: Ubuntu 22.04.1 LTS
 
-init
 main
-cleanup

From a0f7b5011f3661b4597c960fee4ef57386940272 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 3 Apr 2024 08:36:11 +0000
Subject: [PATCH 25/78] PMM-12899 add a flag to skip building server docker

---
 build/local/build.sh | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 0779cbf53c..84c071d851 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -6,6 +6,7 @@ UPDATE_ONLY=0
 NO_CLIENT=0
 NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
+NO_SERVER_DOCKER=0
 START_TIME=$(date +%s)
 LOG_FILE="/tmp/build.log"
 
@@ -24,8 +25,19 @@ while test "$#" -gt 0; do
       NO_CLIENT_DOCKER=1
       ;;
     --no-server-rpm)
+      if [ "$NO_SERVER_DOCKER" -eq 1 ]; then
+        echo "Cannot disable both server RPM and server Docker"
+        exit 1
+      fi
       NO_SERVER_RPM=1
       ;;
+    --no-server-docker)
+      if [ "$NO_SERVER_RPM" -eq 1 ]; then
+        echo "Cannot disable both server RPM and server Docker"
+        exit 1
+      fi
+      NO_SERVER_DOCKER=1
+      ;;
     --log-file)
       shift
       if [ -z "$1" ]; then
@@ -311,9 +323,11 @@ main() {
     build_with_logs build-server-rpm grafana
   fi
 
-  export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
-  export DOCKERFILE=Dockerfile.el9
-  build_with_logs build-server-docker
+  if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
+    export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+    export DOCKERFILE=Dockerfile.el9
+    build_with_logs build-server-docker
+  fi
 
   echo
   echo "Done building PMM artifacts."

From 988cfb0c32ef52158bfc05898362567171578df8 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 3 Apr 2024 09:16:40 +0000
Subject: [PATCH 26/78] PMM-12899 fix a failure of find when no files are found

---
 build/local/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 84c071d851..81cd1cddba 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -198,7 +198,7 @@ purge_files() {
   if [ -d tmp ]; then
     echo "Removing stale files and directories..."
     if [ -d "tmp/pmm-server" ]; then
-      tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$")
+      tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$" || :)
       tmp_files=($tmp_files)
       for f in "${tmp_files[@]}"; do
         rm -rf "$f"

From 0582998db6ff3244ba69fd1335a9ab324b107e68 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 10 Apr 2024 16:05:51 +0300
Subject: [PATCH 27/78] PMM-12899 update the documentation

---
 build/local/README.md | 51 +++++++++++++++++++++++++++++++++++++++----
 build/local/build.sh  | 17 +++++++++------
 2 files changed, 57 insertions(+), 11 deletions(-)

diff --git a/build/local/README.md b/build/local/README.md
index 1517e9da3f..5f9051784c 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -12,6 +12,7 @@ The build process is mostly based on bash scripts, which control the build flow.
 
 The build process is designed to be run on a Linux host. We believe it can be run on other flavors of Linux, including MacOS, with little to no modification (TBC).
 
+
 ## Prerequisites
 
 Below is a list of prerequisites that are required to build PMM locally.
@@ -26,9 +27,51 @@ Below is a list of prerequisites that are required to build PMM locally.
 - yq: 4.42.0+ (tested on 4.42.1)
 - jq: 1.6+ (tested on 1.6)
 
-## Build Steps
+
+## How to build PMM
 
 1. Install the prerequisites
-2. Clone the PMM repository
-3. Change to the `build/local` directory
-4. Run the `build.sh` script
+2. Clone the PMM repository to the user's home directory, e.g.: `git clone https://github.com/percona/pmm /home/user/pmm`. 
+3. Change to the `build/local` directory in the cloned repo.
+4. Run `build.sh --help` to print the help message and check the usage.
+5. Run `build.sh` with parameters of your choice to build PMM v3.
+
+Usually, you will want to rebuild PMM whenever there are changes in at least one of its components. All components of PMM are gathered together in one repository - `github.com/percona-lab/pmm-submodules` (or `pmm-submodules`). Therefore, you can run `build.sh` as often as those changes need to be factored in to the next build.
+
+Once the build is finished, you can proceed with launching a new instance of PMM Server, or installing a freshly built PMM Client, and testing the changes.
+
+
+## The `rpmbuild` image and docker cache
+
+We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`. It comes provisioned with tools which are required to build PMM artifacts, for example RPM packages. As a build tool, it offers a number of benefits, two most obvious of which are:
+
+- it frees the user from installing dependencies on their host machine
+- it leverages a very powerful docker caching system, which results in reduced build times
+
+During the first run, `build.sh` will create a few directories on the host machine, which are necessary to make use of docker cache. Please be aware, that the docker container's user needs to be able to write to these directories. The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure that the directories we create on the host are owned by a user with the same uid and gid. If the build fails, this is the first thing to check.
+
+
+## Avoiding unnecessary builds
+
+Sometimes, the changes you make affect only PMM Client. Other times, they affect only PMM Server. Therefore, you may want to skip building parts of PMM. The `build.sh` script offers several parameters to help control what you want to build.
+
+* --no-update: run the build tasks without pulling the changes from `pmm-submodules` repository
+* --update-only: pull changes from the repo without building PMM
+* --no-client: do not build the client, use the cached PMM Client artifacts
+* --no-client-docker: skip building PMM Client docker container
+* --no-server-rpm: skip building PMM Server RPM artifacts
+* --log-file <path>: change the path of the build log file
+
+It's important to note, however, that once all changes are made and tested, you most probably want to re-build both PMM Client and Server to test them together.
+
+
+## Target environments
+
+Currently, local builds target the following environments:
+- PMM Client
+  - tarball - virtually any amd64 Linux environment
+  - RPM - RHEL9-compatible environments
+  - docker image - docker and Kubernetes environments (amd64)
+- PMM Server
+  - docker image - docker and Kubernetes environments (amd64)
+
diff --git a/build/local/build.sh b/build/local/build.sh
index 81cd1cddba..a821dad1e7 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -9,6 +9,8 @@ NO_SERVER_RPM=0
 NO_SERVER_DOCKER=0
 START_TIME=$(date +%s)
 LOG_FILE="/tmp/build.log"
+BASE_NAME=$(basename $0)
+USAGE="Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>] [--help | -h]"
 
 while test "$#" -gt 0; do
   case "$1" in
@@ -46,9 +48,15 @@ while test "$#" -gt 0; do
       fi
       LOG_FILE="$1"
       ;;
+    --help | -h)
+      shift
+      echo "$USAGE"
+      echo
+      exit 0
+      ;;
     *)
       echo "Unknown argument: $1"
-      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>]"
+      echo "$USAGE"
       echo
       exit 1
       ;;
@@ -236,12 +244,7 @@ init() {
   PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
   export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
-  # We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
-  # Important: the docker container's user needs to be able to write to these directories.
-  # The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
-  # that the directories we create on the host are owned by a user with the same uid and gid.
-
-  # Create cache directories.
+  # Create cache directories. Read more in the section about `rpmbuild`.
   test -d "${root_dir}/go-path" || mkdir -p "go-path"
   test -d "${root_dir}/go-build" || mkdir -p "go-build"
   test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"

From ea42fe7904ab925058749c62d568e1830e854b98 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 12 Oct 2024 11:34:28 +0300
Subject: [PATCH 28/78] PMM-12899 update README.md

---
 build/local/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/build/local/README.md b/build/local/README.md
index 5f9051784c..b775030f66 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -6,11 +6,11 @@ This directory contains a set of scripts aimed at providing a simple way to buil
 
 Historically, PMM used to be built using Jenkins. This worked well for the team, but not for the community. The learning curve was, and still is, rather steep, and it is hard for folks, even internally, to contribute to.
 
-Therefore, we decided to make it possible to build PMM locally. This is a work in progress, but we are definitely committed to making it easier to build PMM locally.
+Therefore, we decided to make it possible to build PMM locally. This is a work in progress, but we are definitely committed to bring the developer experience to an acceptable level.
 
-The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on to make the build process easy to understand and contribute to. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching.
+The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on, since every developer should have at least a basic command of bash. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching.
 
-The build process is designed to be run on a Linux host. We believe it can be run on other flavors of Linux, including MacOS, with little to no modification (TBC).
+The build process is designed to run on a Linux host. We believe it can run on other flavors of Linux, including MacOS, with little to no modification (TBC).
 
 
 ## Prerequisites

From fc248de6e988937af6e7085fe7bfa7b5146e18ef Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 12 Oct 2024 14:20:03 +0300
Subject: [PATCH 29/78] PMM-12899 fix duplicate labels in Dockerfile

---
 build/docker/server/Dockerfile.el9 | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9
index 656ab8615c..9131565bb4 100644
--- a/build/docker/server/Dockerfile.el9
+++ b/build/docker/server/Dockerfile.el9
@@ -33,12 +33,6 @@ RUN install -T -p -m 644 /opt/ansible/ansible.cfg /etc/ansible/ansible.cfg && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm/post-build-actions.yml && \
     sed -i '/^assumeyes/d' /etc/dnf/dnf.conf
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses AGPL-3.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management
-LABEL org.opencontainers.image.vendor Percona LLC
-LABEL org.opencontainers.image.version ${VERSION}
-
 LABEL org.opencontainers.image.created ${BUILD_DATE}
 LABEL org.opencontainers.image.licenses AGPL-3.0
 LABEL org.opencontainers.image.title Percona Monitoring and Management

From 4b469bda90ddc03f14b072020225b5b09157c94b Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 15 Oct 2024 00:45:07 +0300
Subject: [PATCH 30/78] PMM-12899 add a config file for all PMM components

---
 build/local/ci-default.yml | 89 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 build/local/ci-default.yml

diff --git a/build/local/ci-default.yml b/build/local/ci-default.yml
new file mode 100644
index 0000000000..c344eeac13
--- /dev/null
+++ b/build/local/ci-default.yml
@@ -0,0 +1,89 @@
+deps:
+  # COMMON
+  - name: pmm
+    branch: v3
+    path: sources/pmm/src/github.com/percona/pmm
+    url: https://github.com/percona/pmm
+    component: common
+
+  # CLIENT
+  - name: node_exporter
+    branch: main
+    path: sources/node_exporter/src/github.com/prometheus/node_exporter
+    url: https://github.com/percona/node_exporter
+    component: client
+
+  - name: mysqld_exporter
+    branch: main
+    path: sources/mysqld_exporter/src/github.com/percona/mysqld_exporter
+    url: https://github.com/percona/mysqld_exporter
+    component: client
+
+  - name: mongodb_exporter
+    branch: main
+    path: sources/mongodb_exporter/src/github.com/percona/mongodb_exporter
+    url: https://github.com/percona/mongodb_exporter
+    component: client
+
+  - name: postgres_exporter
+    branch: main
+    path: sources/postgres_exporter/src/github.com/percona/postgres_exporter
+    url: https://github.com/percona/postgres_exporter
+    component: client
+
+  - name: proxysql_exporter
+    branch: main
+    path: sources/proxysql_exporter/src/github.com/percona/proxysql_exporter
+    url: https://github.com/percona/proxysql_exporter
+    component: client
+
+  - name: rds_exporter
+    branch: main
+    path: sources/rds_exporter/src/github.com/percona/rds_exporter
+    url: https://github.com/percona/rds_exporter
+    component: client
+
+  - name: azure_metrics_exporter
+    branch: main
+    path: sources/azure_metrics_exporter/src/github.com/percona/azure_metrics_exporter
+    url: https://github.com/percona/azure_metrics_exporter
+    component: client
+
+  - name: percona-toolkit
+    branch: release-v3.5.2
+    path: sources/percona-toolkit/src/github.com/percona/percona-toolkit
+    url: https://github.com/percona/percona-toolkit
+    component: client
+
+  # SERVER
+
+  - name: grafana-dashboards
+    branch: v3
+    path: sources/grafana-dashboards
+    url: https://github.com/percona/grafana-dashboards
+    component: server
+
+  - name: grafana
+    branch: v3
+    path: sources/grafana/src/github.com/grafana/grafana
+    url: https://github.com/percona/grafana
+    component: server
+
+  - name: pmm-dump
+    branch: main
+    path: sources/pmm-dump
+    url: https://github.com/percona/pmm-dump
+    component: server
+
+  # QA
+  - name: pmm-qa
+    branch: v3
+    path: sources/pmm-qa/src/github.com/percona/pmm-qa
+    url: https://github.com/percona/pmm-qa
+    component: qa
+
+  - name: pmm-ui-tests
+    branch: v3
+    path: sources/pmm-ui-tests/src/github.com/percona/pmm-ui-tests
+    url: https://github.com/percona/pmm-ui-tests
+    component: qa
\ No newline at end of file

From f35a85f16afa21f964837605f5194f0094f0ed99 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 19 Oct 2024 18:52:16 +0300
Subject: [PATCH 31/78] PMM-12899 simplify init, add usage

---
 build/local/build.sh | 50 +++++++++++++++++++-------------------------
 1 file changed, 22 insertions(+), 28 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index a821dad1e7..76b0327185 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -50,20 +50,31 @@ while test "$#" -gt 0; do
       ;;
     --help | -h)
       shift
-      echo "$USAGE"
-      echo
+      usage
       exit 0
       ;;
     *)
       echo "Unknown argument: $1"
-      echo "$USAGE"
-      echo
+      usage
       exit 1
       ;;
   esac
   shift
 done
 
+usage() {
+  cat <<-EOF
+Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>] [--help | -h]
+--no-update              Do not fetch the latest changes from the repo
+--update-only            Only fetch the latest changes from the repo
+--no-client              Do not (re)build PMM client
+--no-client-docker       Do not (re)build PMM Client docker image
+--no-server-rpm          Do not (re)build Server RPM packages
+--log-file <path>        Save build logs to a file located at <path>
+--help | -h              Display help
+EOF
+}
+
 needs-to-pull() {
   local UPSTREAM=${1:-'@{u}'}
   local LOCAL=$(git rev-parse @)
@@ -83,7 +94,7 @@ rewind() {
   local DIR="$1"
   local BRANCH="$2"
 
-  cd "$DIR"
+  cd "$DIR" > /dev/null
   CURRENT=$(git branch --show-current)
   git fetch
 
@@ -96,10 +107,10 @@ rewind() {
     git pull origin
     echo "Submodule has pulled from upstream"
     git logs -n 2
-    cd - >/dev/null
+    cd - > /dev/null
     git add "$DIR"
   else
-    cd - >/dev/null
+    cd - > /dev/null
     echo "Submodule is up-to-date with upstream"
   fi
 }
@@ -223,25 +234,8 @@ purge_files() {
 }
 
 init() {
-  # Define global variables
-  pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
-  echo $pmm_commit > apiCommitSha
-  pmm_branch=$(get_branch_name pmm)
-  echo $pmm_branch > apiBranch
-  pmm_url=$(git config -f .gitmodules submodule.pmm.url)
-  echo $pmm_url > apiURL
-  pmm_qa_branch=$(get_branch_name pmm-qa)
-  echo $pmm_qa_branch > pmmQABranch
-  pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
-  echo $pmm_qa_commit > pmmQACommitSha
-  pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
-  echo $pmm_ui_tests_branch > pmmUITestBranch
-  pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
-  echo $pmm_ui_tests_commit > pmmUITestsCommitSha
-  fb_commit_sha=$(git rev-parse HEAD)
-  echo $fb_commit_sha > fbCommitSha
-
   PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+  "${PATH_TO_SCRIPTS}/build-submodules"
   export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
   # Create cache directories. Read more in the section about `rpmbuild`.
@@ -290,10 +284,10 @@ main() {
     build_with_logs build-client-binary
 
     # Building client source rpm takes 13s (caching does not apply)
-    build_with_logs build-client-srpm
+    # build_with_logs build-client-srpm
 
     # Building client rpm takes 1m40s
-    build_with_logs build-client-rpm
+    # build_with_logs build-client-rpm
   fi
 
   # Building client docker image takes 17s
@@ -327,7 +321,7 @@ main() {
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
-    export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+    export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT}
     export DOCKERFILE=Dockerfile.el9
     build_with_logs build-server-docker
   fi

From 1aad3f74dc3617a2ebecb8734627e2e6d5283f62 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 20 Oct 2024 01:38:07 +0300
Subject: [PATCH 32/78] PMM-12899 fix architecture for the client

---
 build/scripts/build-client-binary | 4 ++++
 build/scripts/build-client-rpm    | 3 +--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 53cb44a8b0..8fdacdd3d8 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -27,6 +27,10 @@ gobuild_component() {
     local archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz" | head -n1)
     local build_command
 
+    if [ "$(uname -o)" = "Darwin" ]; then
+      local GOARCH=$(uname -m)
+    fi
+
     if [ -d "${root_dir}/go-build" ]; then
         volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
     fi
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index 6ec65cc016..517993a868 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -25,14 +25,13 @@ main() {
 
         export pmm_version=$pmm_version
         export pmm_release=$pmm_release
-        export sudo_path=\$$(command -v sudo)
         mkdir -p /tmp/pmm
         cd /home/builder/results
         /home/builder/bin/build-client-packages \
             --builddir=/tmp/pmm \
             --build_rpm=1
 
-        \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/rpm
+        sudo_path chown -R $(id -u):$(id -g) /home/builder/results/rpm
     "
 }
 

From 23aaaf6f3c63970753ed645c22ff0aef899bf10e Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 20 Oct 2024 02:44:57 +0300
Subject: [PATCH 33/78] PMM-12899 fix the build time for MacOS

---
 build/local/build.sh              | 234 ++++++++++++++++++------------
 build/scripts/build-server-docker |   5 +-
 2 files changed, 148 insertions(+), 91 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 76b0327185..0811217ca9 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -1,80 +1,86 @@
 #!/bin/bash -e
-
-# Define global variables
-NO_UPDATE=0
-UPDATE_ONLY=0
-NO_CLIENT=0
-NO_CLIENT_DOCKER=0
-NO_SERVER_RPM=0
-NO_SERVER_DOCKER=0
-START_TIME=$(date +%s)
-LOG_FILE="/tmp/build.log"
-BASE_NAME=$(basename $0)
-USAGE="Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>] [--help | -h]"
-
-while test "$#" -gt 0; do
-  case "$1" in
-    --update-only)
-      UPDATE_ONLY=1; NO_UPDATE=0
-      ;;
-    --no-update)
-      NO_UPDATE=1
-      ;;
-    --no-client)
-      NO_CLIENT=1; NO_CLIENT_DOCKER=1
-      ;;
-    --no-client-docker)
-      NO_CLIENT_DOCKER=1
-      ;;
-    --no-server-rpm)
-      if [ "$NO_SERVER_DOCKER" -eq 1 ]; then
-        echo "Cannot disable both server RPM and server Docker"
-        exit 1
-      fi
-      NO_SERVER_RPM=1
-      ;;
-    --no-server-docker)
-      if [ "$NO_SERVER_RPM" -eq 1 ]; then
-        echo "Cannot disable both server RPM and server Docker"
-        exit 1
-      fi
-      NO_SERVER_DOCKER=1
-      ;;
-    --log-file)
-      shift
-      if [ -z "$1" ]; then
-        echo "Missing argument for --log-file"
-        exit 1
-      fi
-      LOG_FILE="$1"
-      ;;
-    --help | -h)
-      shift
-      usage
-      exit 0
-      ;;
-    *)
-      echo "Unknown argument: $1"
-      usage
-      exit 1
-      ;;
-  esac
-  shift
-done
+set -o errexit
+set -o nounset
 
 usage() {
   cat <<-EOF
-Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>] [--help | -h]
+Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
 --no-update              Do not fetch the latest changes from the repo
 --update-only            Only fetch the latest changes from the repo
---no-client              Do not (re)build PMM client
---no-client-docker       Do not (re)build PMM Client docker image
---no-server-rpm          Do not (re)build Server RPM packages
+--no-client              Do not build PMM Client
+--client-docker          Build PMM Client docker image
+--no-server-rpm          Do not build Server RPM packages
+--no-server-docker       Do not build PMM Server docker image
 --log-file <path>        Save build logs to a file located at <path>
 --help | -h              Display help
 EOF
 }
 
+parse-params() {
+  # Define global variables
+  NO_UPDATE=0
+  UPDATE_ONLY=0
+  NO_CLIENT=0
+  NO_CLIENT_DOCKER=1
+  NO_SERVER_RPM=0
+  NO_SERVER_DOCKER=0
+  START_TIME=$(date +%s)
+  LOG_FILE="$(dirname $0)/build.log"
+  BASE_NAME=$(basename $0)
+  SUBMODULES=pmm-submodules
+  PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+
+  while test "$#" -gt 0; do
+    case "$1" in
+      --update-only)
+        UPDATE_ONLY=1; NO_UPDATE=0
+        ;;
+      --no-update)
+        NO_UPDATE=1
+        ;;
+      --no-client)
+        NO_CLIENT=1; NO_CLIENT_DOCKER=1
+        ;;
+      --client-docker)
+        NO_CLIENT_DOCKER=0
+        ;;
+      --no-server-rpm)
+        if [ "$NO_SERVER_DOCKER" -eq 1 ]; then
+          echo "Error: cannot disable both server RPM and server Docker"
+          exit 1
+        fi
+        NO_SERVER_RPM=1
+        ;;
+      --no-server-docker)
+        if [ "$NO_SERVER_RPM" -eq 1 ]; then
+          echo "Error: cannot disable both server RPM and server Docker"
+          exit 1
+        fi
+        NO_SERVER_DOCKER=1
+        ;;
+      --log-file)
+        shift
+        if [ -z "$1" ]; then
+          echo "Missing argument for --log-file"
+          exit 1
+        fi
+        LOG_FILE="$1"
+        ;;
+      --help | -h)
+        shift
+        usage
+        exit 0
+        ;;
+      *)
+        echo "Unknown argument: $1"
+        usage
+        exit 1
+        ;;
+    esac
+    shift
+  done
+}
+
 needs-to-pull() {
   local UPSTREAM=${1:-'@{u}'}
   local LOCAL=$(git rev-parse @)
@@ -95,7 +101,7 @@ rewind() {
   local BRANCH="$2"
 
   cd "$DIR" > /dev/null
-  CURRENT=$(git branch --show-current)
+  local CURRENT=$(git branch --show-current)
   git fetch
 
   if [ "$CURRENT" != "$BRANCH" ]; then
@@ -130,14 +136,17 @@ check-files() {
 update() {
   local DEPS=
   local CURDIR="$PWD"
-  local DIR=pmm-submodules
 
   # Thouroughly verify the presence of known files, otherwise bail out
-  if check-files "."; then # pwd is pmm-submodules
-    DIR="."
-  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
-    if ! check-files "$DIR"; then
-      echo "Fatal: could not locate known files in ${PWD}/${DIR}"
+  if [ ! -d "$SUBMODULES" ] ; then # pwd must outside of pmm-submodules
+    echo "Warn: the current working directory must be outside of pmm-submodules"
+    echo "cd .."
+    cd .. > /dev/null
+  fi
+
+  if [ -d "$SUBMODULES" ]; then # pwd is outside pmm-submodules
+    if ! check-files "$SUBMODULES"; then
+      echo "Fatal: could not locate known files in ${PWD}/${SUBMODULES}"
       exit 1
     fi
   else
@@ -145,7 +154,7 @@ update() {
     exit 1
   fi
 
-  cd "$DIR"
+  cd "$SUBMODULES"
 
   # Join the dependencies from ci-default.yml and ci.yml
   DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
@@ -186,13 +195,17 @@ get_branch_name() {
 }
 
 build_with_logs() {
+  local CURDIR="$PWD"
   local script="$PATH_TO_SCRIPTS/$1"
   local start_time
   local end_time
   local script_name="$1"
 
+  cd "$SUBMODULES" > /dev/null
+
   if [ ! -f "$script" ]; then
     echo "Fatal: script $script does not exist"
+    cd "$CURDIR" > /dev/null
     exit 1
   fi
 
@@ -209,43 +222,82 @@ build_with_logs() {
   echo ---
   echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee -a $LOG_FILE
   echo ---
+
+  cd "$CURDIR" > /dev/null
 }
 
 purge_files() {
+  local CURDIR=$PWD
   local tmp_files
+
+  cd "$SUBMODULES" > /dev/null
   # Remove stale files and directories
   if [ -d tmp ]; then
     echo "Removing stale files and directories..."
+
     if [ -d "tmp/pmm-server" ]; then
       tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$" || :)
-      tmp_files=($tmp_files)
-      for f in "${tmp_files[@]}"; do
-        rm -rf "$f"
-      done
+      if [ -n "$tmp_files" ]; then
+        tmp_files=( $tmp_files )
+        for f in "${tmp_files[@]}"; do
+          rm -rf "$f"
+        done
+      fi
     fi
+
     if [ -d "tmp/source/pmm" ]; then
       rm -rf tmp/source/pmm
     fi
   fi
+
   if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file..."
     rm -f $LOG_FILE
   fi
+
+  cd "$CURDIR"
 }
 
 init() {
-  PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
-  "${PATH_TO_SCRIPTS}/build-submodules"
+  local CURDIR="$PWD"
+
   export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
+  if [ -d "$SUBMODULES" ]; then
+    cd "$SUBMODULES" > /dev/null
+  fi
+
+  pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+  pmm_branch=$(git config -f .gitmodules submodule.pmm.branch)
+  pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+  pmm_qa_branch=$(git config -f .gitmodules submodule.pmm-qa.branch)
+  pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+  pmm_ui_tests_branch=$(git config -f .gitmodules submodule.pmm-ui-tests.branch)
+  pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
+  fb_commit_sha=$(git rev-parse HEAD)
+
+  echo $fb_commit_sha > fbCommitSha
+  echo $pmm_commit > apiCommitSha
+  echo $pmm_branch > apiBranch
+  echo $pmm_url > apiURL
+  echo $pmm_qa_branch > pmmQABranch
+  echo $pmm_qa_commit > pmmQACommitSha
+  echo $pmm_ui_tests_branch > pmmUITestBranch
+  echo $pmm_ui_tests_commit > pmmUITestsCommitSha
+
   # Create cache directories. Read more in the section about `rpmbuild`.
-  test -d "${root_dir}/go-path" || mkdir -p "go-path"
-  test -d "${root_dir}/go-build" || mkdir -p "go-build"
-  test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
-  test -d "${root_dir}/yum-cache" || mkdir -p "yum-cache"
+  test -d "go-path" || mkdir -p "go-path"
+  test -d "go-build" || mkdir -p "go-build"
+  test -d "yarn-cache" || mkdir -p "yarn-cache"
+  test -d "yum-cache" || mkdir -p "yum-cache"
+
+  cd "$CURDIR" > /dev/null
 }
 
 cleanup() {
+  local CURDIR="$PWD"
+  cd "$SUBMODULES" > /dev/null
+
   # Clean up temporary files
   rm -f apiBranch \
     apiCommitSha \
@@ -254,13 +306,15 @@ cleanup() {
     pmmQABranch \
     pmmQACommitSha \
     pmmUITestBranch \
-    pmmUITestsCommitSha
+    pmmUITestsCommitSha || :
+
+  cd "$CURDIR" > /dev/null
 }
 
 main() {
   if [ "$NO_UPDATE" -eq 0 ]; then
     MD5SUM=$(md5sum $(dirname $0)/build.sh)
-    
+
     # Update submodules and PR branches
     update
 
@@ -274,6 +328,7 @@ main() {
   fi
 
   init
+
   purge_files
 
   if [ "$NO_CLIENT" -eq 0 ]; then
@@ -301,7 +356,6 @@ main() {
   # total time: 6m26s - build from scratch, no initial cache
   # total time: 2m49s - subsequent build, using cache from prior builds
 
-
   # Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
   # total time: 8m45s - build from scratch, no initial cache
   # total time: ??? - subsequent build, using cache from prior builds
@@ -335,9 +389,11 @@ main() {
   cleanup
 }
 
-# Local reference test environment
+# Reference test environment
 # CPU: 4 cores
-# RAM: 16GB
+# RAM: 16 GB
 # OS: Ubuntu 22.04.1 LTS
 
+parse-params "$@"
+
 main
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 005fbee466..dd096f5c31 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -34,10 +34,11 @@ main() {
       DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
   fi
 
-  IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+  local IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+  local BUILD_DATE=$(date -u +'%F %T%z' | sed 's@^.\{22\}@&:@')
 
   docker buildx build \
-      --build-arg BUILD_DATE="$(date --rfc-3339=seconds)" \
+      --build-arg BUILD_DATE="$BUILD_DATE" \
       --build-arg VERSION="$IMAGE_VERSION" \
       --progress plain \
       -f ${docker_root}/${docker_file} \

From c2357c5d4674af6ae05cda4f5cbc831482cab688 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 20 Oct 2024 03:06:33 +0300
Subject: [PATCH 34/78] PMM-12899 fix git commit variable

---
 build/local/build.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 0811217ca9..723945780d 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -267,6 +267,8 @@ init() {
     cd "$SUBMODULES" > /dev/null
   fi
 
+  GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
+
   pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
   pmm_branch=$(git config -f .gitmodules submodule.pmm.branch)
   pmm_url=$(git config -f .gitmodules submodule.pmm.url)
@@ -313,6 +315,7 @@ cleanup() {
 
 main() {
   if [ "$NO_UPDATE" -eq 0 ]; then
+    local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build.sh"
     MD5SUM=$(md5sum $(dirname $0)/build.sh)
 
     # Update submodules and PR branches
@@ -320,9 +323,10 @@ main() {
 
     test "$UPDATE_ONLY" -eq 1 && return
 
-    if [ "$MD5SUM" != "$(md5sum $(dirname $0)/build.sh)" ]; then
-      echo "The updated version of this script has been fetched from the repository, exiting..."
-      echo "Please run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+    if [ -f "$UPDATED_SCRIPT" ] && [ "$MD5SUM" != "$(md5sum $UPDATED_SCRIPT)" ]; then
+      echo "The local copy of this script differs from the one fetched from the repo." 
+      echo "Apparently, that version is newer. We will halt to give you the change to run a fresh version."
+      echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
       return
     fi
   fi
@@ -346,7 +350,6 @@ main() {
   fi
 
   # Building client docker image takes 17s
-  GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
   export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
     build_with_logs build-client-docker

From 79250792ebf331db5edfc8a36550d98300658736 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 20 Oct 2024 15:28:58 +0300
Subject: [PATCH 35/78] PMM-12899 don't use S3 cache for local builds

---
 build/local/build.sh | 1 +
 build/scripts/vars   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 723945780d..e52e60b3d5 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -364,6 +364,7 @@ main() {
   # total time: ??? - subsequent build, using cache from prior builds
 
   export RPM_EPOCH=1
+  export FORCE_REBUILD=1 # Don't use S3 cache
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
     build_with_logs build-server-rpm percona-dashboards grafana-dashboards
     build_with_logs build-server-rpm pmm-managed pmm
diff --git a/build/scripts/vars b/build/scripts/vars
index 338e6512e2..7a7d15fb3b 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -38,7 +38,7 @@ unset new_pmm_version
 unset new_pmm_release
 echo -e "\n\n\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release}\n\n\n"
 
-rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:2}
+rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
 rpmbuild_dir=${root_dir}/sources/pmm/src/github.com/percona/pmm/build/packages/rpm/server
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}

From 1d8fe377ed61051b77fe33904d1be98a744758ac Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 14:34:27 +0300
Subject: [PATCH 36/78] PMM-12899 dont chown with an id external to the
 container

---
 build/docker/server/Dockerfile.el9 | 1 -
 build/scripts/build-client-binary  | 2 +-
 build/scripts/build-server-rpm     | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9
index 9131565bb4..8f2009bc44 100644
--- a/build/docker/server/Dockerfile.el9
+++ b/build/docker/server/Dockerfile.el9
@@ -4,7 +4,6 @@ ARG VERSION
 ARG BUILD_DATE
 
 ENV LANG=en_US.utf8
-ENV LC_ALL=en_US.utf8
 ENV GF_PLUGIN_DIR=/srv/grafana/plugins
 ENV PS1="[\u@\h \W] # "
 
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 8fdacdd3d8..96c7b9598b 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -90,7 +90,7 @@ gobuild_component() {
 
         ${build_command}
 
-        sudo chown -R $(id -u):$(id -g) /home/builder/tmp
+        # sudo chown -R $(id -u):$(id -g) /home/builder/tmp
     "
 }
 
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 9d587fe16d..ba56942b3d 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -56,7 +56,7 @@ prepare_specs() {
     local spec_name=$1
     local repo_name=$2
 
-    sudo chown -R $(id -u):$(id -g) ${rpmbuild_dir}/SPECS ${rpmbuild_dir}/SOURCES
+    # sudo chown -R $(id -u):$(id -g) ${rpmbuild_dir}/SPECS ${rpmbuild_dir}/SOURCES
     cp ${rpmbuild_dir}/SPECS/${spec_name}.spec ${rpmbuild_dir}/SOURCES/${spec_name}.spec
     if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
         return

From 48d481fafc32c8ae49add743683626c823054e9c Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 14:35:34 +0300
Subject: [PATCH 37/78] PMM-12899 refactor the log output

---
 build/local/build.sh | 51 ++++++++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 23 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index e52e60b3d5..24fdc89490 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -194,17 +194,17 @@ get_branch_name() {
   echo $branch_name
 }
 
-build_with_logs() {
+run_build_script() {
   local CURDIR="$PWD"
   local script="$PATH_TO_SCRIPTS/$1"
+  local script_name="$1"
   local start_time
   local end_time
-  local script_name="$1"
 
   cd "$SUBMODULES" > /dev/null
 
   if [ ! -f "$script" ]; then
-    echo "Fatal: script $script does not exist"
+    echo "Fatal: script $script does not exist."
     cd "$CURDIR" > /dev/null
     exit 1
   fi
@@ -213,14 +213,14 @@ build_with_logs() {
   if [ "$#" -gt 1 ]; then
     shift
     script_name="${script_name}:($1)"
-    $script "$@" | tee -a $LOG_FILE
+    $script "$@"
   else
-    $script | tee -a $LOG_FILE
+    $script
   fi
   end_time=$(date +%s)
 
   echo ---
-  echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee -a $LOG_FILE
+  echo "Execution time (in sec) for $script_name: $((end_time - start_time))"
   echo ---
 
   cd "$CURDIR" > /dev/null
@@ -240,19 +240,21 @@ purge_files() {
       if [ -n "$tmp_files" ]; then
         tmp_files=( $tmp_files )
         for f in "${tmp_files[@]}"; do
+          echo "Removing file or directory $f ..."
           rm -rf "$f"
         done
       fi
     fi
 
     if [ -d "tmp/source/pmm" ]; then
+      echo "Removing tmp/source/pmm ..."
       rm -rf tmp/source/pmm
     fi
   fi
 
   if [ -f "$LOG_FILE" ]; then
-    echo "Removing the log file..."
-    rm -f $LOG_FILE
+    echo "Removing the log file $LOG_FILE ..."
+    rm -f "$LOG_FILE"
   fi
 
   cd "$CURDIR"
@@ -337,22 +339,22 @@ main() {
 
   if [ "$NO_CLIENT" -eq 0 ]; then
     # Build client source: 4m39s from scratch, 0m27s using cache
-    build_with_logs build-client-source
+    run_build_script build-client-source
 
     # Build client binary: ??? from scratch, 0m20s using cache
-    build_with_logs build-client-binary
+    run_build_script build-client-binary
 
     # Building client source rpm takes 13s (caching does not apply)
-    # build_with_logs build-client-srpm
+    # run_build_script build-client-srpm
 
     # Building client rpm takes 1m40s
-    # build_with_logs build-client-rpm
+    # run_build_script build-client-rpm
   fi
 
   # Building client docker image takes 17s
   export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
-    build_with_logs build-client-docker
+    run_build_script build-client-docker
   fi
 
   # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
@@ -366,28 +368,28 @@ main() {
   export RPM_EPOCH=1
   export FORCE_REBUILD=1 # Don't use S3 cache
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
-    build_with_logs build-server-rpm percona-dashboards grafana-dashboards
-    build_with_logs build-server-rpm pmm-managed pmm
-    build_with_logs build-server-rpm percona-qan-api2 pmm
-    build_with_logs build-server-rpm pmm-update pmm
-    build_with_logs build-server-rpm pmm-dump
-    build_with_logs build-server-rpm vmproxy pmm
+    run_build_script build-server-rpm percona-dashboards grafana-dashboards
+    run_build_script build-server-rpm pmm-managed pmm
+    run_build_script build-server-rpm percona-qan-api2 pmm
+    run_build_script build-server-rpm pmm-update pmm
+    run_build_script build-server-rpm pmm-dump
+    run_build_script build-server-rpm vmproxy pmm
 
     # 3rd-party
-    build_with_logs build-server-rpm victoriametrics
-    build_with_logs build-server-rpm grafana
+    run_build_script build-server-rpm victoriametrics
+    run_build_script build-server-rpm grafana
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
     export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT}
     export DOCKERFILE=Dockerfile.el9
-    build_with_logs build-server-docker
+    run_build_script build-server-docker
   fi
 
   echo
   echo "Done building PMM artifacts."
   echo ---
-  echo "Total execution time, sec: $(($(date +%s) - $START_TIME))" | tee -a $LOG_FILE
+  echo "Total execution time, sec: $(($(date +%s) - $START_TIME))"
   echo ---
 
   cleanup
@@ -400,4 +402,7 @@ main() {
 
 parse-params "$@"
 
+# Capture the output in the log file
+exec > >(tee -a "$LOG_FILE") 2>&1
+
 main

From e8e3f0f4c25d48503e3468b601b142f9cca66f6f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 15:45:28 +0300
Subject: [PATCH 38/78] PMM-12899 don't chown with builder id outside of
 container

---
 build/scripts/build-server-rpm | 2 --
 1 file changed, 2 deletions(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index ba56942b3d..a123f2bd0a 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -123,8 +123,6 @@ build() {
         volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
     fi
 
-    sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
-
     echo "Start building Server RPMs..."
     echo "spec_name: ${spec_name}"
     echo "repo_name: ${repo_name}"

From 344b783238de2510efe53b30fdf976ee0d5d44d1 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 16:26:25 +0300
Subject: [PATCH 39/78] PMM-12899 add aarch to grafana.spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index f2de9a6970..7a1f0ea221 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -19,7 +19,7 @@ Summary:        Grafana is an open source, feature rich metrics dashboard and gr
 License:        AGPLv3
 URL:            https://github.com/percona/grafana
 Source0:        https://github.com/percona/grafana/archive/%{commit}.tar.gz
-ExclusiveArch:  %{ix86} x86_64 %{arm}
+ExclusiveArch:  %{ix86} x86_64 %{arm} %{aarch}
 
 BuildRequires: fontconfig
 

From 7aa9a307956b545359cfee8427bc4f62976009aa Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 16:39:09 +0300
Subject: [PATCH 40/78] PMM-12899 add aarch64 to grafana.spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 7a1f0ea221..2c48a7f35e 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -19,7 +19,7 @@ Summary:        Grafana is an open source, feature rich metrics dashboard and gr
 License:        AGPLv3
 URL:            https://github.com/percona/grafana
 Source0:        https://github.com/percona/grafana/archive/%{commit}.tar.gz
-ExclusiveArch:  %{ix86} x86_64 %{arm} %{aarch}
+ExclusiveArch:  %{ix86} x86_64 %{arm} aarch64
 
 BuildRequires: fontconfig
 

From f28ecdeaa2d44a56ac966d99b6da9319693b4d70 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 19:15:06 +0300
Subject: [PATCH 41/78] PMM-12899 add the platform

---
 build/docker/rpmbuild/Dockerfile.el9 |  5 +++--
 build/local/build.sh                 | 10 ++++++++++
 build/scripts/build-client-binary    |  1 +
 build/scripts/build-server-docker    |  2 +-
 build/scripts/build-server-rpm       |  2 +-
 build/scripts/vars                   |  1 +
 6 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/build/docker/rpmbuild/Dockerfile.el9 b/build/docker/rpmbuild/Dockerfile.el9
index 239f73be9c..88373bb787 100644
--- a/build/docker/rpmbuild/Dockerfile.el9
+++ b/build/docker/rpmbuild/Dockerfile.el9
@@ -28,8 +28,9 @@ RUN yum install -y gcc gcc-c++ \
     yum clean all && rm -rf /var/cache/yum
 
 # keep that format for easier search
-ENV GO_VERSION 1.23.2
-ENV GO_RELEASER_VERSION 1.24.0
+ENV GO_VERSION=1.23.2
+ENV GO_RELEASER_VERSION=1.24.0
+ENV NODE_OPTIONS=--max_old_space_size=8000
 
 RUN if [ `uname -i` == "x86_64" ]; then ARCH=amd64; else ARCH=arm64; fi && \
     curl -fSsL -o /tmp/golang.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-${ARCH}.tar.gz && \
diff --git a/build/local/build.sh b/build/local/build.sh
index 24fdc89490..9298938d3a 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -5,6 +5,7 @@ set -o nounset
 usage() {
   cat <<-EOF
 Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
+--platform <platform>    Build for a specific platform (default - linux/amd64)
 --no-update              Do not fetch the latest changes from the repo
 --update-only            Only fetch the latest changes from the repo
 --no-client              Do not build PMM Client
@@ -27,6 +28,7 @@ parse-params() {
   START_TIME=$(date +%s)
   LOG_FILE="$(dirname $0)/build.log"
   BASE_NAME=$(basename $0)
+  PLATFORM=linux/amd64
   SUBMODULES=pmm-submodules
   PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
 
@@ -58,6 +60,14 @@ parse-params() {
         fi
         NO_SERVER_DOCKER=1
         ;;
+      --platform)
+        shift
+        if [ -z "$1" ]; then
+          echo "Missing argument for --platform"
+          exit 1
+        fi
+        PLATFORM="$1"
+        ;;
       --log-file)
         shift
         if [ -z "$1" ]; then
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 96c7b9598b..a811da16fc 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -59,6 +59,7 @@ gobuild_component() {
     fi
 
     docker run --rm \
+      --platform=${platform}
       ${volume_mounts} \
       ${rpmbuild_docker_image} sh -c "
         set -o errexit
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index dd096f5c31..e3a3953f01 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -22,7 +22,7 @@ main() {
 
   git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
   ls -la ${rpms_dir}
-  docker run --rm -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
+  docker run --rm --patform=${platform} -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
       sudo chown -R builder /home/builder/rpm/RPMS
       until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
           echo "waiting"
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index a123f2bd0a..e9543af810 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -128,7 +128,7 @@ build() {
     echo "repo_name: ${repo_name}"
     echo "rpm_verison: ${rpm_version}"
 
-    docker run --rm ${volume_mounts} ${rpmbuild_docker_image} sh -c "
+    docker run --rm --platform=${platform} ${volume_mounts} ${rpmbuild_docker_image} sh -c "
         set -o errexit
         set -o xtrace
 
diff --git a/build/scripts/vars b/build/scripts/vars
index 7a7d15fb3b..fac887e0f4 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -38,6 +38,7 @@ unset new_pmm_version
 unset new_pmm_release
 echo -e "\n\n\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release}\n\n\n"
 
+platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
 rpmbuild_dir=${root_dir}/sources/pmm/src/github.com/percona/pmm/build/packages/rpm/server

From 87d75a7faccd86e30dedf178c62174e5909cf4ef Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 20:39:36 +0300
Subject: [PATCH 42/78] PMM-12899 use docker volumes for caching

---
 build/local/build.sh                  | 39 ++++++++++++++++-----------
 build/scripts/build-client-binary     |  9 +++----
 build/scripts/build-server-rpm        | 15 ++++-------
 build/scripts/build-server-rpm-script |  2 +-
 4 files changed, 33 insertions(+), 32 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 9298938d3a..606c07e2ff 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -261,13 +261,13 @@ purge_files() {
       rm -rf tmp/source/pmm
     fi
   fi
+  
+  cd "$CURDIR"
 
   if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file $LOG_FILE ..."
     rm -f "$LOG_FILE"
   fi
-
-  cd "$CURDIR"
 }
 
 init() {
@@ -299,11 +299,20 @@ init() {
   echo $pmm_ui_tests_branch > pmmUITestBranch
   echo $pmm_ui_tests_commit > pmmUITestsCommitSha
 
-  # Create cache directories. Read more in the section about `rpmbuild`.
-  test -d "go-path" || mkdir -p "go-path"
-  test -d "go-build" || mkdir -p "go-build"
-  test -d "yarn-cache" || mkdir -p "yarn-cache"
-  test -d "yum-cache" || mkdir -p "yum-cache"
+  # Create docker volume to persist package and build cache
+  # Read more in the section about `rpmbuild`.
+  if ! docker volume ls | grep pmm-gobuild; then
+    docker volume create pmm-gobuild
+  fi
+  if ! docker volume ls | grep pmm-gomod; then
+    docker volume create pmm-gomod
+  fi
+  if ! docker volume ls | grep pmm-yarn; then
+    docker volume create pmm-yarn
+  fi
+  if ! docker volume ls | grep pmm-dnf; then
+    docker volume create pmm-dnf
+  fi
 
   cd "$CURDIR" > /dev/null
 }
@@ -373,21 +382,21 @@ main() {
 
   # Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
   # total time: 8m45s - build from scratch, no initial cache
-  # total time: ??? - subsequent build, using cache from prior builds
+  # total time: N/A - subsequent build, using cache from prior builds
 
   export RPM_EPOCH=1
   export FORCE_REBUILD=1 # Don't use S3 cache
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
     run_build_script build-server-rpm percona-dashboards grafana-dashboards
-    run_build_script build-server-rpm pmm-managed pmm
-    run_build_script build-server-rpm percona-qan-api2 pmm
-    run_build_script build-server-rpm pmm-update pmm
-    run_build_script build-server-rpm pmm-dump
-    run_build_script build-server-rpm vmproxy pmm
+    # run_build_script build-server-rpm pmm-managed pmm
+    # run_build_script build-server-rpm percona-qan-api2 pmm
+    # run_build_script build-server-rpm pmm-update pmm
+    # run_build_script build-server-rpm pmm-dump
+    # run_build_script build-server-rpm vmproxy pmm
 
     # 3rd-party
-    run_build_script build-server-rpm victoriametrics
-    run_build_script build-server-rpm grafana
+    # run_build_script build-server-rpm victoriametrics
+    # run_build_script build-server-rpm grafana
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index a811da16fc..673b8af401 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -31,12 +31,9 @@ gobuild_component() {
       local GOARCH=$(uname -m)
     fi
 
-    if [ -d "${root_dir}/go-build" ]; then
-        volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
-    fi
-    if [ -d "${root_dir}/go-path" ]; then
-        volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
-    fi
+    volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
+    volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
+    volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
 
     if [ ${component} = 'azure_metrics_exporter' ]; then
         result_file=${result_file/_metrics/}
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index e9543af810..fa46adbf4f 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -113,15 +113,10 @@ build() {
     echo "Building PMM Server RPM for ${spec_name} component..."
     echo -------------------------------------------------------
 
-    if [ -d "${root_dir}/go-build" ]; then
-        volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
-    fi
-    if [ -d "${root_dir}/yarn-cache" ]; then
-        volume_mounts+=" -v ${root_dir}/yarn-cache:/home/builder/.yarn"
-    fi
-    if [ -d "${root_dir}/go-path" ]; then
-        volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
-    fi
+    volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
+    volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
+    volume_mounts+=" -v pmm-yarn:/home/builder/.yarn"
+    volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
 
     echo "Start building Server RPMs..."
     echo "spec_name: ${spec_name}"
@@ -132,7 +127,7 @@ build() {
         set -o errexit
         set -o xtrace
 
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache /home/builder/go
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/go
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
         if [ ${RPM_EPOCH} = 1 ]; then
diff --git a/build/scripts/build-server-rpm-script b/build/scripts/build-server-rpm-script
index cece634cdc..02a899e8a8 100755
--- a/build/scripts/build-server-rpm-script
+++ b/build/scripts/build-server-rpm-script
@@ -5,7 +5,7 @@ set -o xtrace
 
 . $(dirname $0)/vars
 
-sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache
+sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
 
 # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
 if [ ${RPM_EPOCH} = 1 ]; then

From 15d9344371684ea37440bd3f9167e4f675da596d Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 21:04:59 +0300
Subject: [PATCH 43/78] PMM-12899 exit if the spec cannot be found

---
 build/scripts/build-client-binary | 4 ----
 build/scripts/build-server-rpm    | 7 +++++--
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 673b8af401..035e1c15a0 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -65,8 +65,6 @@ gobuild_component() {
         export GOARCH=${GOARCH:-amd64}
         export CGO_ENABLED=0
         export GO111MODULE=auto
-        export GOPATH=/tmp/go
-        export GOMODCACHE=/home/builder/go/pkg/mod
 
         # Setting variable for 'make release'
         export PMM_RELEASE_VERSION=${full_pmm_version}
@@ -87,8 +85,6 @@ gobuild_component() {
         tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf ${src_dir}/${archive}
 
         ${build_command}
-
-        # sudo chown -R $(id -u):$(id -g) /home/builder/tmp
     "
 }
 
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index fa46adbf4f..28b526f1cb 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -61,6 +61,9 @@ prepare_specs() {
     if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
         return
     fi
+    if [ ! -f "${rpmbuild_dir}/SOURCES/${spec_name}.spec" ]; then
+      echo "The spec file could not be found."
+    fi
     
     local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
     local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
@@ -127,7 +130,7 @@ build() {
         set -o errexit
         set -o xtrace
 
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/go
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
         if [ ${RPM_EPOCH} = 1 ]; then
@@ -146,7 +149,7 @@ build() {
 
         # Only these two specs have build dependencies
         if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
-            sleep 5s
+            sleep 2s
             sudo yum-builddep -y SOURCES/${spec_name}.spec
         fi
 

From 5a8f6e4b533f3c468e4533eb6a6a48bd68658c69 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 22:26:37 +0300
Subject: [PATCH 44/78] PMM-12899 fix yarn cache

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 28b526f1cb..6ce1f8878d 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -118,7 +118,7 @@ build() {
 
     volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
     volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
-    volume_mounts+=" -v pmm-yarn:/home/builder/.yarn"
+    volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
     volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
 
     echo "Start building Server RPMs..."

From e326d9491795b1b4db9829b930418ff41d11867b Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 22:33:46 +0300
Subject: [PATCH 45/78] PMM-12899 fix the platform

---
 build/scripts/build-client-binary | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 035e1c15a0..b064f6622f 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -56,7 +56,7 @@ gobuild_component() {
     fi
 
     docker run --rm \
-      --platform=${platform}
+      --platform=${platform} \
       ${volume_mounts} \
       ${rpmbuild_docker_image} sh -c "
         set -o errexit

From 9010f426fe778bc8c0a7b1f4c6c3d1f131060144 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 23:32:21 +0300
Subject: [PATCH 46/78] PMM-12899 chown the cache dirs before starting

---
 build/local/build.sh              |  5 -----
 build/scripts/build-client-binary | 12 +++++++-----
 build/scripts/build-server-docker |  1 +
 build/scripts/build-server-rpm    |  9 +++++++++
 4 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 606c07e2ff..acd9b9e3fc 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -263,11 +263,6 @@ purge_files() {
   fi
   
   cd "$CURDIR"
-
-  if [ -f "$LOG_FILE" ]; then
-    echo "Removing the log file $LOG_FILE ..."
-    rm -f "$LOG_FILE"
-  fi
 }
 
 init() {
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index b064f6622f..56ce6e582f 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -27,10 +27,6 @@ gobuild_component() {
     local archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz" | head -n1)
     local build_command
 
-    if [ "$(uname -o)" = "Darwin" ]; then
-      local GOARCH=$(uname -m)
-    fi
-
     volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
     volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
     volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
@@ -62,7 +58,7 @@ gobuild_component() {
         set -o errexit
         set -o xtrace
         export GOOS=${GOOS:-linux}
-        export GOARCH=${GOARCH:-amd64}
+        export GOARCH=${GOARCH:-$(uname -m)}
         export CGO_ENABLED=0
         export GO111MODULE=auto
 
@@ -71,6 +67,12 @@ gobuild_component() {
         export PMM_RELEASE_TIMESTAMP=$(date '+%s')
         export PMM_RELEASE_BRANCH=""
         sudo chown -R builder:builder /home/builder/tmp
+        if [ ! -w /home/builder/.cache/go-build ]; then
+            sudo chown builder /home/builder/.cache/go-build
+        fi
+        if [ ! -w /home/builder/go/pkg/mod ]; then
+            sudo chown builder /home/builder/go/pkg/mod
+        fi
 
         export PMM_RELEASE_PATH=${bin_dir}/bin
         if [ "${component}" = vmagent ]; then
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index e3a3953f01..0e4b2556a2 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -40,6 +40,7 @@ main() {
   docker buildx build \
       --build-arg BUILD_DATE="$BUILD_DATE" \
       --build-arg VERSION="$IMAGE_VERSION" \
+      --platform=${platform} \
       --progress plain \
       -f ${docker_root}/${docker_file} \
       -t ${DOCKER_TAG} \
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 6ce1f8878d..2c8ff8af42 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -131,6 +131,15 @@ build() {
         set -o xtrace
 
         sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+        if [ ! -w /home/builder/.cache/go-build ]; then
+            sudo chown builder /home/builder/.cache/go-build
+        fi
+        if [ ! -w /home/builder/go/pkg/mod ]; then
+            sudo chown builder /home/builder/go/pkg/mod
+        fi
+        if [ ! -w pmm-yarn:/home/builder/.cache/yarn ]; then
+            sudo chown builder /home/builder/.cache/yarn
+        fi
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
         if [ ${RPM_EPOCH} = 1 ]; then

From 0ff180fde917b4643dc8e0ce48f74a3c05ef55c3 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 23:39:35 +0300
Subject: [PATCH 47/78] PMM-12899 remove GOARCH setting

---
 build/scripts/build-client-binary | 2 --
 1 file changed, 2 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 56ce6e582f..9ec917fc08 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -57,8 +57,6 @@ gobuild_component() {
       ${rpmbuild_docker_image} sh -c "
         set -o errexit
         set -o xtrace
-        export GOOS=${GOOS:-linux}
-        export GOARCH=${GOARCH:-$(uname -m)}
         export CGO_ENABLED=0
         export GO111MODULE=auto
 

From c9d8e3522c4d950cbb5bb34e5b3aabd1d281bd55 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 23:43:45 +0300
Subject: [PATCH 48/78] PMM-12899 add --platform to --help

---
 build/local/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index acd9b9e3fc..f588e1fc75 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -4,7 +4,7 @@ set -o nounset
 
 usage() {
   cat <<-EOF
-Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
+Usage: $BASE_NAME [--platform] [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
 --platform <platform>    Build for a specific platform (default - linux/amd64)
 --no-update              Do not fetch the latest changes from the repo
 --update-only            Only fetch the latest changes from the repo
@@ -417,6 +417,6 @@ main() {
 parse-params "$@"
 
 # Capture the output in the log file
-exec > >(tee -a "$LOG_FILE") 2>&1
+exec > >(tee "$LOG_FILE") 2>&1
 
 main

From 46c893443ff0e28355e8e00a0db852a451478355 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 2 Nov 2024 00:27:28 +0300
Subject: [PATCH 49/78] PMM-12899 do not upload to S3 when building locally

---
 build/local/build.sh           |  3 ++-
 build/scripts/build-server-rpm | 11 ++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index f588e1fc75..7b8bf328f2 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -19,6 +19,7 @@ EOF
 
 parse-params() {
   # Define global variables
+  LOCAL_BUILD=1
   NO_UPDATE=0
   UPDATE_ONLY=0
   NO_CLIENT=0
@@ -148,7 +149,7 @@ update() {
   local CURDIR="$PWD"
 
   # Thouroughly verify the presence of known files, otherwise bail out
-  if [ ! -d "$SUBMODULES" ] ; then # pwd must outside of pmm-submodules
+  if [ ! -d "$SUBMODULES" ] ; then # pwd must be outside of pmm-submodules
     echo "Warn: the current working directory must be outside of pmm-submodules"
     echo "cd .."
     cd .. > /dev/null
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 2c8ff8af42..a5a8cbb2d8 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -27,9 +27,7 @@ is_build_needed() {
     local s3_cache_dir=RELEASE
 
     # Structure of S3 build cache
-    # s3://pmm-build-cache/PR-BUILDS/7 - el7
     # s3://pmm-build-cache/PR-BUILDS/9 - el9
-    # s3://pmm-build-cache/RELEASE/7 - el7
     # s3://pmm-build-cache/RELEASE/9 - el9
 
     [ "${FORCE_REBUILD}" = 1 ] && return 0
@@ -38,7 +36,7 @@ is_build_needed() {
     if [ "$RPM_EPOCH" = 1 ]; then
         s3_cache_dir=PR-BUILDS
     fi
-    if command -v aws &> /dev/null; then
+    if command -v aws &> /dev/null && [ -z "$LOCAL_BUILD" ]; then
         aws s3 sync \
             --region us-east-2 \
             --no-sign-request \
@@ -175,8 +173,15 @@ build() {
     # TODO: see if this is even needed
     # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
 
+    if [ -n "$LOCAL_BUILD" ]; then
+        return
+        echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
+    fi
+
     if ! command -v aws &> /dev/null; then
         echo "AWS CLI is not installed. Upload to S3 will be skipped."
+        echo
+        echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
         return
     fi
 

From 10ea625bf6a2365129aec3b7ea291acc092519f8 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 2 Nov 2024 00:33:02 +0300
Subject: [PATCH 50/78] PMM-12899 fix a typo

---
 build/scripts/build-server-docker | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 0e4b2556a2..6cb0c7ed70 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -22,7 +22,7 @@ main() {
 
   git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
   ls -la ${rpms_dir}
-  docker run --rm --patform=${platform} -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
+  docker run --rm --platform=${platform} -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
       sudo chown -R builder /home/builder/rpm/RPMS
       until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
           echo "waiting"

From 891a2f0d249aa64bb8e64d5358e4f9e1c533725e Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 3 Nov 2024 14:50:16 +0300
Subject: [PATCH 51/78] PMM-12899 clean up scripts

---
 build/local/build.sh                | 24 ++++++++++++------------
 build/scripts/build-client-packages | 14 +++-----------
 build/scripts/build-client-rpm      | 17 +++++------------
 build/scripts/build-client-srpm     | 12 +-----------
 build/scripts/build-server-rpm      | 17 +++++++++--------
 5 files changed, 30 insertions(+), 54 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 7b8bf328f2..9530bf96b6 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -297,16 +297,16 @@ init() {
 
   # Create docker volume to persist package and build cache
   # Read more in the section about `rpmbuild`.
-  if ! docker volume ls | grep pmm-gobuild; then
+  if ! docker volume ls | grep pmm-gobuild >/dev/null; then
     docker volume create pmm-gobuild
   fi
-  if ! docker volume ls | grep pmm-gomod; then
+  if ! docker volume ls | grep pmm-gomod >/dev/null; then
     docker volume create pmm-gomod
   fi
-  if ! docker volume ls | grep pmm-yarn; then
+  if ! docker volume ls | grep pmm-yarn >/dev/null; then
     docker volume create pmm-yarn
   fi
-  if ! docker volume ls | grep pmm-dnf; then
+  if ! docker volume ls | grep pmm-dnf >/dev/null; then
     docker volume create pmm-dnf
   fi
 
@@ -367,7 +367,7 @@ main() {
   fi
 
   # Building client docker image takes 17s
-  export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
+  export DOCKER_CLIENT_TAG=percona/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
     run_build_script build-client-docker
   fi
@@ -384,15 +384,15 @@ main() {
   export FORCE_REBUILD=1 # Don't use S3 cache
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
     run_build_script build-server-rpm percona-dashboards grafana-dashboards
-    # run_build_script build-server-rpm pmm-managed pmm
-    # run_build_script build-server-rpm percona-qan-api2 pmm
-    # run_build_script build-server-rpm pmm-update pmm
-    # run_build_script build-server-rpm pmm-dump
-    # run_build_script build-server-rpm vmproxy pmm
+    run_build_script build-server-rpm pmm-managed pmm
+    run_build_script build-server-rpm percona-qan-api2 pmm
+    run_build_script build-server-rpm pmm-update pmm
+    run_build_script build-server-rpm pmm-dump
+    run_build_script build-server-rpm vmproxy pmm
 
     # 3rd-party
-    # run_build_script build-server-rpm victoriametrics
-    # run_build_script build-server-rpm grafana
+    run_build_script build-server-rpm victoriametrics
+    run_build_script build-server-rpm grafana
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index bc6e1a0244..f578a422d0 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -30,12 +30,6 @@ append_arg_to_args () {
 }
 
 parse_arguments() {
-    pick_args=
-    if test "$1" = PICK-ARGS-FROM-ARGV; then
-        pick_args=1
-        shift
-    fi
-
     for arg do
         val=`echo "$arg" | sed -e 's;^--[^=]*=;;'`
         optname=`echo "$arg" | sed -e 's/^\(--[^=]*\)=.*$/\1/'`
@@ -51,9 +45,7 @@ parse_arguments() {
             --install_deps=*) INSTALL="$val" ;;
             --help) usage ;;
             *)
-              if test -n "$pick_args"; then
-                  append_arg_to_args "$arg"
-              fi
+                append_arg_to_args "$arg"
               ;;
         esac
     done
@@ -87,7 +79,7 @@ get_repos() {
 }
 
 check_workdir(){
-    if [ "x$WORKDIR" = "x$CURDIR" ]; then
+    if [ "$WORKDIR" = "$CURDIR" ]; then
         echo >&2 "Current directory cannot be used for building!"
         exit 1
     fi
@@ -461,7 +453,7 @@ if [ -n "$pmm_release" ]; then
     DEB_RELEASE="$DEB_RELEASE.$pmm_release"
 fi
 
-parse_arguments PICK-ARGS-FROM-ARGV "$@"
+parse_arguments "$@"
 get_system
 install_deps
 
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index 517993a868..374b79c5ac 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -11,18 +11,13 @@ main() {
     echo -----------------------------------------
 
     local IMAGE=${1:-${rpmbuild_docker_image}}
-    docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
+    docker run --rm \
+      -v ${bin_dir}:/home/builder/bin \
+      -v ${root_dir}/results:/home/builder/results \
+      ${IMAGE} sh -c "
         set -o errexit
         set -o xtrace
 
-        if grep -q 'CentOS Linux 7' /etc/os-release; then
-            # disable fastestmirror plugin, which mostly fails due to CentOS 7 being EOL
-            sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf
-
-            sed -i -e 's/^\(mirrorlist\)/#\1/g' /etc/yum.repos.d/CentOS-Base.repo
-            sed -i -e 's|^#baseurl.*|baseurl=http://vault.centos.org/centos/\$releasever/os/\$basearch/|g' /etc/yum.repos.d/CentOS-Base.repo
-        fi
-
         export pmm_version=$pmm_version
         export pmm_release=$pmm_release
         mkdir -p /tmp/pmm
@@ -30,9 +25,7 @@ main() {
         /home/builder/bin/build-client-packages \
             --builddir=/tmp/pmm \
             --build_rpm=1
-
-        sudo_path chown -R $(id -u):$(id -g) /home/builder/results/rpm
-    "
+      "
 }
 
 main $*
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index f23a6e24f6..c68b949a33 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -15,23 +15,13 @@ main() {
         set -o errexit
         set -o xtrace
 
-        if grep -q 'CentOS Linux 7' /etc/os-release; then
-            # disable fastestmirror plugin, which mostly fails due to CentOS 7 being EOL
-            sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf
-
-            sed -i -e 's/^\(mirrorlist\)/#\1/g' /etc/yum.repos.d/CentOS-Base.repo
-            sed -i -e 's|^#baseurl.*|baseurl=http://vault.centos.org/centos/\$releasever/os/\$basearch/|g' /etc/yum.repos.d/CentOS-Base.repo
-        fi
-
         export pmm_release=$pmm_release
-        export sudo_path=\$(command -v sudo)
-        [[ ${IMAGE} = ${rpmbuild_docker_image} ]] || \$sudo_path yum -y install git rpm-build
         mkdir -p /tmp/pmm
         cd /home/builder/results
+
         /home/builder/bin/build-client-packages \
             --builddir=/tmp/pmm \
             --build_src_rpm=1
-        \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/srpm
     "
 }
 
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index a5a8cbb2d8..bb94358b23 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -9,7 +9,9 @@ get_rpm_version() {
     local spec_name=$1
 
     local rpm_version=$(
-        docker run --rm -v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
+        docker run --rm \
+            --platform=${platform} \
+            -v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
             rpmspec -q --qf '%{version}-%{release}\n' SOURCES/${spec_name}.spec \
                 | sed -re 's/\.[0-9]{10}\././; s/.${rpmbuild_dist}//' \
                 | head -1
@@ -130,13 +132,13 @@ build() {
 
         sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
         if [ ! -w /home/builder/.cache/go-build ]; then
-            sudo chown builder /home/builder/.cache/go-build
+            sudo chown builder:builder /home/builder/.cache/go-build
         fi
         if [ ! -w /home/builder/go/pkg/mod ]; then
-            sudo chown builder /home/builder/go/pkg/mod
+            sudo chown builder:builder /home/builder/go/pkg/mod
         fi
-        if [ ! -w pmm-yarn:/home/builder/.cache/yarn ]; then
-            sudo chown builder /home/builder/.cache/yarn
+        if [ ! -w /home/builder/.cache/yarn ]; then
+            sudo chown builder:builder /home/builder/.cache/yarn
         fi
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
@@ -167,15 +169,14 @@ build() {
                   -ba SOURCES/${spec_name}.spec
 
         rm -f SOURCES/${spec_name}.spec*
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
     "
 
     # TODO: see if this is even needed
     # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
 
-    if [ -n "$LOCAL_BUILD" ]; then
-        return
+    if [ "$LOCAL_BUILD" -ne 1 ]; then
         echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
+        return
     fi
 
     if ! command -v aws &> /dev/null; then

From b1933557a050b44b0f57dd3a884c3c7a0cc553b9 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 08:51:24 +0300
Subject: [PATCH 52/78] PMM-12899 revert changes to grafana.spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 2c48a7f35e..4e3eba1aec 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -35,8 +35,8 @@ sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go
 sudo npm install -g grunt-cli
 
 %build
-# mkdir -p _build/src
-# export GOPATH="$(pwd)/_build"
+mkdir -p _build/src
+export GOPATH="$(pwd)/_build"
 
 make build-go
 

From 53de2842182760d81a6c4ef7a70a12ca1cf50f85 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 09:12:21 +0300
Subject: [PATCH 53/78] PMM-12899 revert changes to grafana.spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 2 --
 1 file changed, 2 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 4e3eba1aec..781716494d 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -35,8 +35,6 @@ sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go
 sudo npm install -g grunt-cli
 
 %build
-mkdir -p _build/src
-export GOPATH="$(pwd)/_build"
 
 make build-go
 

From a7ebc00d6822e44817882fe92f13263d09d543e1 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 13:53:05 +0300
Subject: [PATCH 54/78] PMM-12899 update goreleaser to latest

---
 build/docker/rpmbuild/Dockerfile.el9 | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/build/docker/rpmbuild/Dockerfile.el9 b/build/docker/rpmbuild/Dockerfile.el9
index 88373bb787..e958731d93 100644
--- a/build/docker/rpmbuild/Dockerfile.el9
+++ b/build/docker/rpmbuild/Dockerfile.el9
@@ -28,15 +28,16 @@ RUN yum install -y gcc gcc-c++ \
     yum clean all && rm -rf /var/cache/yum
 
 # keep that format for easier search
-ENV GO_VERSION=1.23.2
-ENV GO_RELEASER_VERSION=1.24.0
+ARG GO_VERSION
+ENV GO_VERSION=${GO_VERSION:-1.23.2}
+ENV GO_RELEASER_VERSION=2.4.1
 ENV NODE_OPTIONS=--max_old_space_size=8000
 
 RUN if [ `uname -i` == "x86_64" ]; then ARCH=amd64; else ARCH=arm64; fi && \
     curl -fSsL -o /tmp/golang.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-${ARCH}.tar.gz && \
     curl -fSsL -o /tmp/goreleaser.rpm https://github.com/goreleaser/goreleaser/releases/download/v${GO_RELEASER_VERSION}/goreleaser-${GO_RELEASER_VERSION}-1.`uname -i`.rpm && \
     tar -C /usr/local -xzf /tmp/golang.tar.gz && \
-    yum install -y /tmp/goreleaser.rpm && \
+    yum install -y --disablerepo="*" /tmp/goreleaser.rpm && \
     rm /tmp/golang.tar.gz /tmp/goreleaser.rpm
 
 RUN update-alternatives --install "/usr/bin/go" "go" "/usr/local/go/bin/go" 0

From 6ea712549f164e230471a636f41d04ddc26563aa Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 16:02:58 +0300
Subject: [PATCH 55/78] PMM-12899 use a different Go version for grafana

---
 build/local/build.sh           | 10 ++++++----
 build/scripts/build-server-rpm |  2 --
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 9530bf96b6..85785cd814 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -269,12 +269,12 @@ purge_files() {
 init() {
   local CURDIR="$PWD"
 
-  export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
-
   if [ -d "$SUBMODULES" ]; then
     cd "$SUBMODULES" > /dev/null
   fi
 
+  export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+
   GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
 
   pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
@@ -381,8 +381,11 @@ main() {
   # total time: N/A - subsequent build, using cache from prior builds
 
   export RPM_EPOCH=1
-  export FORCE_REBUILD=1 # Don't use S3 cache
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
+    # Grafana build fails to build with Go 1.23.x, see https://github.com/grafana/grafana/issues/89796
+    export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:go1.22.4
+    run_build_script build-server-rpm grafana
+    export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
     run_build_script build-server-rpm percona-dashboards grafana-dashboards
     run_build_script build-server-rpm pmm-managed pmm
     run_build_script build-server-rpm percona-qan-api2 pmm
@@ -392,7 +395,6 @@ main() {
 
     # 3rd-party
     run_build_script build-server-rpm victoriametrics
-    run_build_script build-server-rpm grafana
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index bb94358b23..eedd6cc34f 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -32,8 +32,6 @@ is_build_needed() {
     # s3://pmm-build-cache/PR-BUILDS/9 - el9
     # s3://pmm-build-cache/RELEASE/9 - el9
 
-    [ "${FORCE_REBUILD}" = 1 ] && return 0
-
     # RPM_EPOCH is set to 1 for Feature Builds
     if [ "$RPM_EPOCH" = 1 ]; then
         s3_cache_dir=PR-BUILDS

From f3b7eceed844286aa32058fa18ecd7b9518454b3 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 17:23:47 +0300
Subject: [PATCH 56/78] PMM-12899 fix copying ansible files

---
 build/local/build.sh              | 28 +---------------------------
 build/scripts/build-client-binary |  9 ++++-----
 build/scripts/build-server-docker | 15 ++++-----------
 build/scripts/vars                |  1 -
 4 files changed, 9 insertions(+), 44 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 85785cd814..a89cbbcbcd 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -277,24 +277,6 @@ init() {
 
   GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
 
-  pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
-  pmm_branch=$(git config -f .gitmodules submodule.pmm.branch)
-  pmm_url=$(git config -f .gitmodules submodule.pmm.url)
-  pmm_qa_branch=$(git config -f .gitmodules submodule.pmm-qa.branch)
-  pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
-  pmm_ui_tests_branch=$(git config -f .gitmodules submodule.pmm-ui-tests.branch)
-  pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
-  fb_commit_sha=$(git rev-parse HEAD)
-
-  echo $fb_commit_sha > fbCommitSha
-  echo $pmm_commit > apiCommitSha
-  echo $pmm_branch > apiBranch
-  echo $pmm_url > apiURL
-  echo $pmm_qa_branch > pmmQABranch
-  echo $pmm_qa_commit > pmmQACommitSha
-  echo $pmm_ui_tests_branch > pmmUITestBranch
-  echo $pmm_ui_tests_commit > pmmUITestsCommitSha
-
   # Create docker volume to persist package and build cache
   # Read more in the section about `rpmbuild`.
   if ! docker volume ls | grep pmm-gobuild >/dev/null; then
@@ -317,15 +299,7 @@ cleanup() {
   local CURDIR="$PWD"
   cd "$SUBMODULES" > /dev/null
 
-  # Clean up temporary files
-  rm -f apiBranch \
-    apiCommitSha \
-    apiURL \
-    fbCommitSha \
-    pmmQABranch \
-    pmmQACommitSha \
-    pmmUITestBranch \
-    pmmUITestsCommitSha || :
+  # Implement cleanup logic here
 
   cd "$CURDIR" > /dev/null
 }
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 9ec917fc08..d2dbaac338 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -21,12 +21,11 @@ gobuild_component() {
     local extract_path=${3:-"github.com/percona/$parent_component"}
     local component_path=$4
     local result_file=${5:-bin/$(basename ${component_path:-$extract_path})}
-    local volume_mounts="-v ${tmp_dir}:/home/builder/tmp"
     local src_dir=/home/builder/tmp/source/pmm-client-${pmm_version}
     local bin_dir=/home/builder/tmp/binary/pmm-client-${pmm_version}
     local archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz" | head -n1)
     local build_command
-
+    local volume_mounts="-v ${tmp_dir}:/home/builder/tmp"
     volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
     volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
     volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
@@ -60,10 +59,11 @@ gobuild_component() {
         export CGO_ENABLED=0
         export GO111MODULE=auto
 
-        # Setting variable for 'make release'
+        # Setting variables for 'make release'
         export PMM_RELEASE_VERSION=${full_pmm_version}
         export PMM_RELEASE_TIMESTAMP=$(date '+%s')
-        export PMM_RELEASE_BRANCH=""
+        export PMM_RELEASE_PATH=${bin_dir}/bin
+
         sudo chown -R builder:builder /home/builder/tmp
         if [ ! -w /home/builder/.cache/go-build ]; then
             sudo chown builder /home/builder/.cache/go-build
@@ -72,7 +72,6 @@ gobuild_component() {
             sudo chown builder /home/builder/go/pkg/mod
         fi
 
-        export PMM_RELEASE_PATH=${bin_dir}/bin
         if [ "${component}" = vmagent ]; then
             export PMM_RELEASE_FULLCOMMIT=${vmagent_commit_hash}
         else
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 6cb0c7ed70..ecf27b081b 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -10,14 +10,11 @@ main() {
   echo "Building docker image for PMM Server..."
   echo -----------------------------------------
 
-  if [ -f "${docker_tarball}" ]; then
-      echo Docker tarball found, skipping docker build...
-      return
-  fi
+  local docker_root=$(realpath ${rpms_dir}/..)
+  local pmm_source_dir=$(git config -f .gitmodules submodule.pmm.path)
 
-  docker_root=$(realpath ${rpms_dir}/..)
-  cp -r ${root_dir}/tmp/source/pmm/build/ansible ${docker_root}/ansible
-  cp ${root_dir}/tmp/source/pmm/build/docker/server/* ${docker_root}/
+  cp -r "${root_dir}/${pmm_source_dir}/build/ansible" "${docker_root}/ansible"
+  cp "${root_dir}/${pmm_source_dir}/build/docker/server/*" "${docker_root}"
   cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
 
   git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
@@ -51,10 +48,6 @@ main() {
       echo ${DOCKER_TAG} > ${docker_tag_file}
       docker push ${DOCKER_TAG}
   fi
-  if [ -n "${SAVE_DOCKER}" ]; then
-      mkdir -p $(dirname ${docker_tarball})
-      docker save ${DOCKER_TAG} > ${docker_tarball}
-  fi
 }
 
 main
diff --git a/build/scripts/vars b/build/scripts/vars
index fac887e0f4..039e75b215 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -49,7 +49,6 @@ client_properties=${root_dir}/results/pmm-client.properties
 docker_file=${DOCKERFILE:-Dockerfile}
 docker_tag_file=${root_dir}/results/docker/TAG
 docker_client_tag_file=${root_dir}/results/docker/CLIENT_TAG
-docker_tarball=${root_dir}/results/docker/pmm-server-${pmm_version}.docker
 docker_client_tarball=${root_dir}/results/docker/pmm-client-${pmm_version}.docker
 source_tarball=${root_dir}/results/source_tarball/pmm-client-${pmm_version}.tar.gz
 binary_tarball=${root_dir}/results/tarball/pmm-client-${pmm_version}.tar.gz

From 3293ab58eaef93f6d62c74599c72a564cce454fe Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 17:47:44 +0300
Subject: [PATCH 57/78] PMM-12899 fix copying Dockerfile

---
 build/scripts/build-server-docker | 4 ++--
 build/scripts/build-server-rpm    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index ecf27b081b..f1df62ac72 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -14,8 +14,8 @@ main() {
   local pmm_source_dir=$(git config -f .gitmodules submodule.pmm.path)
 
   cp -r "${root_dir}/${pmm_source_dir}/build/ansible" "${docker_root}/ansible"
-  cp "${root_dir}/${pmm_source_dir}/build/docker/server/*" "${docker_root}"
-  cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
+  cp "${root_dir}/${pmm_source_dir}/build/docker/server/"* "${docker_root}"
+  cp ${root_dir}/results/tarball/pmm-client-*.tar.gz "${docker_root}/pmm-client.tar.gz"
 
   git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
   ls -la ${rpms_dir}
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index eedd6cc34f..5e69e9eba1 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -172,7 +172,7 @@ build() {
     # TODO: see if this is even needed
     # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
 
-    if [ "$LOCAL_BUILD" -ne 1 ]; then
+    if [ "$LOCAL_BUILD" = 1 ]; then
         echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
         return
     fi

From d501abc35119fd80740b3e48c55942ed8f0ea642 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 18:01:26 +0300
Subject: [PATCH 58/78] PMM-12899 fix legacy Docker label format

---
 build/docker/server/Dockerfile.el9 | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9
index 8f2009bc44..4beaaae520 100644
--- a/build/docker/server/Dockerfile.el9
+++ b/build/docker/server/Dockerfile.el9
@@ -32,13 +32,13 @@ RUN install -T -p -m 644 /opt/ansible/ansible.cfg /etc/ansible/ansible.cfg && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm/post-build-actions.yml && \
     sed -i '/^assumeyes/d' /etc/dnf/dnf.conf
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses AGPL-3.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management
-LABEL org.opencontainers.image.vendor Percona LLC
-LABEL org.opencontainers.image.version ${VERSION}
-LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
-LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/server/docker.html
+LABEL org.opencontainers.image.created=${BUILD_DATE}
+LABEL org.opencontainers.image.licenses=AGPL-3.0
+LABEL org.opencontainers.image.title="Percona Monitoring and Management"
+LABEL org.opencontainers.image.vendor="Percona LLC"
+LABEL org.opencontainers.image.version=${VERSION}
+LABEL org.opencontainers.image.sourc=https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
+LABEL org.opencontainers.image.docs=https://docs.percona.com/percona-monitoring-and-management/setting-up/server/docker.html
 
 USER pmm
 

From 15be81593df33e273f933ba467d959e31c6f07a2 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 00:20:30 +0300
Subject: [PATCH 59/78] PMM-12899 do not use the repo subpaths for mutations

---
 build/docker/server/Dockerfile.el9 | 2 +-
 build/scripts/vars                 | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9
index 4beaaae520..5fc8efc221 100644
--- a/build/docker/server/Dockerfile.el9
+++ b/build/docker/server/Dockerfile.el9
@@ -37,7 +37,7 @@ LABEL org.opencontainers.image.licenses=AGPL-3.0
 LABEL org.opencontainers.image.title="Percona Monitoring and Management"
 LABEL org.opencontainers.image.vendor="Percona LLC"
 LABEL org.opencontainers.image.version=${VERSION}
-LABEL org.opencontainers.image.sourc=https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
+LABEL org.opencontainers.image.source=https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
 LABEL org.opencontainers.image.docs=https://docs.percona.com/percona-monitoring-and-management/setting-up/server/docker.html
 
 USER pmm
diff --git a/build/scripts/vars b/build/scripts/vars
index 039e75b215..3689f05476 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -36,12 +36,12 @@ pmm_version=${new_pmm_version}
 pmm_release=${new_pmm_release}
 unset new_pmm_version
 unset new_pmm_release
-echo -e "\n\n\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release}\n\n\n"
+echo -e "\n\---\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release} <<<\n\---\n"
 
 platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
-rpmbuild_dir=${root_dir}/sources/pmm/src/github.com/percona/pmm/build/packages/rpm/server
+rpmbuild_dir=${root_dir}/tmp/pmm-server/SOURCES
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
 source_dir=${root_dir}/tmp/source/pmm-client-${pmm_version}
 binary_dir=${root_dir}/tmp/binary/pmm-client-${pmm_version}

From eeccd8af9fb877e2c193c6821796618904e7b8f1 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 01:41:13 +0300
Subject: [PATCH 60/78] PMM-12899 use a dedicated spec directory

---
 build/scripts/build-server     |  2 +-
 build/scripts/build-server-rpm | 35 +++++++++++++++++-----------------
 build/scripts/vars             |  5 +++--
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/build/scripts/build-server b/build/scripts/build-server
index 7a2b053be9..89e2262622 100755
--- a/build/scripts/build-server
+++ b/build/scripts/build-server
@@ -10,7 +10,7 @@ set -o xtrace
 pmm_version=${pmm_version} ${bin_dir}/build-client
 pmm_version=${pmm_version} ${bin_dir}/build-client-srpm
 pmm_version=${pmm_version} ${bin_dir}/build-client-rpm
-mkdir -p ${rpms_dir}/
+mkdir -p ${rpms_dir} ${rpmbuild_dir}
 cp ${root_dir}/results/rpm/*.rpm ${rpms_dir}/
 
 # install all rpms
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 5e69e9eba1..6c62dff34b 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -11,10 +11,9 @@ get_rpm_version() {
     local rpm_version=$(
         docker run --rm \
             --platform=${platform} \
-            -v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
+            -v ${rpmbuild_dir}:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
             rpmspec -q --qf '%{version}-%{release}\n' SOURCES/${spec_name}.spec \
-                | sed -re 's/\.[0-9]{10}\././; s/.${rpmbuild_dist}//' \
-                | head -1
+                | sed -re 's/\.[0-9]{10}\././' | head -1
         "
     )
 
@@ -46,52 +45,52 @@ is_build_needed() {
 
     packages=$(find ${rpms_dir}/${spec_name}-${rpm_version} -name "*.${rpmbuild_dist}.*.rpm" | wc -l)
 
-    # return result as true or false
+    # return result as true (0) or false (>0)
     [[ ${packages// /} == 0 ]]
 }
 
 prepare_specs() {
     local spec_name=$1
     local repo_name=$2
+    local spec_file=${rpmbuild_dir}/${spec_name}.spec
 
-    # sudo chown -R $(id -u):$(id -g) ${rpmbuild_dir}/SPECS ${rpmbuild_dir}/SOURCES
-    cp ${rpmbuild_dir}/SPECS/${spec_name}.spec ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    cp ${rpmspec_dir}/${spec_name}.spec ${spec_file}
     if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
         return
     fi
-    if [ ! -f "${rpmbuild_dir}/SOURCES/${spec_name}.spec" ]; then
-      echo "The spec file could not be found."
+    if [ ! -f "${spec_file}" ]; then
+      echo "Warning: the spec file could not be found."
     fi
     
-    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
+    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git -type -d | head -1))
     local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
     local short_commit=${full_commit:0:7}
 
-    sed -i -e "s/global commit.*/global commit ${full_commit}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    sed -i -e "s/global commit.*/global commit ${full_commit}/" ${spec_file}
     if [ "${spec_name}" != "grafana" ]; then
-        sed -i -e "s/Version:.*/Version: ${pmm_version}/"       ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+        sed -i -e "s/Version:.*/Version: ${pmm_version}/"       ${spec_file}
     fi
 
     if [ -z "${full_pmm_version}" ]; then
         echo 'The full_pmm_version is not specified.'
         exit 1
     fi
-    sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${spec_file}
 
     if [ -n "$pmm_release" ]; then
-        sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-        grep -r 'define release' ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+        sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${spec_file}
+        grep -r 'define release' ${spec_file}
     fi
 
-    if [ -f "${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz" ]; then
-        echo SOURCES/${repo_name}-${short_commit}.tar.gz already exists, skip archiving...
+    if [ -f "${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz" ]; then
+        echo "${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz" already exists, skip archiving...
         return
     fi
 
     git -C "${git_dir}" archive \
         --format=tar.gz \
         --prefix=${repo_name}-${full_commit}/ \
-        -o ${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz \
+        -o ${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz \
         "${full_commit}"
 }
 
@@ -104,7 +103,6 @@ build() {
     local CH_VERSION=${rpm_version%-*}
     local CH_TAG=${rpm_version#*-}
     local s3_cache_dir=RELEASE
-    local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
 
     if ! is_build_needed "${spec_name}" "${rpm_version}"; then
         return
@@ -114,6 +112,7 @@ build() {
     echo "Building PMM Server RPM for ${spec_name} component..."
     echo -------------------------------------------------------
 
+    local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
     volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
     volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
     volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
diff --git a/build/scripts/vars b/build/scripts/vars
index 3689f05476..e271e36763 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -36,12 +36,13 @@ pmm_version=${new_pmm_version}
 pmm_release=${new_pmm_release}
 unset new_pmm_version
 unset new_pmm_release
-echo -e "\n\---\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release} <<<\n\---\n"
+echo -e "\n---\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release} <<<\n---\n"
 
 platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
-rpmbuild_dir=${root_dir}/tmp/pmm-server/SOURCES
+rpmbuild_dir=${root_dir}/tmp/pmm-server/BUILD
+rpmspec_dir=${root_dir}/$(git config -f .gitmodules submodule.${module}.path)/build/packages/rpm/server/SPECS
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
 source_dir=${root_dir}/tmp/source/pmm-client-${pmm_version}
 binary_dir=${root_dir}/tmp/binary/pmm-client-${pmm_version}

From 8c8bf03db56adeec5dadf59d135bbbe86914c100 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 01:56:39 +0300
Subject: [PATCH 61/78] PMM-12899 fix the spec directory evaluation

---
 build/scripts/build-client-deb  | 1 -
 build/scripts/build-client-sdeb | 1 -
 build/scripts/vars              | 2 +-
 3 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/build/scripts/build-client-deb b/build/scripts/build-client-deb
index f307f4971a..72deb8a62e 100755
--- a/build/scripts/build-client-deb
+++ b/build/scripts/build-client-deb
@@ -6,7 +6,6 @@ set -o errexit
 set -o xtrace
 
 bin_dir=$(cd $(dirname $0); pwd -P)
-tmp_dir=${root_dir}/tmp
 
 main() {
     local IMAGE=${1:-ubuntu:focal}
diff --git a/build/scripts/build-client-sdeb b/build/scripts/build-client-sdeb
index fb6bb50a43..2429d5de8d 100755
--- a/build/scripts/build-client-sdeb
+++ b/build/scripts/build-client-sdeb
@@ -6,7 +6,6 @@ set -o errexit
 set -o xtrace
 
 bin_dir=$(cd $(dirname $0); pwd -P)
-tmp_dir=${root_dir}/tmp
 
 main() {
     local IMAGE=${1:-ubuntu:focal}
diff --git a/build/scripts/vars b/build/scripts/vars
index e271e36763..d1c0ffe67a 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -42,7 +42,7 @@ platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
 rpmbuild_dir=${root_dir}/tmp/pmm-server/BUILD
-rpmspec_dir=${root_dir}/$(git config -f .gitmodules submodule.${module}.path)/build/packages/rpm/server/SPECS
+rpmspec_dir=${root_dir}/$(git -C ${root_dir} config -f .gitmodules submodule.pmm.path)/build/packages/rpm/server/SPECS
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
 source_dir=${root_dir}/tmp/source/pmm-client-${pmm_version}
 binary_dir=${root_dir}/tmp/binary/pmm-client-${pmm_version}

From 4aa5012fd8d831a4e904834d4ee94618a334bc3f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 02:12:53 +0300
Subject: [PATCH 62/78] PMM-12899 create rpms directories first

---
 build/local/build.sh           | 16 ++++++++--------
 build/scripts/build-server-rpm |  1 +
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index a89cbbcbcd..09d9079cc7 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -195,14 +195,14 @@ update() {
 
 get_branch_name() {
   local module="${1:-}"
-  local branch_name
-  local path
-
-  path=$(git config -f .gitmodules submodule.${module}.path)
-  cd "$path" || exit 1
-  branch_name=$(git branch --show-current)
-  cd - > /dev/null
-  echo $branch_name
+  local path=$(git config -f .gitmodules submodule.${module}.path)
+
+  if [ ! -d "$path" ]; then
+    echo "Error: could not resolve the path to submodule ${module}"
+    exit 1
+  fi
+
+  echo $(git -C "$path" branch --show-current)
 }
 
 run_build_script() {
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 6c62dff34b..dce2d7f068 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -54,6 +54,7 @@ prepare_specs() {
     local repo_name=$2
     local spec_file=${rpmbuild_dir}/${spec_name}.spec
 
+    mkdir -p ${rpms_dir} ${rpmbuild_dir} 2>/dev/null || :
     cp ${rpmspec_dir}/${spec_name}.spec ${spec_file}
     if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
         return

From eaaeaf23e1e827ba291e0e476677cbd5c4e416fb Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 02:30:11 +0300
Subject: [PATCH 63/78] PMM-12899 fix find syntax

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index dce2d7f068..a5abdae5e5 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -63,7 +63,7 @@ prepare_specs() {
       echo "Warning: the spec file could not be found."
     fi
     
-    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git -type -d | head -1))
+    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git -type d | head -1))
     local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
     local short_commit=${full_commit:0:7}
 

From 2325c5312e4ecefd75dbbd3966a6e4b71cade123 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 02:38:31 +0300
Subject: [PATCH 64/78] PMM-12899 fix find syntax to search for both files and
 dirs

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index a5abdae5e5..6fb85d34c4 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -63,7 +63,7 @@ prepare_specs() {
       echo "Warning: the spec file could not be found."
     fi
     
-    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git -type d | head -1))
+    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
     local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
     local short_commit=${full_commit:0:7}
 

From dc6caddb76d6e1af619a2fef039e4f17e4cf6690 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 03:12:24 +0300
Subject: [PATCH 65/78] PMM-12899 revert changes made to grafana.spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 781716494d..4e3eba1aec 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -35,6 +35,8 @@ sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go
 sudo npm install -g grunt-cli
 
 %build
+mkdir -p _build/src
+export GOPATH="$(pwd)/_build"
 
 make build-go
 

From 2819fc9a8634eb184f2b7c13949be2c0064d5d7f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 09:58:51 +0300
Subject: [PATCH 66/78] PMM-12899 try not to use cache for grafana

---
 build/scripts/build-server-rpm | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 6fb85d34c4..f717359a1e 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -114,10 +114,12 @@ build() {
     echo -------------------------------------------------------
 
     local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
-    volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
     volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
     volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
-    volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
+    if [ ${spec_name} != "grafana" ]; then
+        volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
+        volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
+    fi
 
     echo "Start building Server RPMs..."
     echo "spec_name: ${spec_name}"

From cd0df5c1486006f6a77c4cb63802cb87e708ee39 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 10:03:34 +0300
Subject: [PATCH 67/78] PMM-12899 try not to use cache for grafana

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index f717359a1e..1dc65beaad 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -131,7 +131,7 @@ build() {
         set -o xtrace
 
         sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
-        if [ ! -w /home/builder/.cache/go-build ]; then
+        if [ -d /home/builder/.cache/go-build ] && [ ! -w /home/builder/.cache/go-build ]; then
             sudo chown builder:builder /home/builder/.cache/go-build
         fi
         if [ ! -w /home/builder/go/pkg/mod ]; then

From f5dc7453fe813ea297be7f72b90a8466c2c6349d Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 11:10:04 +0300
Subject: [PATCH 68/78] PMM-12899 fix the cache directory not being writable

---
 build/scripts/build-server-rpm | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 1dc65beaad..c429e57550 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -131,7 +131,11 @@ build() {
         set -o xtrace
 
         sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
-        if [ -d /home/builder/.cache/go-build ] && [ ! -w /home/builder/.cache/go-build ]; then
+        sudo chown builder:builder /home/builder/.cache
+        if [ ! -d /home/builder/.cache/go-build ]; then
+            mkdir -p /home/builder/.cache/go-build
+        fi
+        if [ ! -w /home/builder/.cache/go-build ]; then
             sudo chown builder:builder /home/builder/.cache/go-build
         fi
         if [ ! -w /home/builder/go/pkg/mod ]; then

From a28edba5b7068c84f06486f54e5a2f6cdd0d7303 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 11:24:31 +0300
Subject: [PATCH 69/78] PMM-12899 use no Go cache for grafana

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index c429e57550..fb0ddf27e4 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -114,9 +114,9 @@ build() {
     echo -------------------------------------------------------
 
     local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
-    volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
     volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
     if [ ${spec_name} != "grafana" ]; then
+        volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
         volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
         volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
     fi

From 44fc04b8817b883b74033b40d1ce595be8b29a1d Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 11:31:26 +0300
Subject: [PATCH 70/78] PMM-12899 use no Go cache for grafana

---
 build/scripts/build-server-rpm | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index fb0ddf27e4..2cb55dde8c 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -135,6 +135,9 @@ build() {
         if [ ! -d /home/builder/.cache/go-build ]; then
             mkdir -p /home/builder/.cache/go-build
         fi
+        if [ ! -d /home/builder/go ]; then
+            mkdir -p /home/builder/go/pkg/mod
+        fi        
         if [ ! -w /home/builder/.cache/go-build ]; then
             sudo chown builder:builder /home/builder/.cache/go-build
         fi

From e1fbecab5c80a19340bc56b79ae55f3a13cd19ab Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 20:26:29 +0300
Subject: [PATCH 71/78] PMM-12899 remove docker tarballs

---
 build/local/build.sh              | 49 ++++++++++++-------------------
 build/scripts/build-client-docker | 10 -------
 build/scripts/vars                |  1 -
 3 files changed, 19 insertions(+), 41 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 09d9079cc7..4c57936d9b 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -4,16 +4,17 @@ set -o nounset
 
 usage() {
   cat <<-EOF
-Usage: $BASE_NAME [--platform] [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
---platform <platform>    Build for a specific platform (default - linux/amd64)
---no-update              Do not fetch the latest changes from the repo
---update-only            Only fetch the latest changes from the repo
---no-client              Do not build PMM Client
---client-docker          Build PMM Client docker image
---no-server-rpm          Do not build Server RPM packages
---no-server-docker       Do not build PMM Server docker image
---log-file <path>        Save build logs to a file located at <path>
---help | -h              Display help
+Usage: $BASE_NAME [OPTIONS] [--platform] [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
+Options:
+      --platform <platform>    Build for a specific platform (defaults to linux/amd64)
+      --no-update              Do not fetch the latest changes from the repo
+      --update-only            Only fetch the latest changes from the repo
+      --no-client              Do not build PMM Client
+      --client-docker          Build PMM Client docker image
+      --no-server-rpm          Do not build Server RPM packages
+      --no-server-docker       Do not build PMM Server docker image
+      --log-file <path>        Save build logs to a file located at <path> (defaults to PWD)
+      --help | -h              Display help
 EOF
 }
 
@@ -346,29 +347,22 @@ main() {
     run_build_script build-client-docker
   fi
 
-  # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
-  # total time: 6m26s - build from scratch, no initial cache
-  # total time: 2m49s - subsequent build, using cache from prior builds
-
-  # Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
-  # total time: 8m45s - build from scratch, no initial cache
-  # total time: N/A - subsequent build, using cache from prior builds
-
   export RPM_EPOCH=1
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
-    # Grafana build fails to build with Go 1.23.x, see https://github.com/grafana/grafana/issues/89796
-    export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:go1.22.4
-    run_build_script build-server-rpm grafana
+    # Grafana build fails to compile with Go 1.23.x, see https://github.com/grafana/grafana/issues/89796
+    # We need to apply this [patch](https://github.com/grafana/grafana/pull/94742) to fix it
     export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+    # 3rd-party components
+    run_build_script build-server-rpm grafana
+    run_build_script build-server-rpm victoriametrics
+
+    # 1st-party components
     run_build_script build-server-rpm percona-dashboards grafana-dashboards
     run_build_script build-server-rpm pmm-managed pmm
     run_build_script build-server-rpm percona-qan-api2 pmm
     run_build_script build-server-rpm pmm-update pmm
     run_build_script build-server-rpm pmm-dump
     run_build_script build-server-rpm vmproxy pmm
-
-    # 3rd-party
-    run_build_script build-server-rpm victoriametrics
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
@@ -386,14 +380,9 @@ main() {
   cleanup
 }
 
-# Reference test environment
-# CPU: 4 cores
-# RAM: 16 GB
-# OS: Ubuntu 22.04.1 LTS
-
 parse-params "$@"
 
-# Capture the output in the log file
+# Capture the build logs in the log file
 exec > >(tee "$LOG_FILE") 2>&1
 
 main
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index 9494d41f5d..a564c5c4cf 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -10,11 +10,6 @@ main() {
     echo "Building docker image for PMM Client..."
     echo -----------------------------------------
 
-    if [ -f "${docker_client_tarball}" ]; then
-        echo Docker client tarball found, skipping docker build...
-        return
-    fi
-
     DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
     cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
 
@@ -22,7 +17,6 @@ main() {
         DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version}
     fi
 
-
     CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
 
     docker buildx build \
@@ -38,10 +32,6 @@ main() {
         echo ${DOCKER_CLIENT_TAG} > ${docker_client_tag_file}
         docker push ${DOCKER_CLIENT_TAG}
     fi
-    if [ -n "${SAVE_DOCKER}" ]; then
-        mkdir -p $(dirname ${docker_client_tarball})
-        docker save ${DOCKER_CLIENT_TAG} | xz > ${docker_client_tarball}
-    fi
 }
 
 main
diff --git a/build/scripts/vars b/build/scripts/vars
index d1c0ffe67a..4c6a25f273 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -50,7 +50,6 @@ client_properties=${root_dir}/results/pmm-client.properties
 docker_file=${DOCKERFILE:-Dockerfile}
 docker_tag_file=${root_dir}/results/docker/TAG
 docker_client_tag_file=${root_dir}/results/docker/CLIENT_TAG
-docker_client_tarball=${root_dir}/results/docker/pmm-client-${pmm_version}.docker
 source_tarball=${root_dir}/results/source_tarball/pmm-client-${pmm_version}.tar.gz
 binary_tarball=${root_dir}/results/tarball/pmm-client-${pmm_version}.tar.gz
 

From c94b549e4298dc8ea352dc640f5e89d628fa1f46 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 21:45:50 +0300
Subject: [PATCH 72/78] PMM-12899 add --client-only parameter

---
 build/local/build.sh | 83 +++++++++++++++++++++++---------------------
 1 file changed, 44 insertions(+), 39 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 4c57936d9b..dd96c41418 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -4,15 +4,15 @@ set -o nounset
 
 usage() {
   cat <<-EOF
-Usage: $BASE_NAME [OPTIONS] [--platform] [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
+Usage: $BASE_NAME [OPTIONS]
 Options:
       --platform <platform>    Build for a specific platform (defaults to linux/amd64)
       --no-update              Do not fetch the latest changes from the repo
       --update-only            Only fetch the latest changes from the repo
       --no-client              Do not build PMM Client
-      --client-docker          Build PMM Client docker image
-      --no-server-rpm          Do not build Server RPM packages
-      --no-server-docker       Do not build PMM Server docker image
+      --no-client-docker       Do not build PMM Client docker image (default)
+      --client-only            Build only PMM Client (client binaries + docker)
+      --no-server              Do not build PMM Server (docker image)
       --log-file <path>        Save build logs to a file located at <path> (defaults to PWD)
       --help | -h              Display help
 EOF
@@ -25,8 +25,7 @@ parse-params() {
   UPDATE_ONLY=0
   NO_CLIENT=0
   NO_CLIENT_DOCKER=1
-  NO_SERVER_RPM=0
-  NO_SERVER_DOCKER=0
+  NO_SERVER=0
   START_TIME=$(date +%s)
   LOG_FILE="$(dirname $0)/build.log"
   BASE_NAME=$(basename $0)
@@ -40,27 +39,34 @@ parse-params() {
         UPDATE_ONLY=1; NO_UPDATE=0
         ;;
       --no-update)
+        if [ "$UPDATE_ONLY" -eq 1 ]; then
+          echo "Error. Mutually exclusive options: --update-only and --no-update"
+          exit 1
+        fi      
         NO_UPDATE=1
         ;;
+      --client-only)
+        NO_CLIENT=0; NO_CLIENT_DOCKER=0; NO_SERVER=1
+        ;;
       --no-client)
         NO_CLIENT=1; NO_CLIENT_DOCKER=1
         ;;
       --client-docker)
         NO_CLIENT_DOCKER=0
         ;;
-      --no-server-rpm)
-        if [ "$NO_SERVER_DOCKER" -eq 1 ]; then
-          echo "Error: cannot disable both server RPM and server Docker"
+      --no-client-docker)
+        if [ "$NO_CLIENT" -eq 1 ]; then
+          echo "Error. Mutually exclusive options: --client-docker and --no-client"
           exit 1
         fi
-        NO_SERVER_RPM=1
-        ;;
-      --no-server-docker)
-        if [ "$NO_SERVER_RPM" -eq 1 ]; then
-          echo "Error: cannot disable both server RPM and server Docker"
+        if [ "$NO_CLIENT_DOCKER" -eq 1 ]; then
+          echo "Error. Mutually exclusive options: --client-docker and --no-client-docker"
           exit 1
         fi
-        NO_SERVER_DOCKER=1
+        NO_CLIENT_DOCKER=1
+        ;;
+      --no-server)
+        NO_SERVER=1
         ;;
       --platform)
         shift
@@ -145,9 +151,17 @@ check-files() {
   return 1
 }
 
+# Update submodules and PR branches
 update() {
   local DEPS=
   local CURDIR="$PWD"
+  local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build.sh"
+  local MD5SUM=$(md5sum $(dirname $0)/build.sh)
+
+   if [ "$NO_UPDATE" -eq 1 ]; then
+    echo "Running without refreshing the source code from repositories..."
+    return
+   fi
 
   # Thouroughly verify the presence of known files, otherwise bail out
   if [ ! -d "$SUBMODULES" ] ; then # pwd must be outside of pmm-submodules
@@ -192,6 +206,13 @@ update() {
   git submodule status
 
   cd "$CURDIR" > /dev/null
+
+  if [ -f "$UPDATED_SCRIPT" ] && [ "$MD5SUM" != "$(md5sum $UPDATED_SCRIPT)" ]; then
+    echo "The local copy of this script differs from the one fetched from the repo." 
+    echo "Apparently, that version is newer. We will halt to give you the change to run a fresh version."
+    echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+    exit 0
+  fi
 }
 
 get_branch_name() {
@@ -306,22 +327,7 @@ cleanup() {
 }
 
 main() {
-  if [ "$NO_UPDATE" -eq 0 ]; then
-    local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build.sh"
-    MD5SUM=$(md5sum $(dirname $0)/build.sh)
-
-    # Update submodules and PR branches
-    update
-
-    test "$UPDATE_ONLY" -eq 1 && return
-
-    if [ -f "$UPDATED_SCRIPT" ] && [ "$MD5SUM" != "$(md5sum $UPDATED_SCRIPT)" ]; then
-      echo "The local copy of this script differs from the one fetched from the repo." 
-      echo "Apparently, that version is newer. We will halt to give you the change to run a fresh version."
-      echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
-      return
-    fi
-  fi
+  update
 
   init
 
@@ -335,10 +341,10 @@ main() {
     run_build_script build-client-binary
 
     # Building client source rpm takes 13s (caching does not apply)
-    # run_build_script build-client-srpm
+    run_build_script build-client-srpm
 
     # Building client rpm takes 1m40s
-    # run_build_script build-client-rpm
+    run_build_script build-client-rpm
   fi
 
   # Building client docker image takes 17s
@@ -347,11 +353,14 @@ main() {
     run_build_script build-client-docker
   fi
 
-  export RPM_EPOCH=1
-  if [ "$NO_SERVER_RPM" -eq 0 ]; then
+  if [ "$NO_SERVER" -eq 0 ]; then
     # Grafana build fails to compile with Go 1.23.x, see https://github.com/grafana/grafana/issues/89796
     # We need to apply this [patch](https://github.com/grafana/grafana/pull/94742) to fix it
     export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+    export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT}
+    export DOCKERFILE=Dockerfile.el9
+    export RPM_EPOCH=1
+
     # 3rd-party components
     run_build_script build-server-rpm grafana
     run_build_script build-server-rpm victoriametrics
@@ -363,11 +372,7 @@ main() {
     run_build_script build-server-rpm pmm-update pmm
     run_build_script build-server-rpm pmm-dump
     run_build_script build-server-rpm vmproxy pmm
-  fi
 
-  if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
-    export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT}
-    export DOCKERFILE=Dockerfile.el9
     run_build_script build-server-docker
   fi
 

From 8944ef5af26daffeb8cb7ebcf43253106aa350fb Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 22:56:23 +0300
Subject: [PATCH 73/78] PMM-12899 rename build.sh to build

---
 build/local/README.md               | 10 +++++-----
 build/local/{build.sh => build}     | 17 +++++++++++------
 build/scripts/build-client-packages | 15 +++++++++------
 3 files changed, 25 insertions(+), 17 deletions(-)
 rename build/local/{build.sh => build} (98%)

diff --git a/build/local/README.md b/build/local/README.md
index b775030f66..7c1fac211a 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -33,10 +33,10 @@ Below is a list of prerequisites that are required to build PMM locally.
 1. Install the prerequisites
 2. Clone the PMM repository to the user's home directory, e.g.: `git clone https://github.com/percona/pmm /home/user/pmm`. 
 3. Change to the `build/local` directory in the cloned repo.
-4. Run `build.sh --help` to print the help message and check the usage.
-5. Run `build.sh` with parameters of your choice to build PMM v3.
+4. Run `./build --help` to print the help message and check the usage.
+5. Run `./build` with parameters of your choice to build PMM v3.
 
-Usually, you will want to rebuild PMM whenever there are changes in at least one of its components. All components of PMM are gathered together in one repository - `github.com/percona-lab/pmm-submodules` (or `pmm-submodules`). Therefore, you can run `build.sh` as often as those changes need to be factored in to the next build.
+Usually, you will want to rebuild PMM whenever there are changes in at least one of its components. All components of PMM are gathered together in one repository - `github.com/percona-lab/pmm-submodules` (or `pmm-submodules`). Therefore, you can run `build` as often as those changes need to be factored in to the next build.
 
 Once the build is finished, you can proceed with launching a new instance of PMM Server, or installing a freshly built PMM Client, and testing the changes.
 
@@ -48,12 +48,12 @@ We use a special docker image to build various PMM artifacts - `perconalab/rpmbu
 - it frees the user from installing dependencies on their host machine
 - it leverages a very powerful docker caching system, which results in reduced build times
 
-During the first run, `build.sh` will create a few directories on the host machine, which are necessary to make use of docker cache. Please be aware, that the docker container's user needs to be able to write to these directories. The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure that the directories we create on the host are owned by a user with the same uid and gid. If the build fails, this is the first thing to check.
+During the first run, `build` will create a few directories on the host machine, which are necessary to make use of docker cache. Please be aware, that the docker container's user needs to be able to write to these directories. The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure that the directories we create on the host are owned by a user with the same uid and gid. If the build fails, this is the first thing to check.
 
 
 ## Avoiding unnecessary builds
 
-Sometimes, the changes you make affect only PMM Client. Other times, they affect only PMM Server. Therefore, you may want to skip building parts of PMM. The `build.sh` script offers several parameters to help control what you want to build.
+Sometimes, the changes you make affect only PMM Client. Other times, they affect only PMM Server. Therefore, you may want to skip building parts of PMM. The `build` script offers several parameters to help control what you want to build.
 
 * --no-update: run the build tasks without pulling the changes from `pmm-submodules` repository
 * --update-only: pull changes from the repo without building PMM
diff --git a/build/local/build.sh b/build/local/build
similarity index 98%
rename from build/local/build.sh
rename to build/local/build
index dd96c41418..9a92834af7 100755
--- a/build/local/build.sh
+++ b/build/local/build
@@ -52,13 +52,13 @@ parse-params() {
         NO_CLIENT=1; NO_CLIENT_DOCKER=1
         ;;
       --client-docker)
-        NO_CLIENT_DOCKER=0
-        ;;
-      --no-client-docker)
         if [ "$NO_CLIENT" -eq 1 ]; then
           echo "Error. Mutually exclusive options: --client-docker and --no-client"
           exit 1
         fi
+        NO_CLIENT_DOCKER=0
+        ;;
+      --no-client-docker)
         if [ "$NO_CLIENT_DOCKER" -eq 1 ]; then
           echo "Error. Mutually exclusive options: --client-docker and --no-client-docker"
           exit 1
@@ -155,8 +155,8 @@ check-files() {
 update() {
   local DEPS=
   local CURDIR="$PWD"
-  local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build.sh"
-  local MD5SUM=$(md5sum $(dirname $0)/build.sh)
+  local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build"
+  local MD5SUM=$(md5sum $(dirname $0)/build)
 
    if [ "$NO_UPDATE" -eq 1 ]; then
     echo "Running without refreshing the source code from repositories..."
@@ -210,7 +210,7 @@ update() {
   if [ -f "$UPDATED_SCRIPT" ] && [ "$MD5SUM" != "$(md5sum $UPDATED_SCRIPT)" ]; then
     echo "The local copy of this script differs from the one fetched from the repo." 
     echo "Apparently, that version is newer. We will halt to give you the change to run a fresh version."
-    echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+    echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build --no-update'"
     exit 0
   fi
 }
@@ -284,6 +284,11 @@ purge_files() {
       rm -rf tmp/source/pmm
     fi
   fi
+
+  if [ -d results/rpm ]; then
+    echo "Removing results/rpm/*.rpm ..."
+    rm results/rpm/*.rpm
+  fi
   
   cd "$CURDIR"
 }
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index f578a422d0..ed4836ec63 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -34,7 +34,6 @@ parse_arguments() {
         val=`echo "$arg" | sed -e 's;^--[^=]*=;;'`
         optname=`echo "$arg" | sed -e 's/^\(--[^=]*\)=.*$/\1/'`
         case "$arg" in
-            # these get passed explicitly to mysqld
             --builddir=*) WORKDIR="$val" ;;
             --build_src_rpm=*) SRPM="$val" ;;
             --build_source_deb=*) SDEB="$val" ;;
@@ -44,9 +43,7 @@ parse_arguments() {
             --branch=*) SUBMODULE_BRANCH="$val" ;;
             --install_deps=*) INSTALL="$val" ;;
             --help) usage ;;
-            *)
-                append_arg_to_args "$arg"
-              ;;
+            *) append_arg_to_args "$arg" ;;
         esac
     done
 }
@@ -59,7 +56,7 @@ get_branches() {
     cd ${WORKDIR}/pmm-submodules
       git reset --hard > /dev/null 2>&1
       git clean -xdf > /dev/null 2>&1
-      git checkout $SUBMODULE_BRANCH > /dev/null 2>&1
+      git checkout $SUBMODULE_BRANCH
       # this is the return value of this function
       git submodule status | grep $COMPONENT | awk '{print $1}' | awk -F'-' '{print $2}'
     cd - >/dev/null
@@ -257,7 +254,13 @@ build_srpm(){
     cp ${WORKDIR}/${NAME}-${VERSION}/rpm/*.spec .
     cp ${WORKDIR}/${TARFILE} ../SOURCES/
     cd ${WORKDIR}
-    rpmbuild -bs --define "_topdir ${WORKDIR}/rpmbuild" --define "version $VERSION" --define "release $RPM_RELEASE" --define "dist %{nil}" "rpmbuild/SPECS/${PACKAGE_NAME}.spec"
+    rpmbuild -bs \
+      --define "_topdir ${WORKDIR}/rpmbuild" \
+      --define "version $VERSION" \
+      --define "release $RPM_RELEASE" \
+      --define "dist %{nil}" \
+      "rpmbuild/SPECS/${PACKAGE_NAME}.spec"
+
     mkdir -p ${WORKDIR}/srpm
     mkdir -p ${CURDIR}/srpm
     cp rpmbuild/SRPMS/*.src.rpm ${CURDIR}/srpm

From 69899db8fc0f7979da0b148e2ddc8c53556bf551 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 23:04:05 +0300
Subject: [PATCH 74/78] PMM-12899 purge more files before starting

---
 build/local/build | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/build/local/build b/build/local/build
index 9a92834af7..4e51fcc4c4 100755
--- a/build/local/build
+++ b/build/local/build
@@ -285,9 +285,9 @@ purge_files() {
     fi
   fi
 
-  if [ -d results/rpm ]; then
-    echo "Removing results/rpm/*.rpm ..."
-    rm results/rpm/*.rpm
+  if [ -d results ]; then
+    echo "Removing results/* ..."
+    rm -rf results/{rpm,srpm,tarball,source_tarball,pmm-client.properties}
   fi
   
   cd "$CURDIR"

From a8756abcce9d6ebc486e14b4fa454e910e357d26 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 23:49:53 +0300
Subject: [PATCH 75/78] PMM-12899 pass NO_VCS parameter to go build

---
 admin/Makefile                    | 8 ++++++--
 agent/Makefile                    | 8 ++++++--
 build/local/build                 | 2 +-
 build/scripts/build-client-binary | 2 +-
 4 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/admin/Makefile b/admin/Makefile
index f6c12416e1..0d29552f70 100644
--- a/admin/Makefile
+++ b/admin/Makefile
@@ -11,6 +11,10 @@ PMM_RELEASE_VERSION ?= $(shell git describe --always --dirty | cut -b2-)
 PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s')
 PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD)
 PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
+
+ifdef NO_VCS
+	NO_VCS := -buildvcs=false
+endif
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
 endif
@@ -31,8 +35,8 @@ clean:                          ## Remove generated files
 	find . -name *_reform.go -delete
 
 release:                        ## Build pmm-admin release binary
-	env CGO_ENABLED=0 go build -v $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm-admin ./cmd/pmm-admin/
-	env CGO_ENABLED=0 go build -v $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm ./cmd/pmm/
+	env CGO_ENABLED=0 go build -v $(NO_VCS) $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm-admin ./cmd/pmm-admin/
+	env CGO_ENABLED=0 go build -v $(NO_VCS) $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm ./cmd/pmm/
 
 install:                        ## Install pmm & pmm-admin binary
 	go build -v $(LD_FLAGS) -o $(GOBIN)/pmm-admin ./cmd/pmm-admin/
diff --git a/agent/Makefile b/agent/Makefile
index 2507f82fcb..6c1e863b6a 100644
--- a/agent/Makefile
+++ b/agent/Makefile
@@ -12,6 +12,10 @@ PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s')
 PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD)
 PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
 PMM_DEV_SERVER_PORT ?= 8443
+
+ifdef NO_VCS
+	NO_VCS := -buildvcs=false
+endif
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
 endif
@@ -25,8 +29,8 @@ VERSION_FLAGS = -X 'github.com/percona/pmm/version.ProjectName=pmm-agent' \
 				-X 'github.com/percona/pmm/version.Branch=$(PMM_RELEASE_BRANCH)'
 
 release:                        ## Build static pmm-agent release binary (Linux only)
-	env CGO_ENABLED=1 go build -v -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent
-	go build -v -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent-entrypoint ./cmd/pmm-agent-entrypoint
+	env CGO_ENABLED=1 go build -v $(NO_VCS) -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent
+	go build -v $(NO_VCS) -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent-entrypoint ./cmd/pmm-agent-entrypoint
 	$(PMM_RELEASE_PATH)/pmm-agent --version
 	ldd $(PMM_RELEASE_PATH)/pmm-agent 2>&1 | grep -Fq 'not a dynamic executable'
 
diff --git a/build/local/build b/build/local/build
index 4e51fcc4c4..2c5b549206 100755
--- a/build/local/build
+++ b/build/local/build
@@ -352,7 +352,7 @@ main() {
     run_build_script build-client-rpm
   fi
 
-  # Building client docker image takes 17s
+  # Building client docker image takes from 17s (using docker cache) to 43s (no docker cache). 
   export DOCKER_CLIENT_TAG=percona/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
     run_build_script build-client-docker
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index d2dbaac338..8706136537 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -39,7 +39,7 @@ gobuild_component() {
         return
     fi
     if [[ ${component} =~ pmm-admin|pmm-agent ]]; then
-        build_command="cd /tmp/go/src/${extract_path}/${component_path} && make release"
+        build_command="cd /tmp/go/src/${extract_path}/${component_path} && NO_VCS=1 make release"
     elif [ ${component} = 'mongodb_exporter' ]; then
         build_command="cd /tmp/go/src/${extract_path} && make build"
     elif [[ ${component} =~ node_exporter|mysqld_exporter|proxysql_exporter|rds_exporter|postgres_exporter|azure_metrics_exporter ]]; then

From be7405eff8fa09596e95c86f62dabbee47d7ee46 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 23:53:07 +0300
Subject: [PATCH 76/78] PMM-12899 add dist directory to .gitignore

---
 .gitignore        | 3 +++
 build/local/build | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index da7596fcd5..e862c50236 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,3 +44,6 @@ api-tests/pmm-api-tests-output.txt
 api-tests/pmm-api-tests-junit-report.xml
 
 packer.log
+
+# Goreleaser
+dist/
diff --git a/build/local/build b/build/local/build
index 2c5b549206..3c65ca4472 100755
--- a/build/local/build
+++ b/build/local/build
@@ -352,7 +352,7 @@ main() {
     run_build_script build-client-rpm
   fi
 
-  # Building client docker image takes from 17s (using docker cache) to 43s (no docker cache). 
+  # Building client docker image takes from 17s (using docker cache) to 43s (no docker cache).
   export DOCKER_CLIENT_TAG=percona/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
     run_build_script build-client-docker

From 93d5d7cbc39f7e8c4b777281480d8937b78b05d5 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 00:05:33 +0300
Subject: [PATCH 77/78] PMM-12899 create cache directories in rpmbuild

---
 build/docker/rpmbuild/Dockerfile.el9 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/docker/rpmbuild/Dockerfile.el9 b/build/docker/rpmbuild/Dockerfile.el9
index e958731d93..6f3c445b23 100644
--- a/build/docker/rpmbuild/Dockerfile.el9
+++ b/build/docker/rpmbuild/Dockerfile.el9
@@ -49,7 +49,7 @@ RUN useradd builder -u 1000 -m -G users,wheel && \
     echo "builder ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers && \
     echo "# macros"                      >  /home/builder/.rpmmacros && \
     echo "%_topdir    /home/builder/rpm" >> /home/builder/.rpmmacros && \
-    mkdir /home/builder/rpm && \
+    mkdir -p /home/builder/rpm /home/builder/.cache/go-build /home/builder/go/pkg/mod && \
     chmod 755 /home/builder && \
     chown -R builder:builder /home/builder
 

From 45fd300b04871b61bc7746808403ea93335c39a5 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 00:17:59 +0300
Subject: [PATCH 78/78] PMM-12899 fix "build_command" extrapolation

---
 admin/Makefile                    | 2 +-
 agent/Makefile                    | 2 +-
 build/scripts/build-client-binary | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/admin/Makefile b/admin/Makefile
index 0d29552f70..b3f4249f03 100644
--- a/admin/Makefile
+++ b/admin/Makefile
@@ -13,7 +13,7 @@ PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD)
 PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
 
 ifdef NO_VCS
-	NO_VCS := -buildvcs=false
+	NO_VCS = -buildvcs=false
 endif
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
diff --git a/agent/Makefile b/agent/Makefile
index 6c1e863b6a..fb346bd539 100644
--- a/agent/Makefile
+++ b/agent/Makefile
@@ -14,7 +14,7 @@ PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
 PMM_DEV_SERVER_PORT ?= 8443
 
 ifdef NO_VCS
-	NO_VCS := -buildvcs=false
+	NO_VCS = -buildvcs=false
 endif
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 8706136537..f20b905ddc 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -83,7 +83,7 @@ gobuild_component() {
 
         tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf ${src_dir}/${archive}
 
-        ${build_command}
+        "${build_command}"
     "
 }