PMM-12899 Use module and build cache

.github/workflows/main.yml

@@ -161,8 +161,8 @@ jobs:
     env:
       SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }}
       SLACK_CHANNEL: "pmm-ci"
-      SLACK_USERNAME: "PR pipelines bot"
-      SLACK_ICON_EMOJI: ":chestnut:"
+      SLACK_USERNAME: "Build bot"
+      SLACK_ICON_EMOJI: ":octocat:"
       SLACK_COLOR: "#00FF00"
       SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow"
       SLACK_MESSAGE: "${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}"
@@ -179,8 +179,8 @@ jobs:
     env:
       SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }}
       SLACK_CHANNEL: "pmm-ci"
-      SLACK_USERNAME: "PR pipelines bot"
-      SLACK_ICON_EMOJI: ":chestnut:"
+      SLACK_USERNAME: "Build bot"
+      SLACK_ICON_EMOJI: ":octocat:"
       SLACK_COLOR: "#FF0000"
       SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow"
       SLACK_MESSAGE: "Workflow failed: ${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}"

.gitignore

@@ -44,3 +44,6 @@ api-tests/pmm-api-tests-output.txt
 api-tests/pmm-api-tests-junit-report.xml
 
 packer.log
+
+# Goreleaser
+dist/

admin/Makefile

@@ -11,6 +11,10 @@ PMM_RELEASE_VERSION ?= $(shell git describe --always --dirty | cut -b2-)
 PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s')
 PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD)
 PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
+
+ifdef NO_VCS
+	NO_VCS = -buildvcs=false
+endif
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
 endif
@@ -31,8 +35,8 @@ clean:                          ## Remove generated files
 	find . -name *_reform.go -delete
 
 release:                        ## Build pmm-admin release binary
-	env CGO_ENABLED=0 go build -v $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm-admin ./cmd/pmm-admin/
-	env CGO_ENABLED=0 go build -v $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm ./cmd/pmm/
+	env CGO_ENABLED=0 go build -v $(NO_VCS) $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm-admin ./cmd/pmm-admin/
+	env CGO_ENABLED=0 go build -v $(NO_VCS) $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm ./cmd/pmm/
 
 install:                        ## Install pmm & pmm-admin binary
 	go build -v $(LD_FLAGS) -o $(GOBIN)/pmm-admin ./cmd/pmm-admin/

agent/Makefile

@@ -12,6 +12,10 @@ PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s')
 PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD)
 PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
 PMM_DEV_SERVER_PORT ?= 8443
+
+ifdef NO_VCS
+	NO_VCS = -buildvcs=false
+endif
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
 endif
@@ -25,8 +29,8 @@ VERSION_FLAGS = -X 'github.com/percona/pmm/version.ProjectName=pmm-agent' \
 				-X 'github.com/percona/pmm/version.Branch=$(PMM_RELEASE_BRANCH)'
 
 release:                        ## Build static pmm-agent release binary (Linux only)
-	env CGO_ENABLED=1 go build -v -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent
-	go build -v -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent-entrypoint ./cmd/pmm-agent-entrypoint
+	env CGO_ENABLED=1 go build -v $(NO_VCS) -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent
+	go build -v $(NO_VCS) -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent-entrypoint ./cmd/pmm-agent-entrypoint
 	$(PMM_RELEASE_PATH)/pmm-agent --version
 	ldd $(PMM_RELEASE_PATH)/pmm-agent 2>&1 | grep -Fq 'not a dynamic executable'
 

build/docker/client/Dockerfile

@@ -5,19 +5,13 @@ RUN microdnf install shadow-utils jq tar -y
 RUN groupadd -g 1002 pmm-agent && \
     useradd -u 1002 -r -g pmm-agent -s /sbin/nologin \
             -d /usr/local/percona/pmm \
-            -c "PMM 2.X Client User" pmm-agent
+            -c "PMM 3.x Client User" pmm-agent
 
 FROM redhat/ubi9-micro
 
 ARG VERSION
 ARG BUILD_DATE
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses Apache-2.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management Client
-LABEL org.opencontainers.image.vendor Percona
-LABEL org.opencontainers.image.version ${VERSION}
-
 COPY LICENSE /licenses/
 COPY --from=users --chown=0:0 /etc/passwd /etc/passwd
 COPY --from=users --chown=0:0 /etc/group /etc/group
@@ -49,10 +43,18 @@ RUN install -d -o pmm-agent -g pmm-agent -m 0775 /usr/local/percona/pmm
 ADD pmm-client.tar.gz /tmp/
 
 RUN cd /tmp/pmm-client-* \
-    && env PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \
+    && PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \
     && cd /tmp \
     && rm -rf /tmp/pmm-client*
 
+LABEL org.opencontainers.image.created ${BUILD_DATE}
+LABEL org.opencontainers.image.licenses Apache-2.0
+LABEL org.opencontainers.image.title Percona Monitoring and Management Client
+LABEL org.opencontainers.image.vendor Percona
+LABEL org.opencontainers.image.version ${VERSION}
+LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/client/Dockerfile
+LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#docker
+
 USER pmm-agent
 WORKDIR /usr/local/percona/pmm/
 ENV PATH /usr/local/percona/pmm/bin/:$PATH

build/docker/rpmbuild/Dockerfile.el9

@@ -28,14 +28,16 @@ RUN yum install -y gcc gcc-c++ \
     yum clean all && rm -rf /var/cache/yum
 
 # keep that format for easier search
-ENV GO_VERSION 1.23.2
-ENV GO_RELEASER_VERSION 1.24.0
+ARG GO_VERSION
+ENV GO_VERSION=${GO_VERSION:-1.23.2}
+ENV GO_RELEASER_VERSION=2.4.1
+ENV NODE_OPTIONS=--max_old_space_size=8000
 
 RUN if [ `uname -i` == "x86_64" ]; then ARCH=amd64; else ARCH=arm64; fi && \
     curl -fSsL -o /tmp/golang.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-${ARCH}.tar.gz && \
     curl -fSsL -o /tmp/goreleaser.rpm https://github.com/goreleaser/goreleaser/releases/download/v${GO_RELEASER_VERSION}/goreleaser-${GO_RELEASER_VERSION}-1.`uname -i`.rpm && \
     tar -C /usr/local -xzf /tmp/golang.tar.gz && \
-    yum install -y /tmp/goreleaser.rpm && \
+    yum install -y --disablerepo="*" /tmp/goreleaser.rpm && \
     rm /tmp/golang.tar.gz /tmp/goreleaser.rpm
 
 RUN update-alternatives --install "/usr/bin/go" "go" "/usr/local/go/bin/go" 0
@@ -47,7 +49,7 @@ RUN useradd builder -u 1000 -m -G users,wheel && \
     echo "builder ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers && \
     echo "# macros"                      >  /home/builder/.rpmmacros && \
     echo "%_topdir    /home/builder/rpm" >> /home/builder/.rpmmacros && \
-    mkdir /home/builder/rpm && \
+    mkdir -p /home/builder/rpm /home/builder/.cache/go-build /home/builder/go/pkg/mod && \
     chmod 755 /home/builder && \
     chown -R builder:builder /home/builder
 

build/docker/server/Dockerfile.el9

@@ -4,7 +4,6 @@ ARG VERSION
 ARG BUILD_DATE
 
 ENV LANG=en_US.utf8
-ENV LC_ALL=en_US.utf8
 ENV GF_PLUGIN_DIR=/srv/grafana/plugins
 ENV PS1="[\u@\h \W] # "
 
@@ -33,11 +32,13 @@ RUN install -T -p -m 644 /opt/ansible/ansible.cfg /etc/ansible/ansible.cfg && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm/post-build-actions.yml && \
     sed -i '/^assumeyes/d' /etc/dnf/dnf.conf
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses AGPL-3.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management
-LABEL org.opencontainers.image.vendor Percona LLC
-LABEL org.opencontainers.image.version ${VERSION}
+LABEL org.opencontainers.image.created=${BUILD_DATE}
+LABEL org.opencontainers.image.licenses=AGPL-3.0
+LABEL org.opencontainers.image.title="Percona Monitoring and Management"
+LABEL org.opencontainers.image.vendor="Percona LLC"
+LABEL org.opencontainers.image.version=${VERSION}
+LABEL org.opencontainers.image.source=https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
+LABEL org.opencontainers.image.docs=https://docs.percona.com/percona-monitoring-and-management/setting-up/server/docker.html
 
 USER pmm
 

build/local/README.md

@@ -0,0 +1,77 @@
+# Local builds
+
+This directory contains a set of scripts aimed at providing a simple way to build PMM locally.
+
+## Background
+
+Historically, PMM used to be built using Jenkins. This worked well for the team, but not for the community. The learning curve was, and still is, rather steep, and it is hard for folks, even internally, to contribute to.
+
+Therefore, we decided to make it possible to build PMM locally. This is a work in progress, but we are definitely committed to bring the developer experience to an acceptable level.
+
+The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on, since every developer should have at least a basic command of bash. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching.
+
+The build process is designed to run on a Linux host. We believe it can run on other flavors of Linux, including MacOS, with little to no modification (TBC).
+
+
+## Prerequisites
+
+Below is a list of prerequisites that are required to build PMM locally.
+
+- OS: Linux (tested on Oracle Linux 9.3, CentOS 7, Ubuntu 22.04.3 LTS)
+- Docker: 25.0.2+ (tested on 25.0.2)
+- Docker Compose Plugin: 2.24.7+ (tested on 2.24.7)
+- make: 4.3+ (tested on 4.3)
+- bash (GNU): 5.1+ (tested on 5.1)
+- git: 2.34.1+ (tested on 2.34.1)
+- curl: 7.81.0+ (tested on 7.81.0)
+- yq: 4.42.0+ (tested on 4.42.1)
+- jq: 1.6+ (tested on 1.6)
+
+
+## How to build PMM
+
+1. Install the prerequisites
+2. Clone the PMM repository to the user's home directory, e.g.: `git clone https://github.com/percona/pmm /home/user/pmm`. 
+3. Change to the `build/local` directory in the cloned repo.
+4. Run `./build --help` to print the help message and check the usage.
+5. Run `./build` with parameters of your choice to build PMM v3.
+
+Usually, you will want to rebuild PMM whenever there are changes in at least one of its components. All components of PMM are gathered together in one repository - `github.com/percona-lab/pmm-submodules` (or `pmm-submodules`). Therefore, you can run `build` as often as those changes need to be factored in to the next build.
+
+Once the build is finished, you can proceed with launching a new instance of PMM Server, or installing a freshly built PMM Client, and testing the changes.
+
+
+## The `rpmbuild` image and docker cache
+
+We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`. It comes provisioned with tools which are required to build PMM artifacts, for example RPM packages. As a build tool, it offers a number of benefits, two most obvious of which are:
+
+- it frees the user from installing dependencies on their host machine
+- it leverages a very powerful docker caching system, which results in reduced build times
+
+During the first run, `build` will create a few directories on the host machine, which are necessary to make use of docker cache. Please be aware, that the docker container's user needs to be able to write to these directories. The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure that the directories we create on the host are owned by a user with the same uid and gid. If the build fails, this is the first thing to check.
+
+
+## Avoiding unnecessary builds
+
+Sometimes, the changes you make affect only PMM Client. Other times, they affect only PMM Server. Therefore, you may want to skip building parts of PMM. The `build` script offers several parameters to help control what you want to build.
+
+* --no-update: run the build tasks without pulling the changes from `pmm-submodules` repository
+* --update-only: pull changes from the repo without building PMM
+* --no-client: do not build the client, use the cached PMM Client artifacts
+* --no-client-docker: skip building PMM Client docker container
+* --no-server-rpm: skip building PMM Server RPM artifacts
+* --log-file <path>: change the path of the build log file
+
+It's important to note, however, that once all changes are made and tested, you most probably want to re-build both PMM Client and Server to test them together.
+
+
+## Target environments
+
+Currently, local builds target the following environments:
+- PMM Client
+  - tarball - virtually any amd64 Linux environment
+  - RPM - RHEL9-compatible environments
+  - docker image - docker and Kubernetes environments (amd64)
+- PMM Server
+  - docker image - docker and Kubernetes environments (amd64)
+