diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e39dbeaf05..a2e1fa0407 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -161,8 +161,8 @@ jobs: env: SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }} SLACK_CHANNEL: "pmm-ci" - SLACK_USERNAME: "PR pipelines bot" - SLACK_ICON_EMOJI: ":chestnut:" + SLACK_USERNAME: "Build bot" + SLACK_ICON_EMOJI: ":octocat:" SLACK_COLOR: "#00FF00" SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow" SLACK_MESSAGE: "${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}" @@ -179,8 +179,8 @@ jobs: env: SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }} SLACK_CHANNEL: "pmm-ci" - SLACK_USERNAME: "PR pipelines bot" - SLACK_ICON_EMOJI: ":chestnut:" + SLACK_USERNAME: "Build bot" + SLACK_ICON_EMOJI: ":octocat:" SLACK_COLOR: "#FF0000" SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow" SLACK_MESSAGE: "Workflow failed: ${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}" diff --git a/.gitignore b/.gitignore index da7596fcd5..e862c50236 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ api-tests/pmm-api-tests-output.txt api-tests/pmm-api-tests-junit-report.xml packer.log + +# Goreleaser +dist/ diff --git a/admin/Makefile b/admin/Makefile index f6c12416e1..b3f4249f03 100644 --- a/admin/Makefile +++ b/admin/Makefile @@ -11,6 +11,10 @@ PMM_RELEASE_VERSION ?= $(shell git describe --always --dirty | cut -b2-) PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s') PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD) PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all) + +ifdef NO_VCS + NO_VCS = -buildvcs=false +endif ifeq ($(GOBIN),) GOBIN := $(shell go env GOPATH)/bin endif @@ -31,8 +35,8 @@ clean: ## Remove generated files find . -name *_reform.go -delete release: ## Build pmm-admin release binary - env CGO_ENABLED=0 go build -v $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm-admin ./cmd/pmm-admin/ - env CGO_ENABLED=0 go build -v $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm ./cmd/pmm/ + env CGO_ENABLED=0 go build -v $(NO_VCS) $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm-admin ./cmd/pmm-admin/ + env CGO_ENABLED=0 go build -v $(NO_VCS) $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm ./cmd/pmm/ install: ## Install pmm & pmm-admin binary go build -v $(LD_FLAGS) -o $(GOBIN)/pmm-admin ./cmd/pmm-admin/ diff --git a/agent/Makefile b/agent/Makefile index 2507f82fcb..fb346bd539 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -12,6 +12,10 @@ PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s') PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD) PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all) PMM_DEV_SERVER_PORT ?= 8443 + +ifdef NO_VCS + NO_VCS = -buildvcs=false +endif ifeq ($(GOBIN),) GOBIN := $(shell go env GOPATH)/bin endif @@ -25,8 +29,8 @@ VERSION_FLAGS = -X 'github.com/percona/pmm/version.ProjectName=pmm-agent' \ -X 'github.com/percona/pmm/version.Branch=$(PMM_RELEASE_BRANCH)' release: ## Build static pmm-agent release binary (Linux only) - env CGO_ENABLED=1 go build -v -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent - go build -v -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent-entrypoint ./cmd/pmm-agent-entrypoint + env CGO_ENABLED=1 go build -v $(NO_VCS) -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent + go build -v $(NO_VCS) -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent-entrypoint ./cmd/pmm-agent-entrypoint $(PMM_RELEASE_PATH)/pmm-agent --version ldd $(PMM_RELEASE_PATH)/pmm-agent 2>&1 | grep -Fq 'not a dynamic executable' diff --git a/build/docker/client/Dockerfile b/build/docker/client/Dockerfile index a5d82c7165..a93c134b3b 100644 --- a/build/docker/client/Dockerfile +++ b/build/docker/client/Dockerfile @@ -5,19 +5,13 @@ RUN microdnf install shadow-utils jq tar -y RUN groupadd -g 1002 pmm-agent && \ useradd -u 1002 -r -g pmm-agent -s /sbin/nologin \ -d /usr/local/percona/pmm \ - -c "PMM 2.X Client User" pmm-agent + -c "PMM 3.x Client User" pmm-agent FROM redhat/ubi9-micro ARG VERSION ARG BUILD_DATE -LABEL org.opencontainers.image.created ${BUILD_DATE} -LABEL org.opencontainers.image.licenses Apache-2.0 -LABEL org.opencontainers.image.title Percona Monitoring and Management Client -LABEL org.opencontainers.image.vendor Percona -LABEL org.opencontainers.image.version ${VERSION} - COPY LICENSE /licenses/ COPY --from=users --chown=0:0 /etc/passwd /etc/passwd COPY --from=users --chown=0:0 /etc/group /etc/group @@ -49,10 +43,18 @@ RUN install -d -o pmm-agent -g pmm-agent -m 0775 /usr/local/percona/pmm ADD pmm-client.tar.gz /tmp/ RUN cd /tmp/pmm-client-* \ - && env PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \ + && PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \ && cd /tmp \ && rm -rf /tmp/pmm-client* +LABEL org.opencontainers.image.created ${BUILD_DATE} +LABEL org.opencontainers.image.licenses Apache-2.0 +LABEL org.opencontainers.image.title Percona Monitoring and Management Client +LABEL org.opencontainers.image.vendor Percona +LABEL org.opencontainers.image.version ${VERSION} +LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/client/Dockerfile +LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#docker + USER pmm-agent WORKDIR /usr/local/percona/pmm/ ENV PATH /usr/local/percona/pmm/bin/:$PATH diff --git a/build/docker/rpmbuild/Dockerfile.el9 b/build/docker/rpmbuild/Dockerfile.el9 index 239f73be9c..6f3c445b23 100644 --- a/build/docker/rpmbuild/Dockerfile.el9 +++ b/build/docker/rpmbuild/Dockerfile.el9 @@ -28,14 +28,16 @@ RUN yum install -y gcc gcc-c++ \ yum clean all && rm -rf /var/cache/yum # keep that format for easier search -ENV GO_VERSION 1.23.2 -ENV GO_RELEASER_VERSION 1.24.0 +ARG GO_VERSION +ENV GO_VERSION=${GO_VERSION:-1.23.2} +ENV GO_RELEASER_VERSION=2.4.1 +ENV NODE_OPTIONS=--max_old_space_size=8000 RUN if [ `uname -i` == "x86_64" ]; then ARCH=amd64; else ARCH=arm64; fi && \ curl -fSsL -o /tmp/golang.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-${ARCH}.tar.gz && \ curl -fSsL -o /tmp/goreleaser.rpm https://github.com/goreleaser/goreleaser/releases/download/v${GO_RELEASER_VERSION}/goreleaser-${GO_RELEASER_VERSION}-1.`uname -i`.rpm && \ tar -C /usr/local -xzf /tmp/golang.tar.gz && \ - yum install -y /tmp/goreleaser.rpm && \ + yum install -y --disablerepo="*" /tmp/goreleaser.rpm && \ rm /tmp/golang.tar.gz /tmp/goreleaser.rpm RUN update-alternatives --install "/usr/bin/go" "go" "/usr/local/go/bin/go" 0 @@ -47,7 +49,7 @@ RUN useradd builder -u 1000 -m -G users,wheel && \ echo "builder ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers && \ echo "# macros" > /home/builder/.rpmmacros && \ echo "%_topdir /home/builder/rpm" >> /home/builder/.rpmmacros && \ - mkdir /home/builder/rpm && \ + mkdir -p /home/builder/rpm /home/builder/.cache/go-build /home/builder/go/pkg/mod && \ chmod 755 /home/builder && \ chown -R builder:builder /home/builder diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9 index d5feb02671..5fc8efc221 100644 --- a/build/docker/server/Dockerfile.el9 +++ b/build/docker/server/Dockerfile.el9 @@ -4,7 +4,6 @@ ARG VERSION ARG BUILD_DATE ENV LANG=en_US.utf8 -ENV LC_ALL=en_US.utf8 ENV GF_PLUGIN_DIR=/srv/grafana/plugins ENV PS1="[\u@\h \W] # " @@ -33,11 +32,13 @@ RUN install -T -p -m 644 /opt/ansible/ansible.cfg /etc/ansible/ansible.cfg && \ ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm/post-build-actions.yml && \ sed -i '/^assumeyes/d' /etc/dnf/dnf.conf -LABEL org.opencontainers.image.created ${BUILD_DATE} -LABEL org.opencontainers.image.licenses AGPL-3.0 -LABEL org.opencontainers.image.title Percona Monitoring and Management -LABEL org.opencontainers.image.vendor Percona LLC -LABEL org.opencontainers.image.version ${VERSION} +LABEL org.opencontainers.image.created=${BUILD_DATE} +LABEL org.opencontainers.image.licenses=AGPL-3.0 +LABEL org.opencontainers.image.title="Percona Monitoring and Management" +LABEL org.opencontainers.image.vendor="Percona LLC" +LABEL org.opencontainers.image.version=${VERSION} +LABEL org.opencontainers.image.source=https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9 +LABEL org.opencontainers.image.docs=https://docs.percona.com/percona-monitoring-and-management/setting-up/server/docker.html USER pmm diff --git a/build/docker/server/create_users.sh b/build/docker/server/create_users.sh deleted file mode 100644 index e37eae3bf8..0000000000 --- a/build/docker/server/create_users.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -users=( - "pmm:1000:/bin/false:/home/pmm:pmm" - "nginx:1001:/sbin/nologin:/var/cache/nginx:nginx" -) - -for user in "${users[@]}"; do - IFS=: read -r name uid shell home_dir group <<< "$user" - group_id="$uid" - - # Check if user already exists - if id "$name" >/dev/null 2>&1; then - echo "User $name already exists" - continue - fi - - # Create user with home directory if it doesn't exist - if [ ! -d "$home_dir" ]; then - mkdir -p "$home_dir" - fi - - # Create user with specified UID, GID, and shell - groupadd -o -g "$group_id" "$group" - useradd -o -u "$uid" -g "$group" -G "$group" -s "$shell" -d "$home_dir" -c "$name" -m "$name" - chown "$uid:$group_id" "$home_dir" - -done - diff --git a/build/local/README.md b/build/local/README.md new file mode 100644 index 0000000000..7c1fac211a --- /dev/null +++ b/build/local/README.md @@ -0,0 +1,77 @@ +# Local builds + +This directory contains a set of scripts aimed at providing a simple way to build PMM locally. + +## Background + +Historically, PMM used to be built using Jenkins. This worked well for the team, but not for the community. The learning curve was, and still is, rather steep, and it is hard for folks, even internally, to contribute to. + +Therefore, we decided to make it possible to build PMM locally. This is a work in progress, but we are definitely committed to bring the developer experience to an acceptable level. + +The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on, since every developer should have at least a basic command of bash. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching. + +The build process is designed to run on a Linux host. We believe it can run on other flavors of Linux, including MacOS, with little to no modification (TBC). + + +## Prerequisites + +Below is a list of prerequisites that are required to build PMM locally. + +- OS: Linux (tested on Oracle Linux 9.3, CentOS 7, Ubuntu 22.04.3 LTS) +- Docker: 25.0.2+ (tested on 25.0.2) +- Docker Compose Plugin: 2.24.7+ (tested on 2.24.7) +- make: 4.3+ (tested on 4.3) +- bash (GNU): 5.1+ (tested on 5.1) +- git: 2.34.1+ (tested on 2.34.1) +- curl: 7.81.0+ (tested on 7.81.0) +- yq: 4.42.0+ (tested on 4.42.1) +- jq: 1.6+ (tested on 1.6) + + +## How to build PMM + +1. Install the prerequisites +2. Clone the PMM repository to the user's home directory, e.g.: `git clone https://github.com/percona/pmm /home/user/pmm`. +3. Change to the `build/local` directory in the cloned repo. +4. Run `./build --help` to print the help message and check the usage. +5. Run `./build` with parameters of your choice to build PMM v3. + +Usually, you will want to rebuild PMM whenever there are changes in at least one of its components. All components of PMM are gathered together in one repository - `github.com/percona-lab/pmm-submodules` (or `pmm-submodules`). Therefore, you can run `build` as often as those changes need to be factored in to the next build. + +Once the build is finished, you can proceed with launching a new instance of PMM Server, or installing a freshly built PMM Client, and testing the changes. + + +## The `rpmbuild` image and docker cache + +We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`. It comes provisioned with tools which are required to build PMM artifacts, for example RPM packages. As a build tool, it offers a number of benefits, two most obvious of which are: + +- it frees the user from installing dependencies on their host machine +- it leverages a very powerful docker caching system, which results in reduced build times + +During the first run, `build` will create a few directories on the host machine, which are necessary to make use of docker cache. Please be aware, that the docker container's user needs to be able to write to these directories. The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure that the directories we create on the host are owned by a user with the same uid and gid. If the build fails, this is the first thing to check. + + +## Avoiding unnecessary builds + +Sometimes, the changes you make affect only PMM Client. Other times, they affect only PMM Server. Therefore, you may want to skip building parts of PMM. The `build` script offers several parameters to help control what you want to build. + +* --no-update: run the build tasks without pulling the changes from `pmm-submodules` repository +* --update-only: pull changes from the repo without building PMM +* --no-client: do not build the client, use the cached PMM Client artifacts +* --no-client-docker: skip building PMM Client docker container +* --no-server-rpm: skip building PMM Server RPM artifacts +* --log-file : change the path of the build log file + +It's important to note, however, that once all changes are made and tested, you most probably want to re-build both PMM Client and Server to test them together. + + +## Target environments + +Currently, local builds target the following environments: +- PMM Client + - tarball - virtually any amd64 Linux environment + - RPM - RHEL9-compatible environments + - docker image - docker and Kubernetes environments (amd64) +- PMM Server + - docker image - docker and Kubernetes environments (amd64) + diff --git a/build/local/build b/build/local/build new file mode 100755 index 0000000000..3c65ca4472 --- /dev/null +++ b/build/local/build @@ -0,0 +1,398 @@ +#!/bin/bash -e +set -o errexit +set -o nounset + +usage() { + cat <<-EOF +Usage: $BASE_NAME [OPTIONS] +Options: + --platform Build for a specific platform (defaults to linux/amd64) + --no-update Do not fetch the latest changes from the repo + --update-only Only fetch the latest changes from the repo + --no-client Do not build PMM Client + --no-client-docker Do not build PMM Client docker image (default) + --client-only Build only PMM Client (client binaries + docker) + --no-server Do not build PMM Server (docker image) + --log-file Save build logs to a file located at (defaults to PWD) + --help | -h Display help +EOF +} + +parse-params() { + # Define global variables + LOCAL_BUILD=1 + NO_UPDATE=0 + UPDATE_ONLY=0 + NO_CLIENT=0 + NO_CLIENT_DOCKER=1 + NO_SERVER=0 + START_TIME=$(date +%s) + LOG_FILE="$(dirname $0)/build.log" + BASE_NAME=$(basename $0) + PLATFORM=linux/amd64 + SUBMODULES=pmm-submodules + PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts" + + while test "$#" -gt 0; do + case "$1" in + --update-only) + UPDATE_ONLY=1; NO_UPDATE=0 + ;; + --no-update) + if [ "$UPDATE_ONLY" -eq 1 ]; then + echo "Error. Mutually exclusive options: --update-only and --no-update" + exit 1 + fi + NO_UPDATE=1 + ;; + --client-only) + NO_CLIENT=0; NO_CLIENT_DOCKER=0; NO_SERVER=1 + ;; + --no-client) + NO_CLIENT=1; NO_CLIENT_DOCKER=1 + ;; + --client-docker) + if [ "$NO_CLIENT" -eq 1 ]; then + echo "Error. Mutually exclusive options: --client-docker and --no-client" + exit 1 + fi + NO_CLIENT_DOCKER=0 + ;; + --no-client-docker) + if [ "$NO_CLIENT_DOCKER" -eq 1 ]; then + echo "Error. Mutually exclusive options: --client-docker and --no-client-docker" + exit 1 + fi + NO_CLIENT_DOCKER=1 + ;; + --no-server) + NO_SERVER=1 + ;; + --platform) + shift + if [ -z "$1" ]; then + echo "Missing argument for --platform" + exit 1 + fi + PLATFORM="$1" + ;; + --log-file) + shift + if [ -z "$1" ]; then + echo "Missing argument for --log-file" + exit 1 + fi + LOG_FILE="$1" + ;; + --help | -h) + shift + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" + usage + exit 1 + ;; + esac + shift + done +} + +needs-to-pull() { + local UPSTREAM=${1:-'@{u}'} + local LOCAL=$(git rev-parse @) + local BASE=$(git merge-base @ "$UPSTREAM") + local REMOTE=$(git rev-parse "$UPSTREAM") + + if [ "$LOCAL" = "$REMOTE" ]; then + return 1 # false, we are up-to-date + fi + + if [ "$LOCAL" = "$BASE" ]; then + return 0 # true, we are behind upstream + fi +} + +rewind() { + local DIR="$1" + local BRANCH="$2" + + cd "$DIR" > /dev/null + local CURRENT=$(git branch --show-current) + git fetch + + if [ "$CURRENT" != "$BRANCH" ]; then + echo "Currently on $CURRENT, checking out $BRANCH" + git checkout "$BRANCH" + fi + + if needs-to-pull; then + git pull origin + echo "Submodule has pulled from upstream" + git logs -n 2 + cd - > /dev/null + git add "$DIR" + else + cd - > /dev/null + echo "Submodule is up-to-date with upstream" + fi +} + +check-files() { + local DIR="$1" + + test -z "DIR" && exit 1 + + if [ -d "$DIR/sources" ] && [ -f "$DIR/ci-default.yml" ] && [ -f "$DIR/ci.yml" ]; then + return 0 + fi + + return 1 +} + +# Update submodules and PR branches +update() { + local DEPS= + local CURDIR="$PWD" + local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build" + local MD5SUM=$(md5sum $(dirname $0)/build) + + if [ "$NO_UPDATE" -eq 1 ]; then + echo "Running without refreshing the source code from repositories..." + return + fi + + # Thouroughly verify the presence of known files, otherwise bail out + if [ ! -d "$SUBMODULES" ] ; then # pwd must be outside of pmm-submodules + echo "Warn: the current working directory must be outside of pmm-submodules" + echo "cd .." + cd .. > /dev/null + fi + + if [ -d "$SUBMODULES" ]; then # pwd is outside pmm-submodules + if ! check-files "$SUBMODULES"; then + echo "Fatal: could not locate known files in ${PWD}/${SUBMODULES}" + exit 1 + fi + else + echo "Fatal: could not locate known files in $PWD" + exit 1 + fi + + cd "$SUBMODULES" + + # Join the dependencies from ci-default.yml and ci.yml + DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps') + + echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'" + + echo "$DEPS" | jq -c '.[]' | while read -r item; do + branch=$(echo "$item" | jq -r '.branch') + path=$(echo "$item" | jq -r '.path') + name=$(echo "$item" | jq -r '.name') + echo + echo "Rewinding submodule '$name' ..." + echo "path: ${path}, branch: ${branch}" + + rewind "$path" "$branch" + done + + echo + echo "Printing git status..." + git status --short + echo + echo "Printing git submodule status..." + git submodule status + + cd "$CURDIR" > /dev/null + + if [ -f "$UPDATED_SCRIPT" ] && [ "$MD5SUM" != "$(md5sum $UPDATED_SCRIPT)" ]; then + echo "The local copy of this script differs from the one fetched from the repo." + echo "Apparently, that version is newer. We will halt to give you the change to run a fresh version." + echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build --no-update'" + exit 0 + fi +} + +get_branch_name() { + local module="${1:-}" + local path=$(git config -f .gitmodules submodule.${module}.path) + + if [ ! -d "$path" ]; then + echo "Error: could not resolve the path to submodule ${module}" + exit 1 + fi + + echo $(git -C "$path" branch --show-current) +} + +run_build_script() { + local CURDIR="$PWD" + local script="$PATH_TO_SCRIPTS/$1" + local script_name="$1" + local start_time + local end_time + + cd "$SUBMODULES" > /dev/null + + if [ ! -f "$script" ]; then + echo "Fatal: script $script does not exist." + cd "$CURDIR" > /dev/null + exit 1 + fi + + start_time=$(date +%s) + if [ "$#" -gt 1 ]; then + shift + script_name="${script_name}:($1)" + $script "$@" + else + $script + fi + end_time=$(date +%s) + + echo --- + echo "Execution time (in sec) for $script_name: $((end_time - start_time))" + echo --- + + cd "$CURDIR" > /dev/null +} + +purge_files() { + local CURDIR=$PWD + local tmp_files + + cd "$SUBMODULES" > /dev/null + # Remove stale files and directories + if [ -d tmp ]; then + echo "Removing stale files and directories..." + + if [ -d "tmp/pmm-server" ]; then + tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$" || :) + if [ -n "$tmp_files" ]; then + tmp_files=( $tmp_files ) + for f in "${tmp_files[@]}"; do + echo "Removing file or directory $f ..." + rm -rf "$f" + done + fi + fi + + if [ -d "tmp/source/pmm" ]; then + echo "Removing tmp/source/pmm ..." + rm -rf tmp/source/pmm + fi + fi + + if [ -d results ]; then + echo "Removing results/* ..." + rm -rf results/{rpm,srpm,tarball,source_tarball,pmm-client.properties} + fi + + cd "$CURDIR" +} + +init() { + local CURDIR="$PWD" + + if [ -d "$SUBMODULES" ]; then + cd "$SUBMODULES" > /dev/null + fi + + export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3 + + GIT_COMMIT=$(git rev-parse HEAD | head -c 8) + + # Create docker volume to persist package and build cache + # Read more in the section about `rpmbuild`. + if ! docker volume ls | grep pmm-gobuild >/dev/null; then + docker volume create pmm-gobuild + fi + if ! docker volume ls | grep pmm-gomod >/dev/null; then + docker volume create pmm-gomod + fi + if ! docker volume ls | grep pmm-yarn >/dev/null; then + docker volume create pmm-yarn + fi + if ! docker volume ls | grep pmm-dnf >/dev/null; then + docker volume create pmm-dnf + fi + + cd "$CURDIR" > /dev/null +} + +cleanup() { + local CURDIR="$PWD" + cd "$SUBMODULES" > /dev/null + + # Implement cleanup logic here + + cd "$CURDIR" > /dev/null +} + +main() { + update + + init + + purge_files + + if [ "$NO_CLIENT" -eq 0 ]; then + # Build client source: 4m39s from scratch, 0m27s using cache + run_build_script build-client-source + + # Build client binary: ??? from scratch, 0m20s using cache + run_build_script build-client-binary + + # Building client source rpm takes 13s (caching does not apply) + run_build_script build-client-srpm + + # Building client rpm takes 1m40s + run_build_script build-client-rpm + fi + + # Building client docker image takes from 17s (using docker cache) to 43s (no docker cache). + export DOCKER_CLIENT_TAG=percona/pmm-client:${GIT_COMMIT} + if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then + run_build_script build-client-docker + fi + + if [ "$NO_SERVER" -eq 0 ]; then + # Grafana build fails to compile with Go 1.23.x, see https://github.com/grafana/grafana/issues/89796 + # We need to apply this [patch](https://github.com/grafana/grafana/pull/94742) to fix it + export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3 + export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT} + export DOCKERFILE=Dockerfile.el9 + export RPM_EPOCH=1 + + # 3rd-party components + run_build_script build-server-rpm grafana + run_build_script build-server-rpm victoriametrics + + # 1st-party components + run_build_script build-server-rpm percona-dashboards grafana-dashboards + run_build_script build-server-rpm pmm-managed pmm + run_build_script build-server-rpm percona-qan-api2 pmm + run_build_script build-server-rpm pmm-update pmm + run_build_script build-server-rpm pmm-dump + run_build_script build-server-rpm vmproxy pmm + + run_build_script build-server-docker + fi + + echo + echo "Done building PMM artifacts." + echo --- + echo "Total execution time, sec: $(($(date +%s) - $START_TIME))" + echo --- + + cleanup +} + +parse-params "$@" + +# Capture the build logs in the log file +exec > >(tee "$LOG_FILE") 2>&1 + +main diff --git a/build/local/ci-default.yml b/build/local/ci-default.yml new file mode 100644 index 0000000000..c344eeac13 --- /dev/null +++ b/build/local/ci-default.yml @@ -0,0 +1,89 @@ +deps: + # COMMON + - name: pmm + branch: v3 + path: sources/pmm/src/github.com/percona/pmm + url: https://github.com/percona/pmm + component: common + + # CLIENT + - name: node_exporter + branch: main + path: sources/node_exporter/src/github.com/prometheus/node_exporter + url: https://github.com/percona/node_exporter + component: client + + - name: mysqld_exporter + branch: main + path: sources/mysqld_exporter/src/github.com/percona/mysqld_exporter + url: https://github.com/percona/mysqld_exporter + component: client + + - name: mongodb_exporter + branch: main + path: sources/mongodb_exporter/src/github.com/percona/mongodb_exporter + url: https://github.com/percona/mongodb_exporter + component: client + + - name: postgres_exporter + branch: main + path: sources/postgres_exporter/src/github.com/percona/postgres_exporter + url: https://github.com/percona/postgres_exporter + component: client + + - name: proxysql_exporter + branch: main + path: sources/proxysql_exporter/src/github.com/percona/proxysql_exporter + url: https://github.com/percona/proxysql_exporter + component: client + + - name: rds_exporter + branch: main + path: sources/rds_exporter/src/github.com/percona/rds_exporter + url: https://github.com/percona/rds_exporter + component: client + + - name: azure_metrics_exporter + branch: main + path: sources/azure_metrics_exporter/src/github.com/percona/azure_metrics_exporter + url: https://github.com/percona/azure_metrics_exporter + component: client + + - name: percona-toolkit + branch: release-v3.5.2 + path: sources/percona-toolkit/src/github.com/percona/percona-toolkit + url: https://github.com/percona/percona-toolkit + component: client + + # SERVER + + - name: grafana-dashboards + branch: v3 + path: sources/grafana-dashboards + url: https://github.com/percona/grafana-dashboards + component: server + + - name: grafana + branch: v3 + path: sources/grafana/src/github.com/grafana/grafana + url: https://github.com/percona/grafana + component: server + + - name: pmm-dump + branch: main + path: sources/pmm-dump + url: https://github.com/percona/pmm-dump + component: server + + # QA + - name: pmm-qa + branch: v3 + path: sources/pmm-qa/src/github.com/percona/pmm-qa + url: https://github.com/percona/pmm-qa + component: qa + + - name: pmm-ui-tests + branch: v3 + path: sources/pmm-ui-tests/src/github.com/percona/pmm-ui-tests + url: https://github.com/percona/pmm-ui-tests + component: qa \ No newline at end of file diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec index a90b2f40ac..4e3eba1aec 100644 --- a/build/packages/rpm/server/SPECS/grafana.spec +++ b/build/packages/rpm/server/SPECS/grafana.spec @@ -1,15 +1,15 @@ -%global debug_package %{nil} -%global commit 7dd51483f34ce324e603160f415395fce0bc55a1 +%global commit f2a6d70344f94674f731e6e9b031a6f147de46cc %global shortcommit %(c=%{commit}; echo ${c:0:7}) -%define build_timestamp %(date -u +"%y%m%d%H%M") %define release 106 %define grafana_version 11.1.5 %define full_pmm_version 2.0.0 %define full_version v%{grafana_version}-%{full_pmm_version} -%define rpm_release %{release}.%{build_timestamp}.%{shortcommit}%{?dist} +%define rpm_release %{release}.%{shortcommit}%{?dist} %if ! 0%{?gobuild:1} -%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom|od -An -tx1|tr -d ' \\n')" -a -v -x %{?**}; +# https://github.com/rpm-software-management/rpm/issues/367 +# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID +%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**}; %endif Name: percona-grafana @@ -19,24 +19,20 @@ Summary: Grafana is an open source, feature rich metrics dashboard and gr License: AGPLv3 URL: https://github.com/percona/grafana Source0: https://github.com/percona/grafana/archive/%{commit}.tar.gz -ExclusiveArch: %{ix86} x86_64 %{arm} +ExclusiveArch: %{ix86} x86_64 %{arm} aarch64 BuildRequires: fontconfig -%if 0%{?rhel} < 9 -BuildRequires: nodejs-grunt-cli -%endif %description -Grafana is an open source, feature rich metrics dashboard and graph editor for -Graphite, InfluxDB & OpenTSDB. +Grafana is an open source observability and data visualization platform. +Visualize metrics, logs, and traces from multiple sources like +Prometheus, Loki, Elasticsearch, InfluxDB, Postgres and many more. %prep %setup -q -n grafana-%{commit} rm -rf Godeps sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go -%if 0%{?rhel} >= 9 - sudo npm install -g grunt-cli -%endif +sudo npm install -g grunt-cli %build mkdir -p _build/src @@ -79,7 +75,7 @@ install -d -p %{buildroot}%{_sharedstatedir}/grafana %pre getent group pmm >/dev/null || echo "Group pmm does not exist. Please create it manually." getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it manually." -exit 0 + %changelog * Fri Sep 06 2024 Matej Kubinec - 11.1.5-1 @@ -94,6 +90,9 @@ exit 0 * Tue Apr 16 2024 Matej Kubinec - 10.4.2-1 - PMM-13059 Grafana 10.4.2 +* Wed Mar 20 2024 Alex Demidoff - 10.4.0-2 +- PMM-12899 Use module and build cache + * Tue Mar 12 2024 Matej Kubinec - 10.4.0-1 - PMM-12991 Grafana 10.4.0 diff --git a/build/packages/rpm/server/SPECS/percona-dashboards.spec b/build/packages/rpm/server/SPECS/percona-dashboards.spec index eb964ce17f..7b85359520 100644 --- a/build/packages/rpm/server/SPECS/percona-dashboards.spec +++ b/build/packages/rpm/server/SPECS/percona-dashboards.spec @@ -1,27 +1,24 @@ -%global debug_package %{nil} %global __strip /bin/true -%global repo grafana-dashboards -%global provider github.com/percona/%{repo} -%global import_path %{provider} -%global commit ad4af6808bcd361284e8eb8cd1f36b1e98e32bce -%global shortcommit %(c=%{commit}; echo ${c:0:7}) -%define build_timestamp %(date -u +"%y%m%d%H%M") +%global repo grafana-dashboards +%global provider github.com/percona/%{repo} +%global import_path %{provider} +%global commit ad4af6808bcd361284e8eb8cd1f36b1e98e32bce +%global shortcommit %(c=%{commit}; echo ${c:0:7}) %define release 22 -%define rpm_release %{release}.%{build_timestamp}.%{shortcommit}%{?dist} +%define rpm_release %{release}.%{shortcommit}%{?dist} -Name: percona-dashboards -Version: %{version} -Release: %{rpm_release} -Summary: Grafana dashboards for monitoring +Name: percona-dashboards +Version: %{version} +Release: %{rpm_release} +Summary: Grafana dashboards for monitoring -License: AGPLv3 -URL: https://%{provider} -Source0: https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz +License: AGPLv3 +URL: https://%{provider} +Source0: https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz BuildRequires: nodejs Requires: percona-grafana -Provides: percona-grafana-dashboards = %{version}-%{release} %description This is a set of Grafana dashboards for database and system monitoring @@ -55,6 +52,9 @@ echo %{version} > %{buildroot}%{_datadir}/%{name}/VERSION %changelog +* Sat Oct 12 2024 Alex Demidoff - 3.0.0-23 +- PMM-12899 Use module and build cache + * Tue Jul 23 2024 Nurlan Moldomurov - 3.0.0-22 - PMM-13053 Remove /setup page diff --git a/build/packages/rpm/server/SPECS/percona-qan-api2.spec b/build/packages/rpm/server/SPECS/percona-qan-api2.spec index 14e89677b5..3f851cb36f 100644 --- a/build/packages/rpm/server/SPECS/percona-qan-api2.spec +++ b/build/packages/rpm/server/SPECS/percona-qan-api2.spec @@ -1,8 +1,3 @@ -# Go build id is not supported for now. -# https://github.com/rpm-software-management/rpm/issues/367 -# https://bugzilla.redhat.com/show_bug.cgi?id=1295951 -%undefine _missing_build_ids_terminate_build - %global repo pmm %global provider github.com/percona/%{repo} %global import_path %{provider} @@ -10,9 +5,8 @@ # see: https://github.com/percona/pmm/blob/main/build/scripts/build-server-rpm#L58 %global commit 0000000000000000000000000000000000000000 %global shortcommit %(c=%{commit}; echo ${c:0:7}) -%define build_timestamp %(date -u +"%y%m%d%H%M") -%define release 17 -%define rpm_release %{release}.%{build_timestamp}.%{shortcommit}%{?dist} +%define release 18 +%define rpm_release %{release}.%{shortcommit}%{?dist} # the line below is sed'ed by build/bin/build-server-rpm to set a correct version %define full_pmm_version 2.0.0 @@ -20,7 +14,7 @@ Name: percona-qan-api2 Version: %{version} Release: %{rpm_release} -Summary: Query Analytics API v2 for PMM +Summary: Query Analytics API for PMM License: AGPLv3 URL: https://%{provider} @@ -32,10 +26,7 @@ See PMM docs for more information - https://docs.percona.com/percona-monitoring- %prep -%setup -T -c -n %{repo}-%{version} -%setup -q -c -a 0 -n %{repo}-%{version} -mkdir -p src/github.com/percona -mv %{repo}-%{commit} src/%{provider} +%setup -q -n %{repo}-%{commit} %build @@ -43,29 +34,31 @@ export PMM_RELEASE_VERSION=%{full_pmm_version} export PMM_RELEASE_FULLCOMMIT=%{commit} export PMM_RELEASE_BRANCH="" -cd src/%{provider}/qan-api2 -make release +make -C qan-api2 release %install install -d -p %{buildroot}%{_sbindir} -install -p -m 0755 src/%{provider}/bin/qan-api2 %{buildroot}%{_sbindir}/%{name} +install -p -m 0755 ./bin/qan-api2 %{buildroot}%{_sbindir}/%{name} %files %attr(0755, root, root) %{_sbindir}/%{name} -%license src/%{provider}/qan-api2/LICENSE -%doc src/%{provider}/qan-api2/README.md +%license qan-api2/LICENSE +%doc qan-api2/README.md %changelog -* Mon Nov 7 2022 Alexander Tymchuk - 2.0.0-17 +* Wed Apr 1 2024 Alex Demidoff - 3.0.0-18 +- PMM-12899 Use module and build cache + +* Mon Nov 7 2022 Alexander Tymchuk - 2.0.0-17 - PMM-10117 migrate QAN API to monorepo * Mon May 16 2022 Nikita Beletskii - 2.0.0-16 - PMM-10027 remove useless packages -* Thu Jul 2 2020 Mykyta Solomko - 2.0.0-15 +* Thu Jul 2 2020 Mykyta Solomko - 2.0.0-15 - PMM-5645 built using Golang 1.14 * Tue Mar 19 2019 Vadim Yalovets - 2.0.0-4 diff --git a/build/packages/rpm/server/SPECS/pmm-dump.spec b/build/packages/rpm/server/SPECS/pmm-dump.spec index 9a5e04cf14..434c034621 100644 --- a/build/packages/rpm/server/SPECS/pmm-dump.spec +++ b/build/packages/rpm/server/SPECS/pmm-dump.spec @@ -4,18 +4,17 @@ %global provider github.com/percona/%{repo} %global commit f226dbb3afb62ac4b9b39032935b5694a48d526f %global shortcommit %(c=%{commit}; echo ${c:0:7}) -%define build_timestamp %(date -u +"%y%m%d%H%M") -%define release 1 -%define rpm_release %{release}.%{build_timestamp}.%{shortcommit}%{?dist} +%define release 2 +%define rpm_release %{release}.%{shortcommit}%{?dist} -Name: pmm-dump -Version: 3.0.0 -Release: %{rpm_release} -Summary: Percona PMM Dump allows to export and import monitoring metrics and query analytics. +Name: pmm-dump +Version: 3.0.0 +Release: %{rpm_release} +Summary: Percona PMM Dump allows to export and import monitoring metrics and query analytics. -License: AGPLv3 -URL: https://%{provider} -Source0: https://%{provider}/archive/%{commit}.tar.gz +License: AGPLv3 +URL: https://%{provider} +Source0: https://%{provider}/archive/%{commit}.tar.gz %description %{summary} @@ -40,6 +39,9 @@ install -p -m 0755 pmm-dump %{buildroot}%{_sbindir}/pmm-dump * Thu Aug 8 2024 Alex Demidoff - 3.0.0-1 - PMM-13282 Migrate pmm-dump to v3 API +* Mon Apr 1 2024 Alex Demidoff - 0.7.0-2 +- PMM-12899 Use module and build cache + * Tue Nov 23 2023 Artem Gavrilov - 0.7.0-ga - PMM-12460 Update pmm-dump to v0.7.0-ga version diff --git a/build/packages/rpm/server/SPECS/pmm-managed.spec b/build/packages/rpm/server/SPECS/pmm-managed.spec index b1da81ea78..a7ad40049e 100644 --- a/build/packages/rpm/server/SPECS/pmm-managed.spec +++ b/build/packages/rpm/server/SPECS/pmm-managed.spec @@ -5,21 +5,20 @@ %global provider github.com/percona/%{repo} %global commit 8f3d007617941033867aea6a134c48b39142427f %global shortcommit %(c=%{commit}; echo ${c:0:7}) -%define build_timestamp %(date -u +"%y%m%d%H%M") -%define release 20 -%define rpm_release %{release}.%{build_timestamp}.%{shortcommit}%{?dist} +%define release 21 +%define rpm_release %{release}.%{shortcommit}%{?dist} # the line below is sed'ed by build/bin/build-server-rpm to set a correct version %define full_pmm_version 2.0.0 -Name: pmm-managed -Version: %{version} -Release: %{rpm_release} -Summary: Percona Monitoring and Management management daemon +Name: pmm-managed +Version: %{version} +Release: %{rpm_release} +Summary: Percona Monitoring and Management management daemon -License: AGPLv3 -URL: https://%{provider} -Source0: https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz +License: AGPLv3 +URL: https://%{provider} +Source0: https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz %description pmm-managed manages configuration of PMM server components (VictoriaMetrics, @@ -28,22 +27,15 @@ See PMM docs for more information. %prep -%setup -q -n pmm-%{commit} -mkdir -p src/github.com/percona -ln -s $(pwd) src/%{provider} - +%setup -q -n %{repo}-%{commit} %build - export PMM_RELEASE_VERSION=%{full_pmm_version} export PMM_RELEASE_FULLCOMMIT=%{commit} export PMM_RELEASE_BRANCH="" -cd src/github.com/percona/pmm/managed -make release - -cd ../ui -make release +make -C managed release +make -C ui release %install install -d -p %{buildroot}%{_bindir} @@ -54,13 +46,12 @@ install -p -m 0755 bin/pmm-managed %{buildroot}%{_sbindir}/pmm-managed install -p -m 0755 bin/pmm-managed-init %{buildroot}%{_sbindir}/pmm-managed-init install -p -m 0755 bin/pmm-managed-starlark %{buildroot}%{_sbindir}/pmm-managed-starlark -cd src/github.com/percona/pmm -cp -pa ./api/swagger %{buildroot}%{_datadir}/%{name} -cp -pa ./ui/dist/. %{buildroot}%{_datadir}/pmm-ui +cp -pa api/swagger %{buildroot}%{_datadir}/%{name} +cp -pa ui/dist/. %{buildroot}%{_datadir}/pmm-ui %files -%license src/%{provider}/LICENSE -%doc src/%{provider}/README.md +%license managed/LICENSE +%doc managed/README.md %{_sbindir}/pmm-managed %{_sbindir}/pmm-managed-init %{_sbindir}/pmm-managed-starlark @@ -68,6 +59,9 @@ cp -pa ./ui/dist/. %{buildroot}%{_datadir}/pmm-ui %{_datadir}/pmm-ui %changelog +* Mon Apr 1 2024 Alex Demidoff - 3.0.0-2 +- PMM-12899 Use module and build cache + * Fri Mar 22 2024 Matej Kubinec - 3.0.0-1 - PMM-11231 add pmm ui diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec index e68fc9f55d..5e93dea47a 100644 --- a/build/packages/rpm/server/SPECS/pmm-update.spec +++ b/build/packages/rpm/server/SPECS/pmm-update.spec @@ -1,5 +1,4 @@ %undefine _missing_build_ids_terminate_build - # TODO: remove it as soon as we remove all noarch pmm-update rpms # from 'pmm3-components/yum/laboratory' %define _binaries_in_noarch_packages_terminate_build 0 @@ -10,20 +9,19 @@ %global import_path %{provider} %global commit 592eddf656bce32a11bd958af0a32c62bd5ea34c %global shortcommit %(c=%{commit}; echo ${c:0:7}) -%define build_timestamp %(date -u +"%y%m%d%H%M") -%define release 67 -%define rpm_release %{release}.%{build_timestamp}.%{shortcommit}%{?dist} +%define release 68 +%define rpm_release %{release}.%{shortcommit}%{?dist} # the line below is sed'ed by build/bin/build-server-rpm to set a correct version %define full_pmm_version 2.0.0 -Name: pmm-update +Name: pmm-update Version: %{full_pmm_version} Release: %{rpm_release} Summary: Tool for updating packages and OS configuration for PMM Server License: AGPLv3 -URL: https://%{provider} +URL: https://%{provider} Source0: https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz BuildArch: noarch @@ -34,35 +32,30 @@ BuildArch: noarch %prep %setup -q -n %{repo}-%{commit} -mkdir -p src/github.com/percona -ln -s $(pwd) src/%{provider} %build export PMM_RELEASE_VERSION=%{full_pmm_version} export PMM_RELEASE_FULLCOMMIT=%{commit} export PMM_RELEASE_BRANCH="" -cd src/github.com/percona/pmm/update -make release +make -C update release %install -install -d %{buildroot}%{_datadir}/%{name} -cp -pav ./update/ansible %{buildroot}%{_datadir}/%{name} - install -d %{buildroot}%{_sbindir} -cd src/github.com/percona/pmm/update -install -p -m 0755 bin/pmm-update %{buildroot}%{_sbindir}/ +install -p -m 0755 update/bin/pmm-update %{buildroot}%{_sbindir}/ %files -%license LICENSE -%doc README.md +%license update/LICENSE +%doc update/README.md %{_sbindir}/pmm-update -%{_datadir}/%{name} %changelog +* Mon Apr 1 2024 Alex Demidoff - 3.0.0-68 +- PMM-12899 Use module and build cache + * Thu Dec 8 2022 Michal Kralik - 2.34.0-67 - PMM-11207 Migrate pmm-update to monorepo diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec index ec55d239d1..97e2b39a78 100644 --- a/build/packages/rpm/server/SPECS/victoriametrics.spec +++ b/build/packages/rpm/server/SPECS/victoriametrics.spec @@ -1,19 +1,12 @@ %undefine _missing_build_ids_terminate_build -%define copying() \ -%if 0%{?fedora} >= 21 || 0%{?rhel} >= 7 \ -%license %{*} \ -%else \ -%doc %{*} \ -%endif - %global repo VictoriaMetrics %global provider github.com/VictoriaMetrics/%{repo} %global commit pmm-6401-v1.93.4 Name: percona-victoriametrics Version: 1.93.4 -Release: 1%{?dist} +Release: 2%{?dist} Summary: VictoriaMetrics monitoring solution and time series database License: Apache-2.0 URL: https://%{provider} @@ -50,6 +43,9 @@ install -D -p -m 0755 ./bin/vmalert-pure %{buildroot}%{_sbindir}/vmalert %changelog +* Mon Apr 1 2024 Alex Demidoff - 1.93.4-2 +- PMM-12899 Use module and build cache + * Thu Sep 14 2023 Alex Tymchuk - 1.93.4-1 - upgrade victoriametrics to 1.93.4 release diff --git a/build/packages/rpm/server/SPECS/vmproxy.spec b/build/packages/rpm/server/SPECS/vmproxy.spec index be5cdca4ca..11ca103785 100644 --- a/build/packages/rpm/server/SPECS/vmproxy.spec +++ b/build/packages/rpm/server/SPECS/vmproxy.spec @@ -5,21 +5,20 @@ %global provider github.com/percona/%{repo} %global commit 8f74cea10d85e441ee88ef4b12bc47bc05165ba9 %global shortcommit %(c=%{commit}; echo ${c:0:7}) -%define build_timestamp %(date -u +"%y%m%d%H%M") -%define release 1 -%define rpm_release %{release}.%{build_timestamp}.%{shortcommit}%{?dist} +%define release 2 +%define rpm_release %{release}.%{shortcommit}%{?dist} # the line below is sed'ed by build/bin/build-server-rpm to set a correct version %define full_pmm_version 2.0.0 -Name: vmproxy -Version: %{full_pmm_version} -Release: %{rpm_release} -Summary: Percona VMProxy stateless reverse proxy for VictoriaMetrics +Name: vmproxy +Version: %{full_pmm_version} +Release: %{rpm_release} +Summary: Percona VMProxy stateless reverse proxy for VictoriaMetrics -License: AGPLv3 -URL: https://%{provider} -Source0: https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz +License: AGPLv3 +URL: https://%{provider} +Source0: https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz %description VMProxy is a stateless reverse proxy which proxies requests to VictoriaMetrics and @@ -27,33 +26,31 @@ optionally adds `extra_filters` query based on the provided configuration. %prep -%setup -q -n pmm-%{commit} -mkdir -p src/github.com/percona -ln -s $(pwd) src/%{provider} +%setup -q -n %{repo}-%{commit} %build - export PMM_RELEASE_VERSION=%{full_pmm_version} export PMM_RELEASE_FULLCOMMIT=%{commit} export PMM_RELEASE_BRANCH="" -cd src/github.com/percona/pmm/vmproxy -make release +make -C vmproxy release %install -install -d -p %{buildroot}%{_bindir} install -d -p %{buildroot}%{_sbindir} -install -p -m 0755 bin/vmproxy %{buildroot}%{_sbindir}/vmproxy +install -p -m 0755 ./bin/vmproxy %{buildroot}%{_sbindir}/vmproxy %files -%license src/%{provider}/vmproxy/LICENSE -%doc src/%{provider}/vmproxy/README.md +%license vmproxy/LICENSE +%doc vmproxy/README.md %{_sbindir}/vmproxy %changelog +* Mon Apr 1 2024 Alex Demidoff - 3.0.0-2 +- PMM-12899 Use module and build cache + * Mon Dec 5 2022 Michal Kralik - 2.34.0-1 - Initial release of VMProxy diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary index 1a52fff323..f20b905ddc 100755 --- a/build/scripts/build-client-binary +++ b/build/scripts/build-client-binary @@ -5,6 +5,9 @@ set -o xtrace . $(dirname $0)/vars +# Important: `bin_dir` and `binary_dir` are different directories. `bin_dir` defines the directory for binary files +# inside the container, while `binary_dir` defines the directory for binary files outside the container. + extract_source_tarball() { if [ ! -d "${source_dir}" ]; then mkdir -p $(dirname ${source_dir}) @@ -18,59 +21,69 @@ gobuild_component() { local extract_path=${3:-"github.com/percona/$parent_component"} local component_path=$4 local result_file=${5:-bin/$(basename ${component_path:-$extract_path})} + local src_dir=/home/builder/tmp/source/pmm-client-${pmm_version} + local bin_dir=/home/builder/tmp/binary/pmm-client-${pmm_version} + local archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz" | head -n1) + local build_command + local volume_mounts="-v ${tmp_dir}:/home/builder/tmp" + volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build" + volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod" + volume_mounts+=" -v pmm-dnf:/var/cache/dnf" if [ ${component} = 'azure_metrics_exporter' ]; then result_file=${result_file/_metrics/} fi if [ -x "${binary_dir}/${result_file}" ]; then - echo skip build + echo Skipping the build... return fi - if [ ${component} = 'pmm-admin' -o ${component} = 'pmm-agent' ]; then - build_command="cd /tmp/go/src/${extract_path}/${component_path} && make release" + if [[ ${component} =~ pmm-admin|pmm-agent ]]; then + build_command="cd /tmp/go/src/${extract_path}/${component_path} && NO_VCS=1 make release" elif [ ${component} = 'mongodb_exporter' ]; then build_command="cd /tmp/go/src/${extract_path} && make build" - elif [ ${component} = 'node_exporter' ] || [ ${component} = 'mysqld_exporter' ] || \ - [ ${component} = 'proxysql_exporter' ] || [ ${component} = 'rds_exporter' ] || \ - [ ${component} = 'postgres_exporter' ] || [ ${component} = 'azure_metrics_exporter' ]; then + elif [[ ${component} =~ node_exporter|mysqld_exporter|proxysql_exporter|rds_exporter|postgres_exporter|azure_metrics_exporter ]]; then build_command="cd /tmp/go/src/${extract_path} && make release" elif [ ${component} = 'percona-toolkit' ]; then - build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o \${binary_dir}/${result_file} ." + build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ." else - build_command="go build -o \${binary_dir}/${result_file} ./src/${extract_path}/${component_path}" + build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ." fi - docker run --rm -v ${tmp_dir}:/home/builder/tmp ${rpmbuild_docker_image} sh -c " + + docker run --rm \ + --platform=${platform} \ + ${volume_mounts} \ + ${rpmbuild_docker_image} sh -c " set -o errexit set -o xtrace - export GOOS=${GOOS:-linux} - export GOARCH=${GOARCH:-amd64} export CGO_ENABLED=0 export GO111MODULE=auto - # for 'make release' + # Setting variables for 'make release' export PMM_RELEASE_VERSION=${full_pmm_version} export PMM_RELEASE_TIMESTAMP=$(date '+%s') - export PMM_RELEASE_FULLCOMMIT=$(cd sources/${parent_component}/src/${extract_path} && git rev-parse HEAD) - export COMPONENT_VERSION=$(cd sources/${parent_component}/src/${extract_path} && git describe --abbrev=0 --always) - export COMPONENT_BRANCH=$(cd sources/${parent_component}/src/${extract_path} && git describe --always --contains --all) - export PMM_RELEASE_BRANCH="" - sudo chown -R builder:builder /home/builder/tmp + export PMM_RELEASE_PATH=${bin_dir}/bin - source_dir=/home/builder/tmp/source/pmm-client-${pmm_version} - binary_dir=/home/builder/tmp/binary/pmm-client-${pmm_version} - export PMM_RELEASE_PATH=\${binary_dir}/bin + sudo chown -R builder:builder /home/builder/tmp + if [ ! -w /home/builder/.cache/go-build ]; then + sudo chown builder /home/builder/.cache/go-build + fi + if [ ! -w /home/builder/go/pkg/mod ]; then + sudo chown builder /home/builder/go/pkg/mod + fi + + if [ "${component}" = vmagent ]; then + export PMM_RELEASE_FULLCOMMIT=${vmagent_commit_hash} + else + export PMM_RELEASE_FULLCOMMIT=$(git -C sources/${parent_component}/src/${extract_path} rev-parse HEAD) + fi + + mkdir -p ${bin_dir}/bin + mkdir -p /tmp/go/src/${extract_path} - mkdir -p \$binary_dir/bin + tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf ${src_dir}/${archive} - mkdir -p /tmp/go/src/${extract_path} - archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz") - tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf \${source_dir}/\$archive - pushd /tmp/go - export GOPATH=\$(pwd -P) - ${build_command} - popd - sudo chown -R $(id -u):$(id -g) /home/builder/tmp + "${build_command}" " } @@ -83,8 +96,7 @@ copy_component() { if [ ! -d "${component_dir}" ]; then mkdir -p ${component_dir} - tar \ - -C ${component_dir} \ + tar -C ${component_dir} \ --strip-components=1 \ -zxpf "$archive" fi @@ -93,6 +105,10 @@ copy_component() { } main() { + echo ----------------------------------------- + echo "Building PMM Client binary files..." + echo ----------------------------------------- + extract_source_tarball gobuild_component "vmagent" "" "github.com/VictoriaMetrics/VictoriaMetrics" "app/vmagent" @@ -103,7 +119,6 @@ main() { gobuild_component "proxysql_exporter" gobuild_component "rds_exporter" gobuild_component "azure_metrics_exporter" - gobuild_component "pmm-admin" "pmm" "" "admin" gobuild_component "pmm-agent" "pmm" "" "agent" @@ -120,16 +135,16 @@ main() { copy_component "postgres_exporter" "queries-postgres-uptime.yml" copy_component "postgres_exporter" "queries-mr.yaml" copy_component "postgres_exporter" "queries-lr.yaml" - copy_component "percona-toolkit" "bin/pt-summary" copy_component "percona-toolkit" "bin/pt-mysql-summary" + gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-mongodb-summary" gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-pg-summary" rm -rf ${binary_tarball} mkdir -p $(dirname ${binary_tarball}) || : - # Removes pmm binary in case this is not a PR build - such as RC/GA build + # Removes pmm binary (aka CLI) in case this is not a PR build - such as RC/GA build if [[ "$(basename ${root_dir})" != pmm-submodules_PR-* ]]; then rm -f "${binary_dir}/bin/pmm" fi @@ -138,6 +153,5 @@ main() { } main -exit 0 # vim: expandtab shiftwidth=4 tabstop=4 diff --git a/build/scripts/build-client-deb b/build/scripts/build-client-deb index f1593517b1..72deb8a62e 100755 --- a/build/scripts/build-client-deb +++ b/build/scripts/build-client-deb @@ -6,7 +6,6 @@ set -o errexit set -o xtrace bin_dir=$(cd $(dirname $0); pwd -P) -tmp_dir=${root_dir}/tmp main() { local IMAGE=${1:-ubuntu:focal} @@ -35,6 +34,5 @@ main() { } main $* -exit 0 # vim: expandtab shiftwidth=4 tabstop=4 diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker index ea6287703f..a564c5c4cf 100755 --- a/build/scripts/build-client-docker +++ b/build/scripts/build-client-docker @@ -5,37 +5,35 @@ set -o xtrace . $(dirname $0)/vars -if [ -f "${docker_client_tarball}" ]; then - echo skip docker build - exit 0 -fi - -DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir` -cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz - -if [ -z "${DOCKER_CLIENT_TAG}" ]; then - DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version} -fi - - -CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2` - -docker build --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \ - --build-arg VERSION="$CLIENT_IMAGE_VERSION" \ - --squash \ - --no-cache \ - -f ${DOCKER_FILE_LOCATION}/${docker_file} \ - -t ${DOCKER_CLIENT_TAG} \ - ${DOCKER_FILE_LOCATION} - -if [ -n "${PUSH_DOCKER}" ]; then - mkdir -p $(dirname ${docker_client_tag_file}) - echo ${DOCKER_CLIENT_TAG} > ${docker_client_tag_file} - docker push ${DOCKER_CLIENT_TAG} -fi -if [ -n "${SAVE_DOCKER}" ]; then - mkdir -p $(dirname ${docker_client_tarball}) - docker save ${DOCKER_CLIENT_TAG} | xz > ${docker_client_tarball} -fi +main() { + echo ----------------------------------------- + echo "Building docker image for PMM Client..." + echo ----------------------------------------- + + DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir` + cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz + + if [ -z "${DOCKER_CLIENT_TAG}" ]; then + DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version} + fi + + CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2` + + docker buildx build \ + --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \ + --build-arg VERSION="$CLIENT_IMAGE_VERSION" \ + --progress plain \ + -f ${DOCKER_FILE_LOCATION}/${docker_file} \ + -t ${DOCKER_CLIENT_TAG} \ + ${DOCKER_FILE_LOCATION} + + if [ -n "${PUSH_DOCKER}" ]; then + mkdir -p $(dirname ${docker_client_tag_file}) + echo ${DOCKER_CLIENT_TAG} > ${docker_client_tag_file} + docker push ${DOCKER_CLIENT_TAG} + fi +} + +main # vim: expandtab shiftwidth=4 tabstop=4 diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages index 9d19f7b2d1..ed4836ec63 100755 --- a/build/scripts/build-client-packages +++ b/build/scripts/build-client-packages @@ -17,7 +17,7 @@ Usage: $0 [OPTIONS] --build_source_deb If it is 1 source deb package will be built --build_rpm If it is 1 rpm will be built --build_deb If it is 1 deb will be built - --install_deps Install build dependencies(root previlages are required) + --install_deps Install build dependencies(root privileges are required) --branch Branch from which submodules should be taken(default v3) --help) usage ;; Example $0 --builddir=/tmp/PMM_CLIENT --get_sources=1 --build_src_rpm=1 --build_rpm=1 @@ -30,17 +30,10 @@ append_arg_to_args () { } parse_arguments() { - pick_args= - if test "$1" = PICK-ARGS-FROM-ARGV; then - pick_args=1 - shift - fi - for arg do val=`echo "$arg" | sed -e 's;^--[^=]*=;;'` optname=`echo "$arg" | sed -e 's/^\(--[^=]*\)=.*$/\1/'` case "$arg" in - # these get passed explicitly to mysqld --builddir=*) WORKDIR="$val" ;; --build_src_rpm=*) SRPM="$val" ;; --build_source_deb=*) SDEB="$val" ;; @@ -50,11 +43,7 @@ parse_arguments() { --branch=*) SUBMODULE_BRANCH="$val" ;; --install_deps=*) INSTALL="$val" ;; --help) usage ;; - *) - if test -n "$pick_args"; then - append_arg_to_args "$arg" - fi - ;; + *) append_arg_to_args "$arg" ;; esac done } @@ -67,7 +56,7 @@ get_branches() { cd ${WORKDIR}/pmm-submodules git reset --hard > /dev/null 2>&1 git clean -xdf > /dev/null 2>&1 - git checkout $SUBMODULE_BRANCH > /dev/null 2>&1 + git checkout $SUBMODULE_BRANCH # this is the return value of this function git submodule status | grep $COMPONENT | awk '{print $1}' | awk -F'-' '{print $2}' cd - >/dev/null @@ -87,16 +76,15 @@ get_repos() { } check_workdir(){ - if [ "x$WORKDIR" = "x$CURDIR" ]; then + if [ "$WORKDIR" = "$CURDIR" ]; then echo >&2 "Current directory cannot be used for building!" exit 1 - else - if ! test -d "$WORKDIR"; then - echo >&2 "$WORKDIR is not a directory." - exit 1 - fi fi - return + + if ! test -d "$WORKDIR"; then + echo >&2 "$WORKDIR is not a directory." + exit 1 + fi } get_sources(){ @@ -108,11 +96,11 @@ get_sources(){ git clone $REPO retval=$? if [ $retval != 0 ]; then - echo "There were some issues during repo cloning from github. Please retry one more time" + echo "There were some issues during repo cloning from github. Please retry." exit 1 fi cd pmm-client - if [ ! -z $BRANCH ]; then + if [ -n $BRANCH ]; then git reset --hard git clean -xdf git checkout $BRANCH @@ -122,7 +110,6 @@ get_sources(){ # VERSION=$(pmm_version) mv Makefile build/ - #cat VERSION > $VERSION_FILE echo "VERSION=${VERSION}" > $VERSION_FILE echo "REVISION=${REVISION}" >> $VERSION_FILE echo "RPM_RELEASE=${RPM_RELEASE}" >> $VERSION_FILE @@ -133,7 +120,6 @@ get_sources(){ echo "MongoExp_BRANCH_NAME=${MongoExp_BRANCH_NAME}" >> $VERSION_FILE echo "MysqlExp_BRANCH_NAME=${MysqlExp_BRANCH_NAME}" >> $VERSION_FILE echo "ProxysqlExp_BRANCH_NAME=${ProxysqlExp_BRANCH_NAME}" >> $VERSION_FILE - echo "QAN_BRANCH_NAME=${QAN_BRANCH_NAME}" >> $VERSION_FILE echo "TOOLKIT_REPO=${TOOLKIT_REPO}" >> $VERSION_FILE echo "TOOLKIT_BRANCH_NAME=${TOOLKIT_BRANCH_NAME}" >> $VERSION_FILE PRODUCT=pmm-client @@ -146,10 +132,8 @@ get_sources(){ echo "BUILD_ID=${BUILD_ID}" >> $VERSION_FILE echo "UPLOAD=UPLOAD/experimental/BUILDS/${PRODUCT_NAME}/${VERSION}/${BRANCH_NAME}/${REVISION}/${BUILD_ID}" >> $VERSION_FILE echo "MongoExp_REPO=${MongoExp_REPO}" >> $VERSION_FILE - echo "TOOLKIT_REPO=${TOOLKIT_REPO}" >> $VERSION_FILE echo "MysqlExp_REPO=${MysqlExp_REPO}" >> $VERSION_FILE echo "ProxysqlExp_REPO=${ProxysqlExp_REPO}" >> $VERSION_FILE - echo "QAN_REPO=${QAN_REPO}" >> $VERSION_FILE echo "NodeExp_REPO=${NodeExp_REPO}" >> $VERSION_FILE cd ../ mv ${PRODUCT} ${PRODUCT}-${VERSION} @@ -179,16 +163,27 @@ get_system(){ } install_deps() { + local PACKAGES if [ $INSTALL = 0 ]; then - echo "Dependencies will not be installed" + echo "Dependencies will not be installed." return; fi if [ ! $( id -u ) -eq 0 ]; then - echo "It is not possible to instal dependencies. Please run as root" + echo "It is not possible to install dependencies. Please run as root." exit 1 fi if [ "$OS" = "rpm" ]; then - yum -y install git wget rpmdevtools bison yum-utils rpm-build + # yum -y install git curl wget rpmdevtools bison yum-utils rpm-build + command -v git || PACKAGES+=" git" + command -v curl || PACKAGES+=" curl" + command -v wget || PACKAGES+=" wget" + command -v spectool || PACKAGES+=" rpmdevtools" + command -v bison || PACKAGES+=" bison" + command -v repoquery || PACKAGES+=" yum-utils" + command -v rpmbuild || PACKAGES+=" rpm-build" + if [ -n "$PACKAGES" ]; then + yum -y install "$PACKAGES" + fi else apt-get update apt-get -y install devscripts equivs debhelper @@ -198,9 +193,9 @@ install_deps() { get_tar(){ TARBALL=$1 - TARFILE=$(basename $(find $WORKDIR/$TARBALL -name 'pmm-client*.tar.gz' | sort | tail -n1) || :) + TARFILE=$(basename $(find $WORKDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :) if [ -z $TARFILE ]; then - TARFILE=$(basename $(find $CURDIR/$TARBALL -name 'pmm-client*.tar.gz' | sort | tail -n1) || :) + TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :) if [ -z $TARFILE ]; then echo "There is no $TARBALL for build" exit 1 @@ -216,9 +211,9 @@ get_tar(){ get_deb_sources(){ param=$1 echo $param - FILE=$(basename $(find $WORKDIR/source_deb -name "pmm-client*.$param" | sort | tail -n1) || :) + FILE=$(basename $(find $WORKDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :) if [ -z $FILE ]; then - FILE=$(basename $(find $CURDIR/source_deb -name "pmm-client*.$param" | sort | tail -n1) || :) + FILE=$(basename $(find $CURDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :) if [ -z $FILE ]; then echo "There is no sources for build" exit 1 @@ -232,7 +227,7 @@ get_deb_sources(){ } build_srpm(){ - if [ $SRPM = 0 ]; then + if [ "$SRPM" = 0 ]; then echo "SRC RPM will not be created" return; fi @@ -242,28 +237,30 @@ build_srpm(){ fi cd $WORKDIR get_tar "tarball" - - # + rm -fr rpmbuild ls | grep -v tar.gz | xargs rm -rf - - # - TARFILE=$(basename $(find . -name 'pmm-client-*.tar.gz' | sort | tail -n1)) + + TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1)) NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}') VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}') VERSION=${VERSION_TMP%.tar.gz} - - # + mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS} - - # + tar -C ${WORKDIR} -zxpf ${TARFILE} ${NAME}-${VERSION}/rpm - # + cd ${WORKDIR}/rpmbuild/SPECS cp ${WORKDIR}/${NAME}-${VERSION}/rpm/*.spec . cp ${WORKDIR}/${TARFILE} ../SOURCES/ cd ${WORKDIR} - rpmbuild -bs --define "_topdir ${WORKDIR}/rpmbuild" --define "version $VERSION" --define "release $RPM_RELEASE" --define "dist %{nil}" rpmbuild/SPECS/pmm-client.spec + rpmbuild -bs \ + --define "_topdir ${WORKDIR}/rpmbuild" \ + --define "version $VERSION" \ + --define "release $RPM_RELEASE" \ + --define "dist %{nil}" \ + "rpmbuild/SPECS/${PACKAGE_NAME}.spec" + mkdir -p ${WORKDIR}/srpm mkdir -p ${CURDIR}/srpm cp rpmbuild/SRPMS/*.src.rpm ${CURDIR}/srpm @@ -271,7 +268,7 @@ build_srpm(){ } build_rpm(){ - if [ $RPM = 0 ]; then + if [ "$RPM" = 0 ]; then echo "RPM will not be created" return; fi @@ -280,9 +277,9 @@ build_rpm(){ exit 1 fi - SRC_RPM=$(basename $(find $WORKDIR/srpm -name 'pmm-client*.src.rpm' | sort | tail -n1) || :) + SRC_RPM=$(basename $(find $WORKDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :) if [ -z $SRC_RPM ]; then - SRC_RPM=$(basename $(find $CURDIR/srpm -name 'pmm-client*.src.rpm' | sort | tail -n1) || :) + SRC_RPM=$(basename $(find $CURDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :) if [ -z $SRC_RPM ]; then echo "There is no src rpm for build" echo "You can create it using key --build_src_rpm=1" @@ -298,6 +295,10 @@ build_rpm(){ rm -fr rpmbuild mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS} cp $SRC_RPM rpmbuild/SRPMS/ + if find "${CURDIR}/rpm" -name "${PACKAGE_NAME}-${pmm_version}-*.rpm"; then + # Remove older client rpm packages before building a new one + rm -f "${CURDIR}/rpm/${PACKAGE_NAME}-${pmm_version}-*.rpm" + fi rpmbuild --define "_topdir ${WORKDIR}/rpmbuild" \ --define "version $pmm_version" \ --define "release $RPM_RELEASE" \ @@ -305,17 +306,17 @@ build_rpm(){ --rebuild rpmbuild/SRPMS/$SRC_RPM return_code=$? - if [ $return_code != 0 ]; then + if [ "$return_code" -ne 0 ]; then + echo "Error: rpmbuild exited with a non-zero return code '$return_code'\n" exit $return_code fi - mkdir -p ${WORKDIR}/rpm + mkdir -p ${CURDIR}/rpm - cp rpmbuild/RPMS/*/*.rpm ${WORKDIR}/rpm cp rpmbuild/RPMS/*/*.rpm ${CURDIR}/rpm } build_source_deb(){ - if [ $SDEB = 0 ]; then + if [ "$SDEB" = 0 ]; then echo "source deb package will not be created" return; fi @@ -326,17 +327,16 @@ build_source_deb(){ rm -rf pmm-client* get_tar "tarball" rm -f *.dsc *.orig.tar.gz *.debian.tar.gz *.changes - # - TARFILE=$(basename $(find . -name 'pmm-client-*.tar.gz' | sort | tail -n1)) + + TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1)) NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}') VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}') VERSION=${VERSION_TMP%.tar.gz} rm -fr ${NAME}-${VERSION} - # + NEWTAR=${NAME}_${VERSION}.orig.tar.gz mv ${TARFILE} ${NEWTAR} - # git clone $REPO ${NAME}-${VERSION}_all pushd ${NAME}-${VERSION}_all @@ -363,7 +363,7 @@ build_source_deb(){ mv queries-mr.yaml ../${NAME}-${VERSION}_all/distro/ mv queries-lr.yaml ../${NAME}-${VERSION}_all/distro/ mv debian ../${NAME}-${VERSION}_all/ - mv config/pmm-agent.service ../${NAME}-${VERSION}_all/debian/pmm-client.pmm-agent.service + mv config/pmm-agent.service ../${NAME}-${VERSION}_all/debian/${PACKAGE_NAME}.pmm-agent.service cd ../ rm -rf ${NAME}-${VERSION} @@ -396,12 +396,12 @@ build_source_deb(){ } build_deb(){ - if [ $DEB = 0 ]; then - echo "source deb package will not be created" + if [ "$DEB" = 0 ]; then + echo "deb package will not be created" return; fi if [ "$OS" = "rpm" ]; then - echo "It is not possible to build source deb here" + echo "It is not possible to build deb here" exit 1 fi for file in 'dsc' 'orig.tar.gz' 'changes' 'diff.gz'; do @@ -432,6 +432,7 @@ build_deb(){ #main CURDIR=$(pwd) +PACKAGE_NAME=pmm-client VERSION_FILE=$CURDIR/pmm-client.properties args= WORKDIR= @@ -446,16 +447,16 @@ ARCH= OS= SUBMODULE_BRANCH="v3" INSTALL=0 -RPM_RELEASE=6 -DEB_RELEASE=6 +RPM_RELEASE=1 +DEB_RELEASE=1 +REVISION=0 if [ -n "$pmm_release" ]; then RPM_RELEASE="$RPM_RELEASE.$pmm_release" DEB_RELEASE="$DEB_RELEASE.$pmm_release" fi -REVISION=0 -parse_arguments PICK-ARGS-FROM-ARGV "$@" +parse_arguments "$@" get_system install_deps @@ -480,8 +481,6 @@ if [ "$TARBALL" = '1' -o "$SOURCE" = '1' ]; then fi check_workdir -get_system -install_deps get_sources build_srpm build_source_deb diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm index 938ba39ccb..374b79c5ac 100755 --- a/build/scripts/build-client-rpm +++ b/build/scripts/build-client-rpm @@ -5,40 +5,29 @@ set -o errexit set -o xtrace -bin_dir=$(cd $(dirname $0); pwd -P) -tmp_dir=${root_dir}/tmp - main() { + echo ----------------------------------------- + echo "Building PMM Client RPM files..." + echo ----------------------------------------- + local IMAGE=${1:-${rpmbuild_docker_image}} - docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c " + docker run --rm \ + -v ${bin_dir}:/home/builder/bin \ + -v ${root_dir}/results:/home/builder/results \ + ${IMAGE} sh -c " set -o errexit set -o xtrace - if grep -q 'CentOS Linux 7' /etc/os-release; then - # disable fastestmirror plugin, which mostly fails due to CentOS 7 being EOL - sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf - - sed -i -e 's/^\(mirrorlist\)/#\1/g' /etc/yum.repos.d/CentOS-Base.repo - sed -i -e 's|^#baseurl.*|baseurl=http://vault.centos.org/centos/\$releasever/os/\$basearch/|g' /etc/yum.repos.d/CentOS-Base.repo - fi - export pmm_version=$pmm_version export pmm_release=$pmm_release - export sudo_path=\$(ls /usr/bin/sudo) mkdir -p /tmp/pmm - pushd /home/builder/results - \$sudo_path /home/builder/bin/build-client-packages \ - --builddir=/tmp/pmm \ - --install_deps=1 - /home/builder/bin/build-client-packages \ - --builddir=/tmp/pmm \ - --build_rpm=1 - popd - \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/rpm - " + cd /home/builder/results + /home/builder/bin/build-client-packages \ + --builddir=/tmp/pmm \ + --build_rpm=1 + " } main $* -exit 0 # vim: expandtab shiftwidth=4 tabstop=4 diff --git a/build/scripts/build-client-sdeb b/build/scripts/build-client-sdeb index f52cff6577..2429d5de8d 100755 --- a/build/scripts/build-client-sdeb +++ b/build/scripts/build-client-sdeb @@ -6,7 +6,6 @@ set -o errexit set -o xtrace bin_dir=$(cd $(dirname $0); pwd -P) -tmp_dir=${root_dir}/tmp main() { local IMAGE=${1:-ubuntu:focal} @@ -32,6 +31,5 @@ main() { } main $* -exit 0 # vim: expandtab shiftwidth=4 tabstop=4 diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source index df923cbee8..5f82ccc77b 100755 --- a/build/scripts/build-client-source +++ b/build/scripts/build-client-source @@ -7,11 +7,12 @@ set -o xtrace prepare_tarball() { local repo_name=$1 + local git_dir if [ $repo_name = 'pmm-submodules' ]; then - mkdir -p ${root_dir}/sources/${repo_name} git_dir="${root_dir}/sources/${repo_name}" + mkdir -p "${git_dir}" else - local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1)) + git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1)) fi local commit=$(git -C "${git_dir}" rev-parse --short HEAD) local tarball=${source_dir}/${repo_name}-${commit}.tar.gz @@ -24,43 +25,47 @@ prepare_tarball() { if [ -f "${tarball}" ]; then echo ${tarball} already exists, skipping build - else - if [ ${repo_name} = 'pmm-submodules' ]; then - tar -C sources -zcf ${repo_name}-${commit}.tar.gz ${repo_name} - mv ${repo_name}-${commit}.tar.gz ${source_dir}/ - else - git -C "${git_dir}" archive \ - --format=tar.gz \ - --prefix=${repo_name}-${commit}/ \ - -o ${tarball} \ - "${commit}" - fi + return + fi + + if [ ${repo_name} = 'pmm-submodules' ]; then + tar -C sources -zcf ${repo_name}-${commit}.tar.gz ${repo_name} + mv ${repo_name}-${commit}.tar.gz ${source_dir}/ + return fi + + git -C "${git_dir}" archive \ + --format=tar.gz \ + --prefix=${repo_name}-${commit}/ \ + -o ${tarball} \ + "${commit}" } prepare_vmagent_tarball() { local repo_name=vmagent local commit_hash=${vmagent_commit_hash} - local commit_short=${commit_hash::6} + local commit_short=${commit_hash::7} local tarball=${source_dir}/${repo_name}-${commit_short}.tar.gz local repo_var=$( - echo "${repo_name}_commit" \ - | sed -e 's:-:_:g' \ - | tr '[[:lower:]]' '[[:upper:]]' + echo "${repo_name}_commit" | sed -e 's:-:_:g' | tr '[[:lower:]]' '[[:upper:]]' ) echo "${repo_var}=$commit_short" >> ${client_properties} if [ -f "${tarball}" ]; then echo ${tarball} already exists, skipping download else - wget https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz -O ${tarball} + curl -o ${tarball} -fsSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz fi } main() { + echo ----------------------------------------- + echo "Building PMM Client source files..." + echo ----------------------------------------- + rm -rf ${source_tarball} mkdir -p ${source_dir}/ $(dirname ${source_tarball}) || : - echo "PMM_SUBMODULES_COMMIT=$(git rev-parse --short HEAD)" > ${client_properties} + prepare_vmagent_tarball prepare_tarball pmm-submodules prepare_tarball pmm @@ -78,6 +83,5 @@ main() { } main -exit 0 # vim: expandtab shiftwidth=4 tabstop=4 diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm index 2664613c66..c68b949a33 100755 --- a/build/scripts/build-client-srpm +++ b/build/scripts/build-client-srpm @@ -5,40 +5,26 @@ set -o errexit set -o xtrace -bin_dir=$(cd $(dirname $0); pwd -P) -tmp_dir=${root_dir}/tmp - main() { + echo ----------------------------------------- + echo "Building PMM Client source RPM files..." + echo ----------------------------------------- + local IMAGE=${1:-${rpmbuild_docker_image}} docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c " set -o errexit set -o xtrace - if grep -q 'CentOS Linux 7' /etc/os-release; then - # disable fastestmirror plugin, which mostly fails due to CentOS 7 being EOL - sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf - - sed -i -e 's/^\(mirrorlist\)/#\1/g' /etc/yum.repos.d/CentOS-Base.repo - sed -i -e 's|^#baseurl.*|baseurl=http://vault.centos.org/centos/\$releasever/os/\$basearch/|g' /etc/yum.repos.d/CentOS-Base.repo - fi - export pmm_release=$pmm_release - export sudo_path= - if [ -e /usr/bin/sudo ]; then - export sudo_path=\$(ls /usr/bin/sudo) - fi - [[ ${IMAGE} = ${rpmbuild_docker_image} ]] || \$sudo_path yum -y install git rpm-build mkdir -p /tmp/pmm - pushd /home/builder/results - /home/builder/bin/build-client-packages \ - --builddir=/tmp/pmm \ - --build_src_rpm=1 - popd - \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/srpm + cd /home/builder/results + + /home/builder/bin/build-client-packages \ + --builddir=/tmp/pmm \ + --build_src_rpm=1 " } main $* -exit 0 # vim: expandtab shiftwidth=4 tabstop=4 diff --git a/build/scripts/build-rpmbuild-docker b/build/scripts/build-rpmbuild-docker deleted file mode 100755 index 77706f3621..0000000000 --- a/build/scripts/build-rpmbuild-docker +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o xtrace - -. $(dirname $0)/vars - -root_dir=$(cd $(dirname $0)/../..; pwd -P) -docker build --squash -f ${docker_file} -t rpmbuild:local ${root_dir}/build/docker/rpmbuild/ - -# vim: expandtab shiftwidth=4 tabstop=4 diff --git a/build/scripts/build-server b/build/scripts/build-server index 7a2b053be9..89e2262622 100755 --- a/build/scripts/build-server +++ b/build/scripts/build-server @@ -10,7 +10,7 @@ set -o xtrace pmm_version=${pmm_version} ${bin_dir}/build-client pmm_version=${pmm_version} ${bin_dir}/build-client-srpm pmm_version=${pmm_version} ${bin_dir}/build-client-rpm -mkdir -p ${rpms_dir}/ +mkdir -p ${rpms_dir} ${rpmbuild_dir} cp ${root_dir}/results/rpm/*.rpm ${rpms_dir}/ # install all rpms diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker index 421a4dd688..f1df62ac72 100755 --- a/build/scripts/build-server-docker +++ b/build/scripts/build-server-docker @@ -5,51 +5,51 @@ set -o xtrace . $(dirname $0)/vars -if [ -f "${docker_tarball}" ]; then - echo skip docker build - exit 0 -fi - -docker_root=$(realpath ${rpms_dir}/..) -cp -r ${root_dir}/tmp/source/pmm/build/ansible ${docker_root}/ansible -ls ${docker_root}/ansible -ls ${docker_root}/ansible/roles/pmm-images -ls ${docker_root}/ansible/roles/pmm-images/tasks -cp ${root_dir}/tmp/source/pmm/build/docker/server/* ${docker_root}/ -cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz - -git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit -ls -la ${rpms_dir} -docker run --rm -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c " - sudo chown -R builder /home/builder/rpm/RPMS - until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do - echo "waiting" - sleep 1 - done -" - -if [ -z "${DOCKER_TAG}" ]; then - DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version} -fi - -IMAGE_VERSION=`echo $DOCKER_TAG | cut -d ':' -f2` - -docker build --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \ - --build-arg VERSION="$IMAGE_VERSION" \ - --squash \ - --no-cache \ - -f ${docker_root}/${docker_file} \ - -t ${DOCKER_TAG} \ - ${docker_root}/ - -if [ -n "${PUSH_DOCKER}" ]; then - mkdir -p $(dirname ${docker_tag_file}) - echo ${DOCKER_TAG} > ${docker_tag_file} - docker push ${DOCKER_TAG} -fi -if [ -n "${SAVE_DOCKER}" ]; then - mkdir -p $(dirname ${docker_tarball}) - docker save ${DOCKER_TAG} > ${docker_tarball} -fi +main() { + echo ----------------------------------------- + echo "Building docker image for PMM Server..." + echo ----------------------------------------- + + local docker_root=$(realpath ${rpms_dir}/..) + local pmm_source_dir=$(git config -f .gitmodules submodule.pmm.path) + + cp -r "${root_dir}/${pmm_source_dir}/build/ansible" "${docker_root}/ansible" + cp "${root_dir}/${pmm_source_dir}/build/docker/server/"* "${docker_root}" + cp ${root_dir}/results/tarball/pmm-client-*.tar.gz "${docker_root}/pmm-client.tar.gz" + + git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit + ls -la ${rpms_dir} + docker run --rm --platform=${platform} -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c " + sudo chown -R builder /home/builder/rpm/RPMS + until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do + echo "waiting" + sleep 1 + done + " + + if [ -z "${DOCKER_TAG}" ]; then + DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version} + fi + + local IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2) + local BUILD_DATE=$(date -u +'%F %T%z' | sed 's@^.\{22\}@&:@') + + docker buildx build \ + --build-arg BUILD_DATE="$BUILD_DATE" \ + --build-arg VERSION="$IMAGE_VERSION" \ + --platform=${platform} \ + --progress plain \ + -f ${docker_root}/${docker_file} \ + -t ${DOCKER_TAG} \ + ${docker_root} + + if [ -n "${PUSH_DOCKER}" ]; then + mkdir -p $(dirname ${docker_tag_file}) + echo ${DOCKER_TAG} > ${docker_tag_file} + docker push ${DOCKER_TAG} + fi +} + +main # vim: expandtab shiftwidth=4 tabstop=4 diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm index 8fdaa28d32..2cb55dde8c 100755 --- a/build/scripts/build-server-rpm +++ b/build/scripts/build-server-rpm @@ -9,10 +9,11 @@ get_rpm_version() { local spec_name=$1 local rpm_version=$( - docker run --rm -v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c " + docker run --rm \ + --platform=${platform} \ + -v ${rpmbuild_dir}:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c " rpmspec -q --qf '%{version}-%{release}\n' SOURCES/${spec_name}.spec \ - | sed -re 's/\.[0-9]{10}\././; s/.${rpmbuild_dist}//' \ - | head -1 + | sed -re 's/\.[0-9]{10}\././' | head -1 " ) @@ -24,148 +25,186 @@ is_build_needed() { local spec_name=$1 local rpm_version=$2 local packages= + local s3_cache_dir=RELEASE # Structure of S3 build cache - # s3://pmm-build-cache/PR-BUILDS/7 - el7 # s3://pmm-build-cache/PR-BUILDS/9 - el9 - # s3://pmm-build-cache/RELEASE/7 - el7 # s3://pmm-build-cache/RELEASE/9 - el9 - # RPM_EPOCH is set for Feature Builds - if [ -n "$RPM_EPOCH" ]; then - aws s3 sync \ - --region us-east-2 \ - --no-sign-request \ - s3://pmm-build-cache/PR-BUILDS/${rpmbuild_dist}/${spec_name}-${rpm_version} \ - ${rpms_dir}/${spec_name}-${rpm_version} - else + # RPM_EPOCH is set to 1 for Feature Builds + if [ "$RPM_EPOCH" = 1 ]; then + s3_cache_dir=PR-BUILDS + fi + if command -v aws &> /dev/null && [ -z "$LOCAL_BUILD" ]; then aws s3 sync \ --region us-east-2 \ --no-sign-request \ - s3://pmm-build-cache/RELEASE/${rpmbuild_dist}/${spec_name}-${rpm_version} \ - ${rpms_dir}/${spec_name}-${rpm_version} + s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \ + ${rpms_dir}/${spec_name}-${rpm_version} || : fi packages=$(find ${rpms_dir}/${spec_name}-${rpm_version} -name "*.${rpmbuild_dist}.*.rpm" | wc -l) - # return result as true or false + # return result as true (0) or false (>0) [[ ${packages// /} == 0 ]] } prepare_specs() { local spec_name=$1 local repo_name=$2 + local spec_file=${rpmbuild_dir}/${spec_name}.spec - sudo chown -R $(id -u):$(id -g) ${rpmbuild_dir}/SPECS ${rpmbuild_dir}/SOURCES - cp ${rpmbuild_dir}/SPECS/${spec_name}.spec ${rpmbuild_dir}/SOURCES/${spec_name}.spec - if [ -d "${root_dir}/sources/${repo_name}" ]; then - local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1)) - local full_commit=$(git -C "${git_dir}" rev-parse HEAD) - local short_commit=${full_commit:0:7} - - sed -i -e "s/global commit.*/global commit ${full_commit}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec - if [[ ${spec_name} != "grafana" ]]; then - sed -i -e "s/Version:.*/Version: ${pmm_version}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec - fi + mkdir -p ${rpms_dir} ${rpmbuild_dir} 2>/dev/null || : + cp ${rpmspec_dir}/${spec_name}.spec ${spec_file} + if [ ! -d "${root_dir}/sources/${repo_name}" ]; then + return + fi + if [ ! -f "${spec_file}" ]; then + echo "Warning: the spec file could not be found." + fi + + local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1)) + local full_commit=$(git -C "${git_dir}" rev-parse HEAD) + local short_commit=${full_commit:0:7} + + sed -i -e "s/global commit.*/global commit ${full_commit}/" ${spec_file} + if [ "${spec_name}" != "grafana" ]; then + sed -i -e "s/Version:.*/Version: ${pmm_version}/" ${spec_file} + fi - if [ -z "${full_pmm_version}" ]; then - echo 'A full_pmm_version is not specified.' - exit 1 - fi - sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec + if [ -z "${full_pmm_version}" ]; then + echo 'The full_pmm_version is not specified.' + exit 1 + fi + sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${spec_file} - if [ -n "$pmm_release" ]; then - sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec - grep -r 'define release' ${rpmbuild_dir}/SOURCES/${spec_name}.spec - fi + if [ -n "$pmm_release" ]; then + sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${spec_file} + grep -r 'define release' ${spec_file} + fi - if [ -f "${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz" ]; then - echo SOURCES/${repo_name}-${short_commit}.tar.gz already exists, skip archiving... - else - git -C "${git_dir}" archive \ - --format=tar.gz \ - --prefix=${repo_name}-${full_commit}/ \ - -o ${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz \ - "${full_commit}" - fi + if [ -f "${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz" ]; then + echo "${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz" already exists, skip archiving... + return fi + + git -C "${git_dir}" archive \ + --format=tar.gz \ + --prefix=${repo_name}-${full_commit}/ \ + -o ${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz \ + "${full_commit}" } build() { local spec_name=$1 local repo_name=${2:-$1} prepare_specs "${spec_name}" "${repo_name}" + local rpm_version=$(get_rpm_version "${spec_name}") local CH_VERSION=${rpm_version%-*} local CH_TAG=${rpm_version#*-} + local s3_cache_dir=RELEASE - if [[ -n "${FORCE_REBUILD}" ]] || is_build_needed "${spec_name}" "${rpm_version}"; then - echo "RPM Build Dir: ${rpmbuild_dir}, Docker image: ${rpmbuild_docker_image}" + if ! is_build_needed "${spec_name}" "${rpm_version}"; then + return + fi - docker run --rm -v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c " - set -o errexit - set -o xtrace - echo \"spec_name: ${spec_name}\" - echo \"repo_name: ${repo_name}\" - echo \"rpm_verison: ${rpm_version}\" + echo ------------------------------------------------------- + echo "Building PMM Server RPM for ${spec_name} component..." + echo ------------------------------------------------------- - if grep -q 'CentOS Linux 7' /etc/os-release; then - # disable fastestmirror plugin, which mostly fails due to CentOS 7 being EOL - sudo sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf + local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS" + volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn" + if [ ${spec_name} != "grafana" ]; then + volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod" + volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build" + volume_mounts+=" -v pmm-dnf:/var/cache/dnf" + fi - sudo sed -i -e 's/^\(mirrorlist\)/#\1/g' /etc/yum.repos.d/CentOS-Base.repo - sudo sed -i -e 's|^#baseurl.*|baseurl=http://vault.centos.org/centos/\$releasever/os/\$basearch/|g' /etc/yum.repos.d/CentOS-Base.repo - fi + echo "Start building Server RPMs..." + echo "spec_name: ${spec_name}" + echo "repo_name: ${repo_name}" + echo "rpm_verison: ${rpm_version}" - sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES + docker run --rm --platform=${platform} ${volume_mounts} ${rpmbuild_docker_image} sh -c " + set -o errexit + set -o xtrace - # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build - if [ -n \"$RPM_EPOCH\" ]; then - sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec - fi + sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES + sudo chown builder:builder /home/builder/.cache + if [ ! -d /home/builder/.cache/go-build ]; then + mkdir -p /home/builder/.cache/go-build + fi + if [ ! -d /home/builder/go ]; then + mkdir -p /home/builder/go/pkg/mod + fi + if [ ! -w /home/builder/.cache/go-build ]; then + sudo chown builder:builder /home/builder/.cache/go-build + fi + if [ ! -w /home/builder/go/pkg/mod ]; then + sudo chown builder:builder /home/builder/go/pkg/mod + fi + if [ ! -w /home/builder/.cache/yarn ]; then + sudo chown builder:builder /home/builder/.cache/yarn + fi - rm -rf /home/builder/rpm/RPMS/${spec_name}-* + # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build + if [ ${RPM_EPOCH} = 1 ]; then + sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec + fi - printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \ - | sudo tee /etc/yum.repos.d/local.repo + rm -rf /home/builder/rpm/RPMS/${spec_name}-* - until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do - echo waiting - sleep 1 - done + printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \ + | sudo tee /etc/yum.repos.d/local.repo - sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec + until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do + echo waiting + sleep 1 + done - spectool -C SOURCES -g SOURCES/${spec_name}.spec - rpmbuild --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \ - --define 'dist .${rpmbuild_dist}' \ - --define 'debug_package %{nil}' \ - -ba SOURCES/${spec_name}.spec + # Only these two specs have build dependencies + if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then + sleep 2s + sudo yum-builddep -y SOURCES/${spec_name}.spec + fi - rm -f SOURCES/${spec_name}.spec* - sudo chown -R $(id -u):$(id -g) /home/builder/rpm/RPMS /home/builder/rpm/SOURCES - " + spectool -C SOURCES -g SOURCES/${spec_name}.spec + rpmbuild --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \ + --define 'dist .${rpmbuild_dist}' \ + --define 'debug_package %{nil}' \ + -ba SOURCES/${spec_name}.spec - if [ -n "$RPM_EPOCH" ]; then - aws s3 sync \ - --region us-east-2 \ - ${rpms_dir}/${spec_name}-${rpm_version} \ - s3://pmm-build-cache/PR-BUILDS/${rpmbuild_dist}/${spec_name}-${rpm_version} \ - || : - else - aws s3 sync \ - --region us-east-2 \ - ${rpms_dir}/${spec_name}-${rpm_version} \ - s3://pmm-build-cache/RELEASE/${rpmbuild_dist}/${spec_name}-${rpm_version} \ - || : - fi + rm -f SOURCES/${spec_name}.spec* + " - cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir} + # TODO: see if this is even needed + # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir} + + if [ "$LOCAL_BUILD" = 1 ]; then + echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}" + return + fi + if ! command -v aws &> /dev/null; then + echo "AWS CLI is not installed. Upload to S3 will be skipped." + echo + echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}" + return fi + + if [ "$RPM_EPOCH" = 1 ]; then + s3_cache_dir=PR-BUILDS + fi + aws s3 sync \ + --region us-east-2 \ + ${rpms_dir}/${spec_name}-${rpm_version} \ + s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \ + || : + + echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}" } -build "$1" "$2" -echo "build-server-rpm finished, spec_name: $1, repo_name: ${2:-$1}" +build "$@" # vim: expandtab shiftwidth=4 tabstop=4 diff --git a/build/scripts/build-server-rpm-script b/build/scripts/build-server-rpm-script new file mode 100755 index 0000000000..02a899e8a8 --- /dev/null +++ b/build/scripts/build-server-rpm-script @@ -0,0 +1,40 @@ +#!/bin/bash + +set -o errexit +set -o xtrace + +. $(dirname $0)/vars + +sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES + +# Add 'Epoch' to spec file to prevent update of rpms which are built in PR build +if [ ${RPM_EPOCH} = 1 ]; then + sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec +fi + +rm -rf /home/builder/rpm/RPMS/${spec_name}-* + +printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \ + | sudo tee /etc/yum.repos.d/local.repo + +until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do + echo waiting + sleep 1 +done + +# Only these two specs have build dependencies +if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then + sleep 5s + sudo yum-builddep -y SOURCES/${spec_name}.spec +fi + +spectool -C SOURCES -g SOURCES/${spec_name}.spec +rpmbuild --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \ + --define 'dist .${rpmbuild_dist}' \ + --define 'debug_package %{nil}' \ + -ba SOURCES/${spec_name}.spec + +rm -f SOURCES/${spec_name}.spec* +sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES + +# vim: expandtab shiftwidth=4 tabstop=4 diff --git a/build/scripts/build-submodules b/build/scripts/build-submodules index aabb5e66c6..eca054bc25 100755 --- a/build/scripts/build-submodules +++ b/build/scripts/build-submodules @@ -8,30 +8,23 @@ if [ -s ci.yml ]; then python3 ci.py cat .git-sources . ./.git-sources - echo $pmm_commit > apiCommitSha - echo $pmm_branch > apiBranch - echo $pmm_url > apiURL - echo $pmm_qa_branch > pmmQABranch - echo $pmm_qa_commit > pmmQACommitSha - echo $pmm_ui_tests_branch > pmmUITestBranch - echo $pmm_ui_tests_commit > pmmUITestsCommitSha else - # Define variables using git queries, i.e. w/o python + # Define variables pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}') - echo $pmm_commit > apiCommitSha pmm_branch=$(git config -f .gitmodules submodule.pmm.branch) - echo $pmm_branch > apiBranch pmm_url=$(git config -f .gitmodules submodule.pmm.url) - echo $pmm_url > apiURL pmm_qa_branch=$(git config -f .gitmodules submodule.pmm-qa.branch) - echo $pmm_qa_branch > pmmQABranch pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}') - echo $pmm_qa_commit > pmmQACommitSha pmm_ui_tests_branch=$(git config -f .gitmodules submodule.pmm-ui-tests.branch) - echo $pmm_ui_tests_branch > pmmUITestBranch pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}') - echo $pmm_ui_tests_commit > pmmUITestsCommitSha fi fb_commit_sha=$(git rev-parse HEAD) echo $fb_commit_sha > fbCommitSha +echo $pmm_commit > apiCommitSha +echo $pmm_branch > apiBranch +echo $pmm_url > apiURL +echo $pmm_qa_branch > pmmQABranch +echo $pmm_qa_commit > pmmQACommitSha +echo $pmm_ui_tests_branch > pmmUITestBranch +echo $pmm_ui_tests_commit > pmmUITestsCommitSha diff --git a/build/scripts/vars b/build/scripts/vars index 03dee22067..4c6a25f273 100644 --- a/build/scripts/vars +++ b/build/scripts/vars @@ -2,8 +2,8 @@ bin_dir=$(cd $(dirname $0); pwd -P) # TODO: refactor to pass ${WORKSPACE} as ROOT_DIR from either Jenkins or, if relevant, from GH actions # NOTE: in most cases, this evaluates to Jenkins ${WORKSPACE} directory when called from pipelines # Example: /home/ec2-user/workspace/ol9-build-server/, see how it's used in ol9-build-server.groovy -root_dir_tmp=$(cd $(dirname $0)/../../../../../../../..; pwd -P) -root_dir=${ROOT_DIR:-$root_dir_tmp} +root_dir=$(cd $(dirname $0)/../../../../../../../..; pwd -P) +root_dir=${ROOT_DIR:-$root_dir} tmp_dir=${root_dir}/tmp # In VERSION file we can have numeric value like '2.0.0' as well as @@ -19,11 +19,7 @@ full_pmm_version=${pmm_base_version}-${pmm_branch}-$(git rev-parse --short HEAD) # Replace '/' with '-' to prevent sed from failing on dependabot-authored PRs full_pmm_version=${full_pmm_version//\//-} -# TODO Maybe it makes sense to use variable from job here -if [[ ${pmm_branch} =~ release-* || \ - ${pmm_branch} =~ pmm-3.* || \ - ${pmm_branch} == v3 || \ - ${pmm_branch} == "main" ]]; then +if [[ "${pmm_branch}" =~ ^pmm-3.|^v3$|^main$ ]]; then full_pmm_version=${pmm_base_version} fi @@ -40,11 +36,13 @@ pmm_version=${new_pmm_version} pmm_release=${new_pmm_release} unset new_pmm_version unset new_pmm_release -echo -e "\n\n\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release}\n\n\n" +echo -e "\n---\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release} <<<\n---\n" -rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:2} +platform=${PLATFORM:-linux/amd64} +rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3} rpms_dir=${root_dir}/tmp/pmm-server/RPMS -rpmbuild_dir=${root_dir}/sources/pmm/src/github.com/percona/pmm/build/packages/rpm/server +rpmbuild_dir=${root_dir}/tmp/pmm-server/BUILD +rpmspec_dir=${root_dir}/$(git -C ${root_dir} config -f .gitmodules submodule.pmm.path)/build/packages/rpm/server/SPECS rpmbuild_dist=${RPMBUILD_DIST:-"el9"} source_dir=${root_dir}/tmp/source/pmm-client-${pmm_version} binary_dir=${root_dir}/tmp/binary/pmm-client-${pmm_version} @@ -52,8 +50,6 @@ client_properties=${root_dir}/results/pmm-client.properties docker_file=${DOCKERFILE:-Dockerfile} docker_tag_file=${root_dir}/results/docker/TAG docker_client_tag_file=${root_dir}/results/docker/CLIENT_TAG -docker_tarball=${root_dir}/results/docker/pmm-server-${pmm_version}.docker -docker_client_tarball=${root_dir}/results/docker/pmm-client-${pmm_version}.docker source_tarball=${root_dir}/results/source_tarball/pmm-client-${pmm_version}.tar.gz binary_tarball=${root_dir}/results/tarball/pmm-client-${pmm_version}.tar.gz diff --git a/update/ansible/.gitkeep b/update/ansible/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000