From 1be3294b28d482a0a92cf31feebc0a64706bf35b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 12:33:20 +0000 Subject: [PATCH 01/64] chore(deps): bump docker/build-push-action from 3.3.1 to 5.1.0 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 3.3.1 to 5.1.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/1104d471370f9806843c095c1db02b5a90c5f8b6...4a13e500e55cf31b7a5d59a38ab2040ab0f42f56) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker-builder.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-builder.yml b/.github/workflows/docker-builder.yml index 7fa2db996a0..212b3c50344 100644 --- a/.github/workflows/docker-builder.yml +++ b/.github/workflows/docker-builder.yml @@ -89,7 +89,7 @@ jobs: - uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v2.10.0 - name: Build image ${{ matrix.image }}:${{ matrix.tag }} - uses: docker/build-push-action@1104d471370f9806843c095c1db02b5a90c5f8b6 # v3.3.1 + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 with: file: .github/docker/Dockerfile.${{ matrix.dockerfile }} context: . From 6f8a2e004fbac035e6051b2a90882b95ee5c805a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 12:33:33 +0000 Subject: [PATCH 02/64] chore(deps): bump docker/setup-buildx-action from 2.10.0 to 3.0.0 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2.10.0 to 3.0.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/885d1462b80bc1c1c7f0b00334ad271f09369c55...f95db51fddba0c2d1ec667646a06c2ce06100226) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker-builder.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-builder.yml b/.github/workflows/docker-builder.yml index 7fa2db996a0..50f0b7af4e2 100644 --- a/.github/workflows/docker-builder.yml +++ b/.github/workflows/docker-builder.yml @@ -86,7 +86,7 @@ jobs: username: ${{ secrets.DOCKER_REGISTRY_ID }} password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} - - uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v2.10.0 + - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 - name: Build image ${{ matrix.image }}:${{ matrix.tag }} uses: docker/build-push-action@1104d471370f9806843c095c1db02b5a90c5f8b6 # v3.3.1 From a324fd1bbab3b2cef369694e8197810c2ad05543 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Tue, 9 Jan 2024 10:39:22 +0100 Subject: [PATCH 03/64] initial commit --- ...ockerfile.centreon-collect-debian-bookworm | 67 +++++++++++++++++++ .github/workflows/docker-builder.yml | 8 +++ .github/workflows/package-collect.yml | 5 ++ 3 files changed, 80 insertions(+) create mode 100644 .github/docker/Dockerfile.centreon-collect-debian-bookworm diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bookworm b/.github/docker/Dockerfile.centreon-collect-debian-bookworm new file mode 100644 index 00000000000..cf7a544b362 --- /dev/null +++ b/.github/docker/Dockerfile.centreon-collect-debian-bookworm @@ -0,0 +1,67 @@ +ARG REGISTRY_URL + +FROM ${REGISTRY_URL}/debian:bookworm + +RUN < Date: Tue, 9 Jan 2024 14:47:47 +0100 Subject: [PATCH 04/64] pbuilder fix + conan fix --- .github/docker/Dockerfile.centreon-collect-debian-bookworm | 5 +++-- .github/workflows/docker-builder.yml | 4 ---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bookworm b/.github/docker/Dockerfile.centreon-collect-debian-bookworm index cf7a544b362..452d8872063 100644 --- a/.github/docker/Dockerfile.centreon-collect-debian-bookworm +++ b/.github/docker/Dockerfile.centreon-collect-debian-bookworm @@ -26,7 +26,6 @@ apt-get -y install cmake \ dh-make \ aptitude \ lintian \ - pbuilder \ quilt \ git-buildpackage \ debmake \ @@ -39,6 +38,8 @@ apt-get -y install cmake \ zstd \ sudo +echo http://deb.debian.org/debian | apt-get install -y pbuilder + echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list apt-get update apt-get install -y nfpm @@ -49,7 +50,7 @@ localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py python3 get-pip.py -pip3 install conan==1.61.0 +pip3 install --break-system-packages conan==1.61.0 ln -s /usr/local/bin/conan /usr/bin/conan rm -rf ~/.conan/profiles/default diff --git a/.github/workflows/docker-builder.yml b/.github/workflows/docker-builder.yml index 59b6c5b12ae..7fbba7d4ed4 100644 --- a/.github/workflows/docker-builder.yml +++ b/.github/workflows/docker-builder.yml @@ -59,10 +59,6 @@ jobs: dockerfile: centreon-collect-debian-bookworm image: centreon-collect-debian-bookworm tag: ${{ needs.get-version.outputs.img_version }} - - runner: collect - dockerfile: centreon-collect-debian-bookworm-test - image: centreon-collect-debian-bookworm-test - tag: ${{ needs.get-version.outputs.test_img_version }} - runner: collect-arm64 dockerfile: centreon-collect-debian-bullseye image: centreon-collect-debian-bullseye-arm64 From 9f5886701c7c5636da39d4bd87f55c13e3b0c33f Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Tue, 9 Jan 2024 14:56:35 +0100 Subject: [PATCH 05/64] to revert --- ...ockerfile.centreon-collect-debian-bookworm | 34 +------------------ 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bookworm b/.github/docker/Dockerfile.centreon-collect-debian-bookworm index 452d8872063..91865682237 100644 --- a/.github/docker/Dockerfile.centreon-collect-debian-bookworm +++ b/.github/docker/Dockerfile.centreon-collect-debian-bookworm @@ -6,39 +6,7 @@ RUN < Date: Tue, 9 Jan 2024 15:17:40 +0100 Subject: [PATCH 06/64] Revert "to revert" This reverts commit 9f5886701c7c5636da39d4bd87f55c13e3b0c33f. --- ...ockerfile.centreon-collect-debian-bookworm | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bookworm b/.github/docker/Dockerfile.centreon-collect-debian-bookworm index 91865682237..452d8872063 100644 --- a/.github/docker/Dockerfile.centreon-collect-debian-bookworm +++ b/.github/docker/Dockerfile.centreon-collect-debian-bookworm @@ -6,7 +6,39 @@ RUN < Date: Tue, 9 Jan 2024 15:33:58 +0100 Subject: [PATCH 07/64] add quotes --- .github/docker/Dockerfile.centreon-collect-debian-bookworm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bookworm b/.github/docker/Dockerfile.centreon-collect-debian-bookworm index 452d8872063..98656453fc6 100644 --- a/.github/docker/Dockerfile.centreon-collect-debian-bookworm +++ b/.github/docker/Dockerfile.centreon-collect-debian-bookworm @@ -38,7 +38,7 @@ apt-get -y install cmake \ zstd \ sudo -echo http://deb.debian.org/debian | apt-get install -y pbuilder +echo 'http://deb.debian.org/debian' | apt-get install -y pbuilder echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list apt-get update From 44cb2f12b687ddb3a469f42f8873bdd974e06aa7 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Tue, 9 Jan 2024 15:38:54 +0100 Subject: [PATCH 08/64] moved around pbuilder install --- .github/docker/Dockerfile.centreon-collect-debian-bookworm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bookworm b/.github/docker/Dockerfile.centreon-collect-debian-bookworm index 98656453fc6..1cb8f9b951a 100644 --- a/.github/docker/Dockerfile.centreon-collect-debian-bookworm +++ b/.github/docker/Dockerfile.centreon-collect-debian-bookworm @@ -6,6 +6,8 @@ RUN < Date: Tue, 9 Jan 2024 16:25:47 +0100 Subject: [PATCH 09/64] add bookworm to matrix --- .github/workflows/centreon-collect.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centreon-collect.yml b/.github/workflows/centreon-collect.yml index b339f222923..cf06649e8d2 100644 --- a/.github/workflows/centreon-collect.yml +++ b/.github/workflows/centreon-collect.yml @@ -54,7 +54,7 @@ jobs: strategy: fail-fast: false matrix: - image: [alma8, alma9, debian-bullseye] + image: [alma8, alma9, debian-bullseye, debian-bookworm] name: unit test ${{ matrix.image }} steps: From ac3a83c926c83d4e99e39d06094da08582216ab4 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Tue, 9 Jan 2024 16:33:16 +0100 Subject: [PATCH 10/64] trigger ci --- common/src/hex_dump.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/common/src/hex_dump.cc b/common/src/hex_dump.cc index eabd5e70f0c..cac18c3603a 100644 --- a/common/src/hex_dump.cc +++ b/common/src/hex_dump.cc @@ -18,6 +18,7 @@ #include + #include "com/centreon/common/hex_dump.hh" inline void char_to_hex(unsigned char c, std::string& output) noexcept { From a3a85c8b13b45106758b09aa426c6a9fc75d62d5 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Tue, 9 Jan 2024 16:33:34 +0100 Subject: [PATCH 11/64] remove --- common/src/hex_dump.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/common/src/hex_dump.cc b/common/src/hex_dump.cc index cac18c3603a..eabd5e70f0c 100644 --- a/common/src/hex_dump.cc +++ b/common/src/hex_dump.cc @@ -18,7 +18,6 @@ #include - #include "com/centreon/common/hex_dump.hh" inline void char_to_hex(unsigned char c, std::string& output) noexcept { From c1ee6f69aa535cf570549f5a8bf2dcf8779a3dfa Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Tue, 9 Jan 2024 17:40:26 +0100 Subject: [PATCH 12/64] update package according to package extension and stability --- .github/actions/package/action.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/actions/package/action.yml b/.github/actions/package/action.yml index 3e12de73d08..e369b7dc0af 100644 --- a/.github/actions/package/action.yml +++ b/.github/actions/package/action.yml @@ -58,6 +58,11 @@ runs: export DIST=".${{ inputs.distrib }}" else export DIST="" + if [ "${{ inputs.stability }}" = "unstable" ]; then + export RELEASE="$RELEASE-${{ inputs.distrib }}" + elif [ "${{ inputs.stability }}" = "testing" ]; then + export RELEASE="${{ inputs.distrib }}" + fi fi export RPM_SIGNING_KEY_FILE="$(pwd)/key.gpg" From 139678105267be7ef016bfa1a771e0dc60141477 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Tue, 9 Jan 2024 18:00:12 +0100 Subject: [PATCH 13/64] forgot to add stability variable in package workflows --- .github/actions/package/action.yml | 3 +++ .github/workflows/centreon-collect.yml | 1 + .github/workflows/package-collect.yml | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/.github/actions/package/action.yml b/.github/actions/package/action.yml index e369b7dc0af..7280821d914 100644 --- a/.github/actions/package/action.yml +++ b/.github/actions/package/action.yml @@ -34,6 +34,9 @@ inputs: rpm_gpg_signing_passphrase: description: The rpm gpg signing passphrase required: true + stability: + description: "branch stability (stable, testing, unstable, canary)" + required: true runs: using: composite diff --git a/.github/workflows/centreon-collect.yml b/.github/workflows/centreon-collect.yml index cf06649e8d2..0695bc4a5d3 100644 --- a/.github/workflows/centreon-collect.yml +++ b/.github/workflows/centreon-collect.yml @@ -85,6 +85,7 @@ jobs: img_version: ${{ needs.get-version.outputs.img_version }} release: ${{ needs.get-version.outputs.release }} commit_hash: ${{ github.sha }} + stability: ${{ needs.get-version.outputs.stability }} secrets: inherit deliver-sources: diff --git a/.github/workflows/package-collect.yml b/.github/workflows/package-collect.yml index d2ea6e08d24..4967a776df0 100644 --- a/.github/workflows/package-collect.yml +++ b/.github/workflows/package-collect.yml @@ -15,6 +15,9 @@ on: commit_hash: required: true type: string + stability: + required: true + type: string jobs: package: @@ -137,6 +140,7 @@ jobs: rpm_gpg_key: ${{ secrets.RPM_GPG_SIGNING_KEY }} rpm_gpg_signing_key_id: ${{ secrets.RPM_GPG_SIGNING_KEY_ID }} rpm_gpg_signing_passphrase: ${{ secrets.RPM_GPG_SIGNING_PASSPHRASE }} + stability: ${{ inputs.stability }} - name: Cleaning not needed packages shell: bash From 976f10404e9187edade1ff955d84c8bb296f8687 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Tue, 9 Jan 2024 18:20:29 +0100 Subject: [PATCH 14/64] added canary as option for generating packages with deb distrib --- .github/actions/package/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/package/action.yml b/.github/actions/package/action.yml index 7280821d914..a4ced45c520 100644 --- a/.github/actions/package/action.yml +++ b/.github/actions/package/action.yml @@ -61,7 +61,7 @@ runs: export DIST=".${{ inputs.distrib }}" else export DIST="" - if [ "${{ inputs.stability }}" = "unstable" ]; then + if [ "${{ inputs.stability }}" = "unstable" || "${{ inputs.stability }}" = "canary" ]; then export RELEASE="$RELEASE-${{ inputs.distrib }}" elif [ "${{ inputs.stability }}" = "testing" ]; then export RELEASE="${{ inputs.distrib }}" From 4d81364c6fd29a6ef422667308480478aa01e1bb Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Wed, 10 Jan 2024 09:23:26 +0100 Subject: [PATCH 15/64] forgot brackets --- .github/actions/package/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/package/action.yml b/.github/actions/package/action.yml index a4ced45c520..fc1b3e9f68e 100644 --- a/.github/actions/package/action.yml +++ b/.github/actions/package/action.yml @@ -61,7 +61,7 @@ runs: export DIST=".${{ inputs.distrib }}" else export DIST="" - if [ "${{ inputs.stability }}" = "unstable" || "${{ inputs.stability }}" = "canary" ]; then + if [ "${{ inputs.stability }}" = "unstable" ] || [ "${{ inputs.stability }}" = "canary" ]; then export RELEASE="$RELEASE-${{ inputs.distrib }}" elif [ "${{ inputs.stability }}" = "testing" ]; then export RELEASE="${{ inputs.distrib }}" From d651d7479ebe8d458fd30e30974fd40bc3acb857 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Wed, 10 Jan 2024 10:53:02 +0100 Subject: [PATCH 16/64] add bookworm condition for the rest of the steps --- .github/actions/delivery/action.yml | 2 +- .github/actions/promote-to-stable/action.yml | 2 +- .github/workflows/centreon-collect.yml | 4 +++- .github/workflows/robot-nightly.yml | 5 ++++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/actions/delivery/action.yml b/.github/actions/delivery/action.yml index a9ae5f3c240..b6836194b3d 100644 --- a/.github/actions/delivery/action.yml +++ b/.github/actions/delivery/action.yml @@ -31,7 +31,7 @@ runs: - name: Build name for DEB shell: bash - if: ${{ inputs.distrib == 'bullseye' }} + if: ${{ inputs.distrib == 'bullseye' || inputs.distrib == 'bookworm' }} run: | echo "extfile=deb" >> $GITHUB_ENV diff --git a/.github/actions/promote-to-stable/action.yml b/.github/actions/promote-to-stable/action.yml index 249097b4a8d..94a1dcbf17c 100644 --- a/.github/actions/promote-to-stable/action.yml +++ b/.github/actions/promote-to-stable/action.yml @@ -67,7 +67,7 @@ runs: shell: bash - name: Promote DEB packages to stable - if: ${{ startsWith(inputs.distrib, 'bullseye') }} + if: ${{ startsWith(inputs.distrib, 'bullseye') || startsWith(inputs.distrib, 'bookworm') }} run: | echo "[DEBUG] - Major version: ${{ inputs.major_version }}" echo "[DEBUG] - Minor version: ${{ inputs.minor_version }}" diff --git a/.github/workflows/centreon-collect.yml b/.github/workflows/centreon-collect.yml index 0695bc4a5d3..7fe7fa78f2f 100644 --- a/.github/workflows/centreon-collect.yml +++ b/.github/workflows/centreon-collect.yml @@ -150,6 +150,8 @@ jobs: arch: amd64 - distrib: bullseye arch: arm64 + - distrib: bookworm + arch: amd64 name: deliver ${{ matrix.distrib }} @@ -173,7 +175,7 @@ jobs: runs-on: [self-hosted, common] strategy: matrix: - distrib: [el8, el9, bullseye] + distrib: [el8, el9, bullseye, bookworm] steps: - name: Checkout sources diff --git a/.github/workflows/robot-nightly.yml b/.github/workflows/robot-nightly.yml index 0816720af8d..7f8c4809a93 100644 --- a/.github/workflows/robot-nightly.yml +++ b/.github/workflows/robot-nightly.yml @@ -49,6 +49,7 @@ jobs: needs: [get-version] uses: ./.github/workflows/package-collect.yml with: + stability: ${{ needs.get-version.outputs.stability }} version: ${{ needs.get-version.outputs.version }}.${{ needs.get-version.outputs.patch }} img_version: ${{ needs.get-version.outputs.img_version }} release: ${{ needs.get-version.outputs.release }} @@ -100,7 +101,7 @@ jobs: image: ${{ matrix.image }} image_test: ${{ matrix.image }}:${{ needs.get-version.outputs.test_img_version }} image_version: ${{ needs.get-version.outputs.img_version }} - package_cache_key: cache-${{ github.sha }}-${{ matrix.package_extension}}-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} + package_cache_key: cache-${{ github.sha }}-${{ matrix.package_extension }}-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} package_cache_path: ./*.${{ matrix.package_extension}} database_type: ${{ matrix.database_type }} tests_params: ${{matrix.tests_params}} @@ -143,6 +144,8 @@ jobs: include: - distrib: bullseye arch: amd64 + - distrib: bookworm + arch: amd64 - distrib: bullseye arch: arm64 name: deliver ${{ matrix.distrib }} From d964e5a3043ee1e89a8ea2da1d6a9e45089dcb73 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Wed, 10 Jan 2024 11:02:21 +0100 Subject: [PATCH 17/64] fix --- .github/actions/delivery/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/delivery/action.yml b/.github/actions/delivery/action.yml index b6836194b3d..2567442d56c 100644 --- a/.github/actions/delivery/action.yml +++ b/.github/actions/delivery/action.yml @@ -89,7 +89,7 @@ runs: done shell: bash - - if: ${{ inputs.distrib == 'bullseye' }} + - if: ${{ inputs.distrib == 'bullseye' || inputs.distrib == "bookworm" }} name: Publish DEBs run: | FILES="*.${{ env.extfile }}" From ab26b2d30de94201b9b6da85ff06c1d7e6142b40 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Wed, 10 Jan 2024 11:21:55 +0100 Subject: [PATCH 18/64] add check to prevent 22/23 centreon versions from delivering bookworm packages --- .github/actions/delivery/action.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/actions/delivery/action.yml b/.github/actions/delivery/action.yml index 2567442d56c..31b77c22889 100644 --- a/.github/actions/delivery/action.yml +++ b/.github/actions/delivery/action.yml @@ -89,9 +89,15 @@ runs: done shell: bash - - if: ${{ inputs.distrib == 'bullseye' || inputs.distrib == "bookworm" }} + - if: ${{ inputs.distrib == 'bullseye' || inputs.distrib == 'bookworm' }} name: Publish DEBs run: | + if [ ${{ inputs.distrib == 'bookworm' }} ]; then + if [ ${{ inputs.version }} =~ 22.* ] || [ ${{ inputs.version }} =~ 23.* ]; then + echo "${{ inputs.distrib }} and ${{ inputs.version }} are not compatible. Nothing will be published" + exit 0 + fi + fi FILES="*.${{ env.extfile }}" if [ -z "${{ inputs.module_name }}" ]; then From c4e2526d53790c568f9bfc4b26d41116d6f83252 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Thu, 11 Jan 2024 14:28:48 +0100 Subject: [PATCH 19/64] ensure that testing packages are created --- .github/actions/delivery/action.yml | 4 ++-- .github/workflows/get-version.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/delivery/action.yml b/.github/actions/delivery/action.yml index 31b77c22889..ae0009dfe45 100644 --- a/.github/actions/delivery/action.yml +++ b/.github/actions/delivery/action.yml @@ -76,7 +76,7 @@ runs: mv "$FILE" "$ARCH" done - for ROOT_REPO_PATH in "rpm-standard" "rpm-standard-internal"; do + for ROOT_REPO_PATH in "test-rpm-standard"; do for ARCH in "noarch" "x86_64"; do if [ "$(ls -A $ARCH)" ]; then if [ "${{ inputs.stability }}" == "stable" ]; then @@ -119,6 +119,6 @@ runs: echo "[DEBUG] - Version: $VERSION" - jf rt upload "$FILE" "apt-standard-${{ inputs.version }}-${{ inputs.stability }}/pool/${{ inputs.module_name }}/" --deb "${{ inputs.distrib }}/main/$ARCH" + jf rt upload "$FILE" "test-apt-standard-${{ inputs.version }}-${{ inputs.stability }}/pool/${{ inputs.module_name }}/" --deb "${{ inputs.distrib }}/main/$ARCH" done shell: bash diff --git a/.github/workflows/get-version.yml b/.github/workflows/get-version.yml index 738c5bd6586..a14717b6675 100644 --- a/.github/workflows/get-version.yml +++ b/.github/workflows/get-version.yml @@ -69,7 +69,7 @@ jobs: STABILITY="unstable" ENV="development" ;; - release* | hotfix*) + release* | hotfix* | MON-34065*) STABILITY="testing" ENV="testing" ;; From 90b9e5eab08d427045d531d1e86c25441e52da46 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Thu, 11 Jan 2024 15:00:55 +0100 Subject: [PATCH 20/64] added double brackets for check --- .github/actions/delivery/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/delivery/action.yml b/.github/actions/delivery/action.yml index ae0009dfe45..28ede1befc0 100644 --- a/.github/actions/delivery/action.yml +++ b/.github/actions/delivery/action.yml @@ -93,7 +93,7 @@ runs: name: Publish DEBs run: | if [ ${{ inputs.distrib == 'bookworm' }} ]; then - if [ ${{ inputs.version }} =~ 22.* ] || [ ${{ inputs.version }} =~ 23.* ]; then + if [[ ${{ inputs.version }} =~ 22.* ]] || [[ ${{ inputs.version }} =~ 23.* ]]; then echo "${{ inputs.distrib }} and ${{ inputs.version }} are not compatible. Nothing will be published" exit 0 fi From 24be4c4639f303ed6f47b413df4f60e6f9380423 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Thu, 11 Jan 2024 17:02:43 +0100 Subject: [PATCH 21/64] added environment files --- .github/actions/package/action.yml | 3 +++ packaging/centreon-broker-core.yaml | 4 ++-- packaging/scripts/env/.env.bookworm | 2 ++ packaging/scripts/env/.env.bullseye | 2 ++ 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 packaging/scripts/env/.env.bookworm create mode 100644 packaging/scripts/env/.env.bullseye diff --git a/.github/actions/package/action.yml b/.github/actions/package/action.yml index fc1b3e9f68e..6506d39bab5 100644 --- a/.github/actions/package/action.yml +++ b/.github/actions/package/action.yml @@ -75,6 +75,9 @@ runs: for FILE in ${{ inputs.nfpm_file_pattern }}; do DIRNAME=$(dirname $FILE) BASENAME=$(basename $FILE) + if [ -f $DIRNAME/env/.env.${{ inputs.distrib }} ]; then + source $DIRNAME/env/.env.${{ inputs.distrib }} + fi cd $DIRNAME sed -i "s/@COMMIT_HASH@/${{ inputs.commit_hash }}/g" $BASENAME nfpm package --config $BASENAME --packager ${{ inputs.package_extension }} diff --git a/packaging/centreon-broker-core.yaml b/packaging/centreon-broker-core.yaml index f505fb4dd55..998732bbd50 100644 --- a/packaging/centreon-broker-core.yaml +++ b/packaging/centreon-broker-core.yaml @@ -54,10 +54,10 @@ overrides: - centreon-broker-core-devel deb: depends: - - lua5.3 + - ${LUA_VERSION} - centreon-clib (= ${VERSION}-${RELEASE}${DIST}) - centreon-broker (= ${VERSION}-${RELEASE}${DIST}) - - libgnutls30 + - ${LIB_GNU_TLS_VERSION} conflicts: - centreon-broker-storage - centreon-broker-core-dev diff --git a/packaging/scripts/env/.env.bookworm b/packaging/scripts/env/.env.bookworm new file mode 100644 index 00000000000..ced9099e553 --- /dev/null +++ b/packaging/scripts/env/.env.bookworm @@ -0,0 +1,2 @@ +LIB_GNU_TLS_VERSION=libgnutls30 +LUA_VERSION=lua5.3 \ No newline at end of file diff --git a/packaging/scripts/env/.env.bullseye b/packaging/scripts/env/.env.bullseye new file mode 100644 index 00000000000..ced9099e553 --- /dev/null +++ b/packaging/scripts/env/.env.bullseye @@ -0,0 +1,2 @@ +LIB_GNU_TLS_VERSION=libgnutls30 +LUA_VERSION=lua5.3 \ No newline at end of file From c7e237aaf69b4c0e8e1d88fe016bcf5ac064fd6b Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Thu, 11 Jan 2024 17:09:03 +0100 Subject: [PATCH 22/64] uwu --- packaging/scripts/env/.env.bullseye | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/scripts/env/.env.bullseye b/packaging/scripts/env/.env.bullseye index ced9099e553..08e537f39b1 100644 --- a/packaging/scripts/env/.env.bullseye +++ b/packaging/scripts/env/.env.bullseye @@ -1,2 +1,2 @@ LIB_GNU_TLS_VERSION=libgnutls30 -LUA_VERSION=lua5.3 \ No newline at end of file +LUA_VERSION=lua5.3https://github.com/centreon/centreon-modules/pull/2063 \ No newline at end of file From 9353708e92321c350b2ff6c059f2bfd7e4ee5210 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Thu, 11 Jan 2024 17:09:16 +0100 Subject: [PATCH 23/64] uwu --- packaging/scripts/env/.env.bullseye | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/scripts/env/.env.bullseye b/packaging/scripts/env/.env.bullseye index 08e537f39b1..ced9099e553 100644 --- a/packaging/scripts/env/.env.bullseye +++ b/packaging/scripts/env/.env.bullseye @@ -1,2 +1,2 @@ LIB_GNU_TLS_VERSION=libgnutls30 -LUA_VERSION=lua5.3https://github.com/centreon/centreon-modules/pull/2063 \ No newline at end of file +LUA_VERSION=lua5.3 \ No newline at end of file From b75e76e8ec773952e572b59574cc5204bf2483ca Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Thu, 11 Jan 2024 17:24:34 +0100 Subject: [PATCH 24/64] reverted changes to push testing packages --- .github/actions/delivery/action.yml | 4 ++-- .github/workflows/get-version.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/delivery/action.yml b/.github/actions/delivery/action.yml index 28ede1befc0..d4e9486563b 100644 --- a/.github/actions/delivery/action.yml +++ b/.github/actions/delivery/action.yml @@ -76,7 +76,7 @@ runs: mv "$FILE" "$ARCH" done - for ROOT_REPO_PATH in "test-rpm-standard"; do + for ROOT_REPO_PATH in "rpm-standard" "rpm-standard-internal"; do for ARCH in "noarch" "x86_64"; do if [ "$(ls -A $ARCH)" ]; then if [ "${{ inputs.stability }}" == "stable" ]; then @@ -119,6 +119,6 @@ runs: echo "[DEBUG] - Version: $VERSION" - jf rt upload "$FILE" "test-apt-standard-${{ inputs.version }}-${{ inputs.stability }}/pool/${{ inputs.module_name }}/" --deb "${{ inputs.distrib }}/main/$ARCH" + jf rt upload "$FILE" "apt-standard-${{ inputs.version }}-${{ inputs.stability }}/pool/${{ inputs.module_name }}/" --deb "${{ inputs.distrib }}/main/$ARCH" done shell: bash diff --git a/.github/workflows/get-version.yml b/.github/workflows/get-version.yml index a14717b6675..738c5bd6586 100644 --- a/.github/workflows/get-version.yml +++ b/.github/workflows/get-version.yml @@ -69,7 +69,7 @@ jobs: STABILITY="unstable" ENV="development" ;; - release* | hotfix* | MON-34065*) + release* | hotfix*) STABILITY="testing" ENV="testing" ;; From 9ae423736e02d795cd312582f87cb694607e59d0 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Wed, 17 Jan 2024 12:05:42 +0100 Subject: [PATCH 25/64] fixes and changes brought by reviews --- .github/actions/package/action.yml | 2 +- .github/workflows/centreon-collect.yml | 2 +- .github/workflows/robot-nightly.yml | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/package/action.yml b/.github/actions/package/action.yml index 6506d39bab5..921e54dd9c9 100644 --- a/.github/actions/package/action.yml +++ b/.github/actions/package/action.yml @@ -64,7 +64,7 @@ runs: if [ "${{ inputs.stability }}" = "unstable" ] || [ "${{ inputs.stability }}" = "canary" ]; then export RELEASE="$RELEASE-${{ inputs.distrib }}" elif [ "${{ inputs.stability }}" = "testing" ]; then - export RELEASE="${{ inputs.distrib }}" + export RELEASE="1-${{ inputs.distrib }}" fi fi diff --git a/.github/workflows/centreon-collect.yml b/.github/workflows/centreon-collect.yml index 7fe7fa78f2f..cb693924714 100644 --- a/.github/workflows/centreon-collect.yml +++ b/.github/workflows/centreon-collect.yml @@ -152,7 +152,6 @@ jobs: arch: arm64 - distrib: bookworm arch: amd64 - name: deliver ${{ matrix.distrib }} steps: @@ -160,6 +159,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Publish DEB packages + if: ${{ !contains('bookworm', matrix.distrib) || !startsWith('22.', needs.get-version.outputs.version) || !startsWith('23.', needs.get-version.outputs.version) }} uses: ./.github/actions/delivery with: module_name: collect diff --git a/.github/workflows/robot-nightly.yml b/.github/workflows/robot-nightly.yml index 7f8c4809a93..9c35c740ae2 100644 --- a/.github/workflows/robot-nightly.yml +++ b/.github/workflows/robot-nightly.yml @@ -136,7 +136,7 @@ jobs: stability: ${{ needs.get-version.outputs.stability }} deliver-deb: - if: ${{ contains(fromJson('["unstable"]'), needs.get-version.outputs.stability) }} + if: ${{ contains(fromJson('["testing", "stable"]'), needs.get-version.outputs.stability) }} needs: [robot-test, get-version] runs-on: [self-hosted, common] strategy: @@ -155,6 +155,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Publish DEB packages + if: ${{ !contains('bookworm', matrix.distrib) || !startsWith('22.', needs.get-version.outputs.version) || !startsWith('23.', needs.get-version.outputs.version) }} uses: ./.github/actions/delivery with: module_name: collect From bb1c22c572fda0b6af079b00f25e0f4bf4a55e48 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Wed, 17 Jan 2024 12:07:04 +0100 Subject: [PATCH 26/64] fix --- .github/workflows/centreon-collect.yml | 2 +- .github/workflows/robot-nightly.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centreon-collect.yml b/.github/workflows/centreon-collect.yml index cb693924714..7a0c04654e4 100644 --- a/.github/workflows/centreon-collect.yml +++ b/.github/workflows/centreon-collect.yml @@ -139,7 +139,7 @@ jobs: stability: ${{ needs.get-version.outputs.stability }} deliver-deb: - if: ${{ contains(fromJson('["testing", "stable"]'), needs.get-version.outputs.stability) }} + if: ${{ contains(fromJson('["unstable"]'), needs.get-version.outputs.stability) }} needs: [get-version, package] environment: ${{ needs.get-version.outputs.environment }} runs-on: [self-hosted, common] diff --git a/.github/workflows/robot-nightly.yml b/.github/workflows/robot-nightly.yml index 9c35c740ae2..9c77b7b4826 100644 --- a/.github/workflows/robot-nightly.yml +++ b/.github/workflows/robot-nightly.yml @@ -136,7 +136,7 @@ jobs: stability: ${{ needs.get-version.outputs.stability }} deliver-deb: - if: ${{ contains(fromJson('["testing", "stable"]'), needs.get-version.outputs.stability) }} + if: ${{ contains(fromJson('["unstable"]'), needs.get-version.outputs.stability) }} needs: [robot-test, get-version] runs-on: [self-hosted, common] strategy: From 6a1125a8659b7bc8b5d8ae7550819e6afc0db090 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Wed, 17 Jan 2024 12:07:53 +0100 Subject: [PATCH 27/64] fix --- .github/workflows/centreon-collect.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centreon-collect.yml b/.github/workflows/centreon-collect.yml index 7a0c04654e4..cb693924714 100644 --- a/.github/workflows/centreon-collect.yml +++ b/.github/workflows/centreon-collect.yml @@ -139,7 +139,7 @@ jobs: stability: ${{ needs.get-version.outputs.stability }} deliver-deb: - if: ${{ contains(fromJson('["unstable"]'), needs.get-version.outputs.stability) }} + if: ${{ contains(fromJson('["testing", "stable"]'), needs.get-version.outputs.stability) }} needs: [get-version, package] environment: ${{ needs.get-version.outputs.environment }} runs-on: [self-hosted, common] From 0653c659cc58c60c99142aa2605b38287045fe70 Mon Sep 17 00:00:00 2001 From: David Boucher Date: Wed, 17 Jan 2024 15:33:30 +0100 Subject: [PATCH 28/64] fix(broker/lua): functions in error could keep data on the stack (#1069) REFS: MON-33334 --- .../inc/com/centreon/broker/lua/luabinding.hh | 38 +++---- broker/lua/src/luabinding.cc | 94 ++++++++++++------ broker/lua/test/lua.cc | 98 ++++++++++++++++--- 3 files changed, 165 insertions(+), 65 deletions(-) diff --git a/broker/lua/inc/com/centreon/broker/lua/luabinding.hh b/broker/lua/inc/com/centreon/broker/lua/luabinding.hh index 9f3e7ba0aeb..74b5a048ce0 100644 --- a/broker/lua/inc/com/centreon/broker/lua/luabinding.hh +++ b/broker/lua/inc/com/centreon/broker/lua/luabinding.hh @@ -1,20 +1,20 @@ -/* -** Copyright 2018 Centreon -** -** Licensed under the Apache License, Version 2.0 (the "License"); -** you may not use this file except in compliance with the License. -** You may obtain a copy of the License at -** -** http://www.apache.org/licenses/LICENSE-2.0 -** -** Unless required by applicable law or agreed to in writing, software -** distributed under the License is distributed on an "AS IS" BASIS, -** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -** See the License for the specific language governing permissions and -** limitations under the License. -** -** For more information : contact@centreon.com -*/ +/** + * Copyright 2018-2024 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ #ifndef CCB_LUA_LUABINDING_HH #define CCB_LUA_LUABINDING_HH @@ -112,7 +112,7 @@ class luabinding { macro_cache& cache); luabinding(luabinding const&) = delete; luabinding& operator=(luabinding const&) = delete; - ~luabinding() noexcept = default; + ~luabinding() noexcept; bool has_filter() const noexcept; int32_t write(std::shared_ptr const& data) noexcept; bool has_flush() const noexcept; @@ -125,6 +125,6 @@ void push_event_as_table(lua_State* L, io::data const& d); } // namespace lua -} +} // namespace com::centreon::broker #endif // !CCB_LUA_LUA_HH diff --git a/broker/lua/src/luabinding.cc b/broker/lua/src/luabinding.cc index 6fc6250830d..a2ac8750d1e 100644 --- a/broker/lua/src/luabinding.cc +++ b/broker/lua/src/luabinding.cc @@ -1,22 +1,23 @@ /** -* Copyright 2018 Centreon -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* For more information : contact@centreon.com -*/ + * Copyright 2018-2024 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ #include + #include #include "com/centreon/broker/log_v2.hh" @@ -40,6 +41,11 @@ static int l_pairs(lua_State* L) { return 3; } #endif + +#define RETURN_AND_POP(val) \ + lua_pop(_L, lua_gettop(_L)); \ + return val + /** * Constructor. * @@ -73,9 +79,17 @@ luabinding::luabinding(std::string const& lua_script, } } +/** + * @brief Destructor of luabinding. + */ +luabinding::~luabinding() noexcept { + stop(); +} + int32_t luabinding::stop() { - int32_t retval = flush(); + int32_t retval = 0; if (_L) { + retval = flush(); lua_close(_L); _L = nullptr; } @@ -166,9 +180,30 @@ void luabinding::_load_script(const std::string& lua_script) { "lua: filter() global function is missing, the write() function will " "be called for each event"); _filter = false; - } else + } else { _filter = true; - lua_pop(_L, 1); + /* Just a call with cat = 1 and elem = 2 of filter to check its return. + * It is not sufficent but checks almost all cases. */ + lua_pushinteger(_L, 1); + lua_pushinteger(_L, 2); + if (lua_pcall(_L, 2, 1, 0) != 0) { + const char* ret = lua_tostring(_L, -1); + if (ret) + log_v2::lua()->error( + "lua: The filter() function doesn't work correctly: {}", ret); + else + log_v2::lua()->error( + "lua: The filter() function doesn't work correctly"); + _filter = false; + } else { + if (!lua_isboolean(_L, -1)) { + log_v2::lua()->error( + "lua: The filter() function should return a boolean."); + _filter = false; + } + } + } + lua_pop(_L, lua_gettop(_L)); // Checking for flush() availability: this function is optional lua_getglobal(_L, "flush"); @@ -231,6 +266,7 @@ void luabinding::_load_script(const std::string& lua_script) { // Registers the broker cache broker_cache::broker_cache_reg(_L, _cache, _broker_api_version); + assert(lua_gettop(_L) == 0); } /** @@ -330,18 +366,18 @@ int luabinding::write(std::shared_ptr const& data) noexcept { SPDLOG_LOGGER_ERROR( log_v2::lua(), "lua: unknown error while running function `filter()'"); - return 0; + RETURN_AND_POP(0); } if (!lua_isboolean(_L, -1)) { SPDLOG_LOGGER_ERROR(log_v2::lua(), "lua: `filter' must return a boolean"); - return 0; + RETURN_AND_POP(0); } execute_write = lua_toboolean(_L, -1); SPDLOG_LOGGER_DEBUG(log_v2::lua(), "lua: `filter' returned {}", (execute_write ? "true" : "false")); - lua_pop(_L, -1); + lua_pop(_L, lua_gettop(_L)); } if (!execute_write) @@ -371,15 +407,14 @@ int luabinding::write(std::shared_ptr const& data) noexcept { else SPDLOG_LOGGER_ERROR(log_v2::lua(), "lua: unknown error running function `write'"); - return 0; + RETURN_AND_POP(0); } if (!lua_isboolean(_L, -1)) { SPDLOG_LOGGER_ERROR(log_v2::lua(), "lua: `write' must return a boolean"); - return 0; + RETURN_AND_POP(0); } int acknowledge = lua_toboolean(_L, -1); - lua_pop(_L, -1); // We have to acknowledge rejected events by the filter. It is only possible // when an acknowledgement is sent by the write function. @@ -387,7 +422,7 @@ int luabinding::write(std::shared_ptr const& data) noexcept { retval = _total; _total = 0; } - return retval; + RETURN_AND_POP(retval); } /** @@ -417,19 +452,18 @@ int32_t luabinding::flush() noexcept { else SPDLOG_LOGGER_ERROR(log_v2::lua(), "lua: unknown error running function `flush'"); - return 0; + RETURN_AND_POP(0); } if (!lua_isboolean(_L, -1)) { SPDLOG_LOGGER_ERROR(log_v2::lua(), "lua: `flush' must return a boolean"); - return 0; + RETURN_AND_POP(0); } bool acknowledge = lua_toboolean(_L, -1); - lua_pop(_L, -1); int32_t retval = 0; if (acknowledge) { retval = _total; _total = 0; } - return retval; + RETURN_AND_POP(retval); } diff --git a/broker/lua/test/lua.cc b/broker/lua/test/lua.cc index 3153f45e2b5..4a087abf5e6 100644 --- a/broker/lua/test/lua.cc +++ b/broker/lua/test/lua.cc @@ -1,20 +1,20 @@ /** -* Copyright 2018-2022 Centreon -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* For more information : contact@centreon.com -*/ + * Copyright 2018-2024 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ #include #include @@ -4764,3 +4764,69 @@ TEST_F(LuaTest, JsonDecodeNull) { RemoveFile(filename); RemoveFile("/tmp/log"); } + +TEST_F(LuaTest, BadLua) { + config::applier::modules modules; + modules.load_file("./lib/10-neb.so"); + std::map conf; + std::string filename("/tmp/bad.lua"); + CreateScript(filename, + "function init(conf)\n" + " broker_log:set_parameters(3, '/tmp/log')\n" + "end\n\n" + "function write(d)\n" + " bad_function()\n" + " return true\n" + "end\n"); + auto binding{std::make_unique(filename, conf, *_cache)}; + auto s{std::make_unique()}; + s->host_id = 12; + s->service_id = 18; + s->output = "Bonjour"; + std::shared_ptr svc(s.release()); + ASSERT_EQ(binding->write(svc), 0); + RemoveFile(filename); +} + +// When a lua script that contains a bad filter() function. "Bad" here +// is because the return value is not a boolean. +// Then has_filter() returns false and there is no leak on the stack. +// (the leak is seen when compiled with -g). +TEST_F(LuaTest, WithBadFilter1) { + std::map conf; + std::string filename("/tmp/with_bad_filter.lua"); + CreateScript(filename, + "function init()\n" + "end\n" + "function filter(c, e)\n" + " return \"foo\", \"bar\"\n" + "end\n" + "function write(d)\n" + " return 1\n" + "end"); + auto bb{std::make_unique(filename, conf, *_cache)}; + ASSERT_FALSE(bb->has_filter()); + RemoveFile(filename); +} + +// When a lua script that contains a bad filter() function. "Bad" here +// because the filter calls a function that doesn't exist. +// Then has_filter() returns false and there is no leak on the stack. +// (the leak is seen when compiled with -g). +TEST_F(LuaTest, WithBadFilter2) { + std::map conf; + std::string filename("/tmp/with_bad_filter.lua"); + CreateScript(filename, + "function init()\n" + "end\n" + "function filter(c, e)\n" + " unexisting_function()\n" + " return \"foo\", \"bar\"\n" + "end\n" + "function write(d)\n" + " return 1\n" + "end"); + auto bb{std::make_unique(filename, conf, *_cache)}; + ASSERT_FALSE(bb->has_filter()); + RemoveFile(filename); +} From d897ca5e0f32dc1fb66f21f4ad57ce0eb7bf808d Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Wed, 17 Jan 2024 15:44:30 +0100 Subject: [PATCH 29/64] fix --- .github/actions/delivery/action.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/actions/delivery/action.yml b/.github/actions/delivery/action.yml index d4e9486563b..24ec39cbc6b 100644 --- a/.github/actions/delivery/action.yml +++ b/.github/actions/delivery/action.yml @@ -92,12 +92,6 @@ runs: - if: ${{ inputs.distrib == 'bullseye' || inputs.distrib == 'bookworm' }} name: Publish DEBs run: | - if [ ${{ inputs.distrib == 'bookworm' }} ]; then - if [[ ${{ inputs.version }} =~ 22.* ]] || [[ ${{ inputs.version }} =~ 23.* ]]; then - echo "${{ inputs.distrib }} and ${{ inputs.version }} are not compatible. Nothing will be published" - exit 0 - fi - fi FILES="*.${{ env.extfile }}" if [ -z "${{ inputs.module_name }}" ]; then From ac7e914deffec26ecff538cccf0c1e46afd7ffc3 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Wed, 17 Jan 2024 15:56:23 +0100 Subject: [PATCH 30/64] fix --- packaging/scripts/env/.env.bookworm | 2 +- packaging/scripts/env/.env.bullseye | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/scripts/env/.env.bookworm b/packaging/scripts/env/.env.bookworm index ced9099e553..538e115d971 100644 --- a/packaging/scripts/env/.env.bookworm +++ b/packaging/scripts/env/.env.bookworm @@ -1,2 +1,2 @@ LIB_GNU_TLS_VERSION=libgnutls30 -LUA_VERSION=lua5.3 \ No newline at end of file +LUA_VERSION=lua5.3 diff --git a/packaging/scripts/env/.env.bullseye b/packaging/scripts/env/.env.bullseye index ced9099e553..538e115d971 100644 --- a/packaging/scripts/env/.env.bullseye +++ b/packaging/scripts/env/.env.bullseye @@ -1,2 +1,2 @@ LIB_GNU_TLS_VERSION=libgnutls30 -LUA_VERSION=lua5.3 \ No newline at end of file +LUA_VERSION=lua5.3 From e173da3f5182c25460f645ffea2edc48ffa5ea02 Mon Sep 17 00:00:00 2001 From: jean-christophe81 <98889244+jean-christophe81@users.noreply.github.com> Date: Thu, 18 Jan 2024 10:27:32 +0100 Subject: [PATCH 31/64] fix/enh(broker): cbd memory leak in sql module and new library to trace allocations (#1071) REFS: MON-33334 --- CMakeLists.txt | 6 + broker/core/sql/src/mysql_connection.cc | 1 + malloc-trace/CMakeLists.txt | 42 +++ malloc-trace/README.md | 67 ++++ malloc-trace/create_malloc_trace_table.sql | 32 ++ .../malloc_trace/by_thread_trace_active.hh | 99 ++++++ .../centreon/malloc_trace/funct_info_cache.hh | 50 +++ .../centreon/malloc_trace/intrusive_map.hh | 82 +++++ .../centreon/malloc_trace/orphan_container.hh | 211 ++++++++++++ .../centreon/malloc_trace/simply_allocator.hh | 59 ++++ malloc-trace/precomp_inc/precomp.hh | 34 ++ malloc-trace/remove_malloc_free.py | 62 ++++ malloc-trace/src/by_thread_trace_active.cc | 57 ++++ malloc-trace/src/malloc_trace.cc | 242 ++++++++++++++ malloc-trace/src/orphan_container.cc | 304 ++++++++++++++++++ malloc-trace/src/simply_allocator.cc | 78 +++++ 16 files changed, 1426 insertions(+) create mode 100644 malloc-trace/CMakeLists.txt create mode 100644 malloc-trace/README.md create mode 100644 malloc-trace/create_malloc_trace_table.sql create mode 100644 malloc-trace/inc/com/centreon/malloc_trace/by_thread_trace_active.hh create mode 100644 malloc-trace/inc/com/centreon/malloc_trace/funct_info_cache.hh create mode 100644 malloc-trace/inc/com/centreon/malloc_trace/intrusive_map.hh create mode 100644 malloc-trace/inc/com/centreon/malloc_trace/orphan_container.hh create mode 100644 malloc-trace/inc/com/centreon/malloc_trace/simply_allocator.hh create mode 100644 malloc-trace/precomp_inc/precomp.hh create mode 100755 malloc-trace/remove_malloc_free.py create mode 100644 malloc-trace/src/by_thread_trace_active.cc create mode 100644 malloc-trace/src/malloc_trace.cc create mode 100644 malloc-trace/src/orphan_container.cc create mode 100644 malloc-trace/src/simply_allocator.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 273ca6aedef..e9cbd44ba88 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,8 @@ if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT CMAKE_CXX_COMPILER_ID FATAL_ERROR "You can build broker with g++ or clang++. CMake will exit.") endif() +option(WITH_MALLOC_TRACE "compile centreon-malloc-trace library." OFF) + # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -stdlib=libc++") # set(CMAKE_CXX_COMPILER "clang++") add_definitions("-D_GLIBCXX_USE_CXX11_ABI=1") @@ -190,6 +192,10 @@ add_subdirectory(engine) add_subdirectory(connectors) add_subdirectory(ccc) +if (WITH_MALLOC_TRACE) + add_subdirectory(malloc-trace) +endif() + add_custom_target(test-broker COMMAND tests/ut_broker) add_custom_target(test-engine COMMAND tests/ut_engine) add_custom_target(test-clib COMMAND tests/ut_clib) diff --git a/broker/core/sql/src/mysql_connection.cc b/broker/core/sql/src/mysql_connection.cc index db18b4e50f3..23a38f455a8 100644 --- a/broker/core/sql/src/mysql_connection.cc +++ b/broker/core/sql/src/mysql_connection.cc @@ -848,6 +848,7 @@ void mysql_connection::_run() { ::mysql_error(_conn))); _state = finished; _start_condition.notify_all(); + _clear_connection(); return; } _last_access = std::time(nullptr); diff --git a/malloc-trace/CMakeLists.txt b/malloc-trace/CMakeLists.txt new file mode 100644 index 00000000000..dd1954a6c6c --- /dev/null +++ b/malloc-trace/CMakeLists.txt @@ -0,0 +1,42 @@ +# +# Copyright 2024 Centreon +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# For more information : contact@centreon.com +# + +# Global options. +project("Centreon malloc trace" C CXX) + +# Set directories. +set(INC_DIR "${PROJECT_SOURCE_DIR}/inc/com/centreon/malloc_trace") +set(SRC_DIR "src") + +add_library(centreon-malloc-trace SHARED + "src/by_thread_trace_active.cc" + "src/malloc_trace.cc" + "src/orphan_container.cc" + "src/simply_allocator.cc" +) + +target_link_libraries(centreon-malloc-trace + CONAN_PKG::fmt +) + +target_include_directories(centreon-malloc-trace PRIVATE + ${INC_DIR} + ${CMAKE_SOURCE_DIR}/common/inc +) + +target_precompile_headers(centreon-malloc-trace PRIVATE "precomp_inc/precomp.hh") \ No newline at end of file diff --git a/malloc-trace/README.md b/malloc-trace/README.md new file mode 100644 index 00000000000..78e0e7b640c --- /dev/null +++ b/malloc-trace/README.md @@ -0,0 +1,67 @@ +# malloc-trace + +## Description + +The goal of this little library is to trace each orphan malloc free call. It overrides weak **malloc**, **realloc**, **calloc** and **free** +We store in a container in memory every malloc, free calls. We remove malloc from that container each time a free with the same address is called otherwise free is also store in the container. +Every minute (by default), we flush to disk container content: + * malloc that had not be freed and that are older than one minute + * free that has not corresponding malloc in the container. + +In order to use it you have to feed LD_PRELOAD env variable +```bash +export LD_PRELOAD=malloc-trace.so +``` +Then you can launch your executable and each call will be recorded in /tmp/malloc-trace.csv with ';' as field separator + +The columns are: +* function (malloc or free) +* thread id +* address in process mem space +* size of memory allocated +* timestamp in ms +* call stack contained in a json array + ```json + [ + { + "f": "free", + "s": "", + "l": 0 + }, + { + "f": "__gnu_cxx::new_allocator, (__gnu_cxx::_Lock_policy)2> >::deallocate(std::_Sp_counted_ptr_inplace, (__gnu_cxx::_Lock_policy)2>*, unsigned long)", + "s": "", + "l": 0 + } + ] + ``` +f field is function name +s field is source file if available +l field is source line + +This library works in that manner: +We replace all malloc, realloc, calloc and free in order to trace all calls. +We store all malloc in a container. Each time free is called, if the corresponding malloc is found, it's erased from container, +otherwise orphan free is stored in the container. +Every 60s (by default), we flush the container, all malloc older than 60s and not freed are dumped to disk, all orphan freed are also dumped. + +Output file may be moved during running, in that case it's automatically recreated. + +## Environment variables +Some parameters of the library can be overriden with environment variables. +| Environment variable | default value | description | +| ------------------------ | --------------------- | ----------------------------------------------------------------------------- | +| out_file_path | /tmp/malloc-trace.csv | path of the output file | +| out_file_max_size | 0x100000000 | when the output file size exceeds this limit, the ouput file is truncated | +| malloc_second_peremption | one minute | delay between two flushes and delay after which malloc is considered orphaned | + +## Provided scripts + +### create_malloc_trace_table.sql +This script creates a table that can store an output_file +In this scripts, you will find in comments how to load output csv file in that table. + +### remove_malloc_free.py +We store in output file malloc that aren't freed in the next minute, we also store orphan free. +So if a malloc is dumped and it's corresponding free is operated two minutes later, the two are stored in output file. +The purpose of this script is to remove these malloc-free pairs. \ No newline at end of file diff --git a/malloc-trace/create_malloc_trace_table.sql b/malloc-trace/create_malloc_trace_table.sql new file mode 100644 index 00000000000..c5bdda41c48 --- /dev/null +++ b/malloc-trace/create_malloc_trace_table.sql @@ -0,0 +1,32 @@ +-- Copyright 2024 Centreon +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- For more information : contact@centreon.com +-- + + +-- this sql code lets you to create table where you can load data from /tmp/malloc-trace.csv +-- you have to connect to the bdd with this command line: +-- mysql -h 127.0.0.1 --local-infile=1 -D centreon_storage -u centreon -pcentreon +-- then you load data with +-- load data local infile '/tmp/malloc-trace.csv' into table malloc_trace fields terminated by ';'; + +CREATE TABLE `centreon_storage`.`malloc_trace` ( + `funct_name` VARCHAR(10) NOT NULL, + `thread_id` INT UNSIGNED NULL, + `address` BIGINT UNSIGNED NULL, + `size` INT UNSIGNED NULL, + `ms_timestamp` BIGINT UNSIGNED NULL, + `call_stack` TEXT(65535) NULL, + FULLTEXT INDEX `call_stack_ind` (`call_stack`) VISIBLE); diff --git a/malloc-trace/inc/com/centreon/malloc_trace/by_thread_trace_active.hh b/malloc-trace/inc/com/centreon/malloc_trace/by_thread_trace_active.hh new file mode 100644 index 00000000000..abadb5b6c7c --- /dev/null +++ b/malloc-trace/inc/com/centreon/malloc_trace/by_thread_trace_active.hh @@ -0,0 +1,99 @@ +/** + * Copyright 2024 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ + +#ifndef CMT_BY_THREAD_TRACE_ACTIVE_HH +#define CMT_BY_THREAD_TRACE_ACTIVE_HH + +#include "intrusive_map.hh" + +namespace com::centreon::malloc_trace { + +/** + * @brief This class is used to store the tracing of a thread + * The problem is: malloc is called, we explore call stack and this research may + * do another malloc and we risk an infinite loop + * So the first malloc set the _malloc_trace_active and explore call stack + * The next malloc called by stacktrace process try to set _malloc_trace_active + * and as it's yet setted we don't try to explore call stack + * + */ +class thread_trace_active : public boost::intrusive::set_base_hook<> { + pid_t _thread_id; + mutable bool _malloc_trace_active = false; + + public: + thread_trace_active() {} + thread_trace_active(pid_t thread_id) : _thread_id(thread_id) {} + + pid_t get_thread_id() const { return _thread_id; } + + /** + * @brief Set _malloc_trace_active + * + * @return old value of _malloc_trace_active + */ + bool set_malloc_trace_active() const { + if (_malloc_trace_active) { + return true; + } + _malloc_trace_active = true; + return false; + } + + /** + * @brief reset _malloc_trace_active + * + * @return old value of _malloc_trace_active + */ + bool reset_malloc_trace_active() const { + if (!_malloc_trace_active) { + return false; + } + _malloc_trace_active = false; + return true; + } + + bool is_malloc_trace_active() const { return _malloc_trace_active; } + + struct key_extractor { + using type = pid_t; + type operator()(const thread_trace_active& node) const { + return node._thread_id; + } + }; +}; + +/** + * @brief container of thread_trace_active with zero allocation + * the drawback is that we are limited to store 4096 thread trace states + * + */ +class thread_dump_active + : protected intrusive_map { + std::mutex _protect; + + public: + bool set_dump_active(pid_t thread_id); + void reset_dump_active(pid_t thread_id); +}; + +} // namespace com::centreon::malloc_trace + +#endif diff --git a/malloc-trace/inc/com/centreon/malloc_trace/funct_info_cache.hh b/malloc-trace/inc/com/centreon/malloc_trace/funct_info_cache.hh new file mode 100644 index 00000000000..77a5eef010a --- /dev/null +++ b/malloc-trace/inc/com/centreon/malloc_trace/funct_info_cache.hh @@ -0,0 +1,50 @@ +/** + * Copyright 2024 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ +#ifndef CMT_FUNCT_INFO_CACHE_HH +#define CMT_FUNCT_INFO_CACHE_HH + +namespace com::centreon::malloc_trace { +/** + * @brief symbol information research is very expensive + * so we store function informations in a cache + * + */ +class funct_info { + const std::string _funct_name; + const std::string _source_file; + const size_t _source_line; + + public: + funct_info(std::string&& funct_name, + std::string&& source_file, + size_t source_line) + : _funct_name(funct_name), + _source_file(source_file), + _source_line(source_line) {} + + const std::string& get_funct_name() const { return _funct_name; } + const std::string& get_source_file() const { return _source_file; } + size_t get_source_line() const { return _source_line; } +}; + +using funct_cache_map = + std::map; + +} // namespace com::centreon::malloc_trace + +#endif diff --git a/malloc-trace/inc/com/centreon/malloc_trace/intrusive_map.hh b/malloc-trace/inc/com/centreon/malloc_trace/intrusive_map.hh new file mode 100644 index 00000000000..0962f8f52b7 --- /dev/null +++ b/malloc-trace/inc/com/centreon/malloc_trace/intrusive_map.hh @@ -0,0 +1,82 @@ +/** + * Copyright 2024 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ +#ifndef CMT_INTRUSIVE_MAP_HH +#define CMT_INTRUSIVE_MAP_HH + +namespace com::centreon::malloc_trace { + +/** + * @brief The goal of this class is to provide map without allocation + * + * @tparam node_type node (key and data) that must inherit from + * boost::intrusive::set_base_hook<> + * @tparam key_extractor struct with an operator that extract key from node_type + * @tparam node_arrray_size size max of the container + */ +template +class intrusive_map { + public: + using key_type = typename key_extractor::type; + + private: + node_type _nodes_array[node_array_size]; + node_type* _free_node = _nodes_array; + const node_type* _array_end = _free_node + node_array_size; + + using node_map = + boost::intrusive::set >; + + node_map _nodes; + + public: + + ~intrusive_map() { _nodes.clear(); } + + const node_type* find(const key_type& key) const { + auto found = _nodes.find(key); + if (found == _nodes.end()) { + return nullptr; + } else { + return &*found; + } + } + + const node_type* insert_and_get(const key_type& key) { + if (_free_node >= _array_end) { + return nullptr; + } + + node_type* to_insert = _free_node++; + new (to_insert) node_type(key); + _nodes.insert(*to_insert); + return to_insert; + } + + /** + * @brief sometimes method are called before object construction + * + * @return true constructor has been called + * @return false + */ + bool is_initialized() const { return _free_node; } +}; + +} // namespace com::centreon::malloc_trace + +#endif diff --git a/malloc-trace/inc/com/centreon/malloc_trace/orphan_container.hh b/malloc-trace/inc/com/centreon/malloc_trace/orphan_container.hh new file mode 100644 index 00000000000..3c0d76dc546 --- /dev/null +++ b/malloc-trace/inc/com/centreon/malloc_trace/orphan_container.hh @@ -0,0 +1,211 @@ +/** + * Copyright 2024 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ + +#ifndef CMT_ORPHAN_CONTAINER_HH +#define CMT_ORPHAN_CONTAINER_HH + +#include + +#include "funct_info_cache.hh" + +namespace com::centreon::malloc_trace { + +constexpr size_t max_backtrace_size = 10; + +/** + * @brief information of free or malloc with stacktrace + * In this bean, we store: + * - allocated address + * - size of memory allocated (0 if free) + * - thread id + * - function name: malloc, free, realloc or freerealloc + * - length of the backtrace + * - backtrace + * - timestamp + */ +class backtrace_info { + const void* _allocated; + const size_t _allocated_size; + const pid_t _thread_id; + const std::string_view _funct_name; + size_t _backtrace_size; + boost::stacktrace::frame::native_frame_ptr_t _backtrace[max_backtrace_size]; + const std::chrono::system_clock::time_point _last_allocated; + + public: + backtrace_info(const void* allocated, + size_t allocated_size, + pid_t thread_id, + const std::string_view& funct_name, + const boost::stacktrace::stacktrace& backtrace, + size_t backtrace_offset); + + const void* get_allocated() const { return _allocated; } + size_t get_allocated_size() const { return _allocated_size; } + size_t get_backtrace_size() const { return _backtrace_size; } + const boost::stacktrace::frame::native_frame_ptr_t* get_backtrace() const { + return _backtrace; + } + const std::chrono::system_clock::time_point& get_last_allocated() const { + return _last_allocated; + } + + void to_file(int fd, funct_cache_map& funct_info_cache) const; +}; + +/** + * @brief infos of malloc + * as this object is stored in 2 containers, it has to set hooks: + * set_base_hook and allocated_time_hook + * + */ +class orphan_malloc : public backtrace_info, + public boost::intrusive::set_base_hook<> { + public: + boost::intrusive::set_member_hook<> allocated_time_hook; + + orphan_malloc(const void* allocated, + size_t allocated_size, + pid_t thread_id, + const std::string_view& funct_name, + const boost::stacktrace::stacktrace& backtrace, + size_t backtrace_offset) + : backtrace_info(allocated, + allocated_size, + thread_id, + funct_name, + backtrace, + backtrace_offset) {} + + // key extractor used to create a map addr to orphan_malloc + struct address_extractor { + using type = const void*; + type operator()(const orphan_malloc& node) const { + return node.get_allocated(); + } + }; + + // key extractor used to create a map time_alloc to orphan_malloc + struct time_allocated_extractor { + using type = std::chrono::system_clock::time_point; + const type& operator()(const orphan_malloc& node) const { + return node.get_last_allocated(); + } + }; +}; + +/** + * @brief infos of a free + * this object is stored in single link list: slist_base_hook + * + */ +class orphan_free : public backtrace_info, + public boost::intrusive::slist_base_hook<> { + public: + orphan_free(const void* allocated, + pid_t thread_id, + const std::string_view& funct_name, + const boost::stacktrace::stacktrace& backtrace, + size_t backtrace_offset) + : backtrace_info(allocated, + 0, + thread_id, + funct_name, + backtrace, + backtrace_offset) {} +}; + +/** + * @brief this object contains all orphan mallocs (malloc without free) and all + * orphan frees In order to limit allocation and improve performance, all + * objects are stored in intrusive container and allocated in simple node + * allocator (allocator that doesn't allow to allocate multiple object at once) + * + */ +class orphan_container { + // malloc part + // map alloc adress => orphan_malloc + using orphan_malloc_address_set = boost::intrusive::set< + orphan_malloc, + boost::intrusive::key_of_value>; + + // map time alloc => orphan_malloc + using orphan_malloc_time_hook = + boost::intrusive::member_hook, + &orphan_malloc::allocated_time_hook>; + using orphan_malloc_time_set = boost::intrusive::multiset< + orphan_malloc, + orphan_malloc_time_hook, + boost::intrusive::key_of_value>; + + // node allocator used to create orphan_malloc + using orphan_malloc_allocator = com::centreon::common:: + node_allocator, 0x100000>; + + orphan_malloc_address_set _address_to_malloc; + orphan_malloc_time_set _time_to_malloc; + orphan_malloc_allocator _malloc_allocator; + + // free part + // orphan_free are stored in single linked list + using orphan_free_list = + boost::intrusive::slist>; + + // node allocator used to create orphan_free + using orphan_free_allocator = com::centreon::common:: + node_allocator, 0x100000>; + + orphan_free_list _free; + orphan_free_allocator _free_allocator; + + funct_cache_map _funct_info_cache; + + std::chrono::system_clock::duration _malloc_peremption; + std::chrono::system_clock::time_point _last_flush; + size_t _max_file_size; + std::string_view _out_file_path; + + mutable std::mutex _protect; + + int open_file(); + + public: + orphan_container(); + + void add_malloc(const void* addr, + size_t allocated_size, + pid_t thread_id, + const std::string_view& funct_name, + const boost::stacktrace::stacktrace& backtrace, + size_t backtrace_offset); + + bool free(const void* addr); + + void add_free(const void* addr, + pid_t thread_id, + const std::string_view& funct_name, + const boost::stacktrace::stacktrace& backtrace, + size_t backtrace_offset); + + void flush_to_file(); +}; + +} // namespace com::centreon::malloc_trace + +#endif diff --git a/malloc-trace/inc/com/centreon/malloc_trace/simply_allocator.hh b/malloc-trace/inc/com/centreon/malloc_trace/simply_allocator.hh new file mode 100644 index 00000000000..23daa26b958 --- /dev/null +++ b/malloc-trace/inc/com/centreon/malloc_trace/simply_allocator.hh @@ -0,0 +1,59 @@ +/** + * Copyright 2024 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ + +#ifndef CMT_SIMPLY_ALLOCATOR_HH +#define CMT_SIMPLY_ALLOCATOR_HH + +namespace com::centreon::malloc_trace { + +constexpr unsigned block_size = 4096; +constexpr unsigned nb_block = 256; +/** + * @brief basic allocator + * At the beginning, we don't know original malloc + * we must provide a simple malloc free for dlsym + * + */ +class simply_allocator { + class node_block { + unsigned char _buff[block_size]; + bool _free = true; + + public: + struct key_extractor { + using type = unsigned char const*; + type operator()(const node_block& block) const { return block._buff; } + }; + + bool is_free() const { return _free; } + void set_free(bool free) { _free = free; } + unsigned char* get_buff() { return _buff; } + }; + + node_block _blocks[nb_block]; + std::mutex _protect; + + public: + void* malloc(size_t size); + void* realloc(void* p, size_t size); + bool free(void* p); +}; + +} // namespace com::centreon::malloc_trace + +#endif diff --git a/malloc-trace/precomp_inc/precomp.hh b/malloc-trace/precomp_inc/precomp.hh new file mode 100644 index 00000000000..48f8663ed1a --- /dev/null +++ b/malloc-trace/precomp_inc/precomp.hh @@ -0,0 +1,34 @@ +/* +** Copyright 2024 Centreon +** +** This file is part of Centreon Engine. +** +** Centreon Engine is free software: you can redistribute it and/or +** modify it under the terms of the GNU General Public License version 2 +** as published by the Free Software Foundation. +** +** Centreon Engine is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +** General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with Centreon Engine. If not, see +** . +*/ + +#ifndef CMT_PRECOMP_HH +#define CMT_PRECOMP_HH + +#include +#include + +#include +#include +#include + +#include +#include +#include + +#endif diff --git a/malloc-trace/remove_malloc_free.py b/malloc-trace/remove_malloc_free.py new file mode 100755 index 00000000000..551aa4651e0 --- /dev/null +++ b/malloc-trace/remove_malloc_free.py @@ -0,0 +1,62 @@ +#!/usr/bin/python3 +# +# Copyright 2024 Centreon +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For more information : contact@centreon.com +# +# This script takes a file produced by centreon-malloc-trace library and remove all malloc free pairs + +import sys +import getopt +import csv + + + + +def main(argv): + csv.field_size_limit(sys.maxsize) + inputfile = '' + outputfile = '' + opts, args = getopt.getopt(argv,"hi:o:",["in_file=","out_file="]) + for opt, arg in opts: + if opt == '-h': + print ('remove_malloc_free.py -i -o ') + return + elif opt in ("-i", "--in_file"): + inputfile = arg + elif opt in ("-o", "--out_file"): + outputfile = arg + + if inputfile == '' or outputfile == '': + print ('remove_malloc_free.py -i -o ') + return + allocated = {} + with open(inputfile) as csv_file: + csv_reader = csv.reader(csv_file, delimiter=';') + for row in csv_reader: + if len(row) > 2 and row[2].isdigit(): + if row[0].find('free') >= 0: + if (row[2] in allocated): + allocated.pop(row[2]) + else: + allocated[row[2]] = row + with open(outputfile, 'w') as f: + for row in allocated.values(): + f.write(';'.join(row) ) + f.write('\n') + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/malloc-trace/src/by_thread_trace_active.cc b/malloc-trace/src/by_thread_trace_active.cc new file mode 100644 index 00000000000..91fbc934f8a --- /dev/null +++ b/malloc-trace/src/by_thread_trace_active.cc @@ -0,0 +1,57 @@ +/** + * Copyright 2024 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ + +#include "by_thread_trace_active.hh" + +using namespace com::centreon::malloc_trace; + +/** + * @brief Set the flag to true in _by_thread_dump_active + * + * @return true the flag was not setted before call + * @return false the flag was yet setted before call + */ +bool thread_dump_active::set_dump_active(pid_t thread_id) { + std::lock_guard l(_protect); + if (!is_initialized()) { + return false; + } + const thread_trace_active* exist = find(thread_id); + + if (!exist) { + const thread_trace_active* inserted = insert_and_get(thread_id); + if (!inserted) { + return false; + } + inserted->set_malloc_trace_active(); + return true; + } else { + return !exist->set_malloc_trace_active(); + } +} + +void thread_dump_active::reset_dump_active(pid_t thread_id) { + std::lock_guard l(_protect); + if (!is_initialized()) { + return; + } + const thread_trace_active* exist = find(thread_id); + if (exist) { + exist->reset_malloc_trace_active(); + } +} diff --git a/malloc-trace/src/malloc_trace.cc b/malloc-trace/src/malloc_trace.cc new file mode 100644 index 00000000000..3a712d76d8e --- /dev/null +++ b/malloc-trace/src/malloc_trace.cc @@ -0,0 +1,242 @@ +/** + * Copyright 2024 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ + +#include +#include + +#include "by_thread_trace_active.hh" +#include "funct_info_cache.hh" +#include "orphan_container.hh" +#include "simply_allocator.hh" + +using namespace com::centreon::malloc_trace; + +extern void* __libc_malloc(size_t size); + +pid_t gettid() __attribute__((weak)); + +/** + * @brief gettid is not available on alma8 + * + * @return pid_t + */ +pid_t m_gettid() { + if (gettid) { + return gettid(); + } else { + return syscall(__NR_gettid); + } +} + +/** + * @brief when we enter in malloc or free, we store information of stack trace, + * this will generate allocation that we don't want to store, this object allow + * us to store the active tracing on the active thread in order to avoid store + * recursion. + * + */ +static thread_dump_active _thread_dump_active; + +/** + * @brief simply allocator used by dlsym + * + */ +static simply_allocator _first_malloc; + +/** + * @brief the container that store every malloc and free + * + */ +static orphan_container* _orphans = new orphan_container; + +static void* first_malloc(size_t size) { + return _first_malloc.malloc(size); +} + +static void* first_realloc(void* p, size_t size) { + return _first_malloc.realloc(p, size); +} + +typedef void* (*malloc_signature)(size_t); + +// will be filled by original malloc +static malloc_signature original_malloc = first_malloc; + +typedef void* (*realloc_signature)(void*, size_t); + +// will be filled by original realloc +static realloc_signature original_realloc = first_realloc; + +static void first_free(void* p) { + _first_malloc.free(p); +} + +typedef void (*free_signature)(void*); +// will be filled by original free +static free_signature original_free = first_free; + +/** + * @brief there is 3 stages + * on the first alloc, we don't know malloc, realloc and free address + * So we call dlsym to get these address + * As dlsym allocates memory, we are in dlsym_running state and we provide + * allocation mechanism by simply_allocator. + * Once dlsym are done, we are in hook state + * + */ +enum class e_library_state { not_hooked, dlsym_running, hooked }; +static e_library_state _state = e_library_state::not_hooked; + +static void search_symbols() { + original_malloc = + reinterpret_cast(dlsym(RTLD_NEXT, "malloc")); + original_free = reinterpret_cast(dlsym(RTLD_NEXT, "free")); + original_realloc = + reinterpret_cast(dlsym(RTLD_NEXT, "realloc")); +} + +/** + * @brief our malloc + * + * @param size + * @param funct_name function name logged + * @return void* + */ +static void* malloc(size_t size, const char* funct_name) { + switch (_state) { + case e_library_state::not_hooked: + _state = e_library_state::dlsym_running; + search_symbols(); + _state = e_library_state::hooked; + break; + case e_library_state::dlsym_running: + return first_malloc(size); + default: + break; + } + + void* p = original_malloc(size); + + pid_t thread_id = m_gettid(); + bool have_to_dump = _thread_dump_active.set_dump_active(thread_id); + + // if this thread is not yet dumping => store it + if (have_to_dump) { + if (_orphans) { + constexpr std::string_view _funct_name("malloc"); + _orphans->add_malloc(p, size, thread_id, _funct_name, + boost::stacktrace::stacktrace(), 2); + _orphans->flush_to_file(); + } + _thread_dump_active.reset_dump_active(thread_id); + } + return p; +} + +/** + * @brief our realloc function + * + * @param p + * @param size + * @return void* + */ +void* realloc(void* p, size_t size) { + switch (_state) { + case e_library_state::not_hooked: + _state = e_library_state::dlsym_running; + search_symbols(); + _state = e_library_state::hooked; + break; + case e_library_state::dlsym_running: + return first_realloc(p, size); + default: + break; + } + void* new_p = original_realloc(p, size); + pid_t thread_id = m_gettid(); + bool have_to_dump = _thread_dump_active.set_dump_active(thread_id); + // if this thread is not yet dumping => call dump_callstack + if (have_to_dump) { + constexpr std::string_view realloc_funct_name("realloc"); + // if pointer has changed, we record a free + if (new_p != p && p) { + if (!_orphans->free(p)) { + constexpr std::string_view free_funct_name("freerealloc"); + _orphans->add_free(p, thread_id, free_funct_name, + boost::stacktrace::stacktrace(), 2); + } + } + _orphans->add_malloc(new_p, size, thread_id, realloc_funct_name, + boost::stacktrace::stacktrace(), 2); + _orphans->flush_to_file(); + _thread_dump_active.reset_dump_active(thread_id); + } + return new_p; +} + +/** + * @brief replacement of the original malloc + * + * @param size + * @return void* + */ +void* malloc(size_t size) { + return malloc(size, "malloc"); +} + +/** + * @brief our calloc function + * call to malloc + * + * @param num + * @param size + * @return void* + */ +void* calloc(size_t num, size_t size) { + size_t total_size = num * size; + void* p = malloc(total_size, "calloc"); + memset(p, 0, total_size); + return p; +} + +/** + * @brief our free + * + * @param p + */ +void free(void* p) { + if (_first_malloc.free(p)) + return; + original_free(p); + if (!p) + return; + + pid_t thread_id = m_gettid(); + bool have_to_dump = _thread_dump_active.set_dump_active(thread_id); + + // if this thread is not yet dumping => call dump_callstack + if (have_to_dump) { + if (!_orphans->free(p)) { + constexpr std::string_view free_funct_name("free"); + _orphans->add_free(p, thread_id, free_funct_name, + boost::stacktrace::stacktrace(), 2); + _orphans->flush_to_file(); + } + _thread_dump_active.reset_dump_active(thread_id); + } +} \ No newline at end of file diff --git a/malloc-trace/src/orphan_container.cc b/malloc-trace/src/orphan_container.cc new file mode 100644 index 00000000000..45f7fce6b10 --- /dev/null +++ b/malloc-trace/src/orphan_container.cc @@ -0,0 +1,304 @@ +/** + * Copyright 2024 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ + +#include +#include + +#include "orphan_container.hh" + +using namespace com::centreon::malloc_trace; + +/** + * @brief Construct a new backtrace info::backtrace info object + * + * @param allocated address allocated or freed + * @param allocated_size size allocated + * @param thread_id + * @param funct_name malloc, free or freerealloc + * @param backtrace + * @param backtrace_offset we will ignore backtrace_offset first frames + */ +backtrace_info::backtrace_info(const void* allocated, + size_t allocated_size, + pid_t thread_id, + const std::string_view& funct_name, + const boost::stacktrace::stacktrace& backtrace, + size_t backtrace_offset) + : _allocated(allocated), + _allocated_size(allocated_size), + _thread_id(thread_id), + _funct_name(funct_name), + _backtrace_size(0), + _last_allocated(std::chrono::system_clock::now()) { + if (backtrace_offset >= backtrace.size()) { + return; + } + + boost::stacktrace::stacktrace::const_iterator frame_iter = backtrace.begin(); + std::advance(frame_iter, backtrace_offset); + for (; frame_iter != backtrace.end() && _backtrace_size < max_backtrace_size; + ++frame_iter) { + _backtrace[_backtrace_size++] = frame_iter->address(); + } +} + +/** + * @brief add a line in output file + * + * @param fd file descriptor + * @param funct_info_cache map address=>function info where we store infos of + * functions (name, source line) + */ +void backtrace_info::to_file(int fd, funct_cache_map& funct_info_cache) const { + if (fd <= 0) { + return; + } + constexpr unsigned size_buff = 0x40000; + char buff[size_buff]; + char* end_buff = buff + size_buff - 10; + *end_buff = 0; + + char* work_pos = + fmt::format_to_n(buff, size_buff, "\"{}\";{};{};{};{};\"[", _funct_name, + _thread_id, reinterpret_cast(_allocated), + _allocated_size, + std::chrono::duration_cast( + _last_allocated.time_since_epoch()) + .count()) + .out; + + for (unsigned stack_cpt = 0; stack_cpt < _backtrace_size; ++stack_cpt) { + const boost::stacktrace::frame::native_frame_ptr_t addr = + _backtrace[stack_cpt]; + boost::stacktrace::frame frame(addr); + if (end_buff - work_pos < 1000) { + break; + } + + if (stack_cpt) { + *work_pos++ = ','; + } + funct_cache_map::const_iterator cache_entry = funct_info_cache.find(addr); + if (cache_entry == funct_info_cache.end()) { // not found => search and + // save + funct_info to_insert(frame.name(), frame.source_file(), + frame.source_line()); + cache_entry = funct_info_cache.emplace(addr, to_insert).first; + } + + if (cache_entry->second.get_source_file().empty()) { + work_pos = fmt::format_to_n(work_pos, end_buff - work_pos, + "{{\\\"f\\\":\\\"{}\\\" }}", + cache_entry->second.get_funct_name()) + .out; + } else { + work_pos = + fmt::format_to_n( + work_pos, end_buff - work_pos, + "{{\\\"f\\\":\\\"{}\\\" , \\\"s\\\":\\\"{}\\\" , \\\"l\\\":{}}}", + cache_entry->second.get_funct_name(), + cache_entry->second.get_source_file(), + cache_entry->second.get_source_line()) + .out; + } + } + + *work_pos++ = ']'; + *work_pos++ = '"'; + *work_pos++ = '\n'; + + ::write(fd, buff, work_pos - buff); +} + +/********************************************************************************* + orphan_container +*********************************************************************************/ + +/** + * @brief Construct a new orphan container::orphan container object + * + */ +orphan_container::orphan_container() + : _malloc_allocator(std::allocator()), + _free_allocator(std::allocator()) { + char* env_out_file_max_size = getenv("out_file_max_size"); + if (env_out_file_max_size && atoll(env_out_file_max_size) > 0) + _max_file_size = atoll(env_out_file_max_size); + else + _max_file_size = 0x100000000; + + char* env_out_file_path = getenv("out_file_path"); + if (env_out_file_path && strlen(env_out_file_path) > 0) + _out_file_path = env_out_file_path; + + else + _out_file_path = "/tmp/malloc-trace.csv"; + + char* malloc_second_peremption = getenv("malloc_second_peremption"); + if (malloc_second_peremption && atoi(malloc_second_peremption) > 0) + _malloc_peremption = std::chrono::seconds(atoi(malloc_second_peremption)); + else + _malloc_peremption = std::chrono::minutes(1); +} + +/** + * @brief register a malloc action, it can be a malloc or a realloc + * + * @param addr address allocated + * @param allocated_size size allocated + * @param thread_id + * @param funct_name + * @param backtrace + * @param backtrace_offset we will ignore backtrace_offset first frames + */ +void orphan_container::add_malloc( + const void* addr, + size_t allocated_size, + pid_t thread_id, + const std::string_view& funct_name, + const boost::stacktrace::stacktrace& backtrace, + size_t backtrace_offset) { + std::lock_guard l(_protect); + orphan_malloc* new_node = _malloc_allocator.allocate(); + new (new_node) orphan_malloc(addr, allocated_size, thread_id, funct_name, + backtrace, backtrace_offset); + + if (!_address_to_malloc.insert(*new_node).second) { + _malloc_allocator.deallocate(new_node); + } else { + _time_to_malloc.insert(*new_node); + } +} + +/** + * @brief when program call free we try to unregister previous malloc at addr + * + * @param addr address to free + * @return true malloc was found and unregistered + * @return false no malloc found for this address + */ +bool orphan_container::free(const void* addr) { + std::lock_guard l(_protect); + auto found = _address_to_malloc.find(addr); + if (found != _address_to_malloc.end()) { + orphan_malloc& to_erase = *found; + + //_time_to_malloc is only indexed by alloc timestamp (ms) + auto where_to_search = + _time_to_malloc.equal_range(to_erase.get_last_allocated()); + for (; where_to_search.first != where_to_search.second; + ++where_to_search.first) { + if (&*where_to_search.first == &to_erase) { + _time_to_malloc.erase(where_to_search.first); + break; + } + } + + _address_to_malloc.erase(found); + _malloc_allocator.deallocate(&to_erase); + return true; + } else { + return false; + } +} + +/** + * @brief in case or free has returned false, we have to add free orphan in this + * container + * + * @param addr address freed + * @param thread_id + * @param funct_name + * @param backtrace + * @param backtrace_offset we will ignore backtrace_offset first frames + */ +void orphan_container::add_free(const void* addr, + pid_t thread_id, + const std::string_view& funct_name, + const boost::stacktrace::stacktrace& backtrace, + size_t backtrace_offset) { + std::lock_guard l(_protect); + orphan_free* new_free = _free_allocator.allocate(); + new (new_free) + orphan_free(addr, thread_id, funct_name, backtrace, backtrace_offset); + _free.push_back(*new_free); +} + +/** + * @brief flush contents to disk + * all malloc older than _malloc_peremption are flushed + * more recent mallocs are not flushed because we hope a free that will + * unregister its all orphan free are flushed all data flushed are remove from + * container + * + */ +void orphan_container::flush_to_file() { + std::lock_guard l(_protect); + std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + if (_last_flush + _malloc_peremption > now) { + return; + } + + int fd = open_file(); + + _last_flush = now; + + // we flush to disk oldest malloc and remove its from the container + if (!_time_to_malloc.empty()) { + orphan_malloc_time_set::iterator upper = _time_to_malloc.upper_bound( + std::chrono::system_clock::now() - _malloc_peremption); + for (orphan_malloc_time_set::iterator to_flush = _time_to_malloc.begin(); + to_flush != upper; ++to_flush) { + to_flush->to_file(fd, _funct_info_cache); + _address_to_malloc.erase(to_flush->get_allocated()); + } + _time_to_malloc.erase_and_dispose( + _time_to_malloc.begin(), upper, [this](orphan_malloc* to_dispose) { + _malloc_allocator.deallocate(to_dispose); + }); + } + + // we flush all free + for (const orphan_free& to_flush : _free) { + to_flush.to_file(fd, _funct_info_cache); + } + _free.clear_and_dispose([this](orphan_free* to_dispose) { + _free_allocator.deallocate(to_dispose); + }); + ::close(fd); +} + +/** + * @brief this function open out file if it hasn't be done + * if file size exceed 256Mo, file is truncated + * + * @return int file descriptor + */ +int orphan_container::open_file() { + int out_file_fd = open(_out_file_path.data(), O_APPEND | O_CREAT | O_RDWR, + S_IRUSR | S_IWUSR); + if (out_file_fd > 0) { + struct stat file_stat; + if (!fstat(out_file_fd, &file_stat)) { + if (file_stat.st_size > _max_file_size) { + ftruncate(out_file_fd, 0); + } + } + } + return out_file_fd; +} diff --git a/malloc-trace/src/simply_allocator.cc b/malloc-trace/src/simply_allocator.cc new file mode 100644 index 00000000000..84b6e4c7841 --- /dev/null +++ b/malloc-trace/src/simply_allocator.cc @@ -0,0 +1,78 @@ +/** + * Copyright 2024 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ + +#include "simply_allocator.hh" + +using namespace com::centreon::malloc_trace; + +/** + * @brief same as malloc + * + * @param size + * @return void* + */ +void* simply_allocator::malloc(size_t size) { + if (size > block_size) { + return nullptr; + } + std::lock_guard l(_protect); + for (node_block* search = _blocks; search != _blocks + nb_block; ++search) { + if (search->is_free()) { + search->set_free(false); + return search->get_buff(); + } + } + return nullptr; +} + +/** + * @brief reallocate a pointer, + * if size > block_size or p doesn't belong to simply_allocator, it returns + * nullptr + * + * @param p + * @param size + * @return void* + */ +void* simply_allocator::realloc(void* p, size_t size) { + if (p < _blocks || p >= _blocks + block_size) + return nullptr; + if (size > block_size) { + return nullptr; + } + return p; +} + +/** + * @brief same as free + * + * @param p + * @return true if the pointer belong to this allocator + */ +bool simply_allocator::free(void* p) { + if (p < _blocks || p >= _blocks + block_size) + return false; + std::lock_guard l(_protect); + for (node_block* search = _blocks; search != _blocks + nb_block; ++search) { + if (search->get_buff() == p) { + search->set_free(true); + return true; + } + } + return false; +} From 79fe11123b08d4e70e2bb082f3f85b015dd824eb Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Thu, 18 Jan 2024 11:46:20 +0100 Subject: [PATCH 32/64] attempt to remove pbuilder --- .github/docker/Dockerfile.centreon-collect-debian-bookworm | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bookworm b/.github/docker/Dockerfile.centreon-collect-debian-bookworm index 1cb8f9b951a..c23fcef80bc 100644 --- a/.github/docker/Dockerfile.centreon-collect-debian-bookworm +++ b/.github/docker/Dockerfile.centreon-collect-debian-bookworm @@ -6,8 +6,6 @@ RUN < Date: Thu, 18 Jan 2024 14:19:34 +0100 Subject: [PATCH 33/64] Update .github/workflows/centreon-collect.yml Co-authored-by: Kevin Duret --- .github/workflows/centreon-collect.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/centreon-collect.yml b/.github/workflows/centreon-collect.yml index cb693924714..d785d3de598 100644 --- a/.github/workflows/centreon-collect.yml +++ b/.github/workflows/centreon-collect.yml @@ -152,6 +152,7 @@ jobs: arch: arm64 - distrib: bookworm arch: amd64 + name: deliver ${{ matrix.distrib }} steps: From c2f12a7492e8cb704e5e0a593bb9bd53f2be1313 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Thu, 18 Jan 2024 14:38:12 +0100 Subject: [PATCH 34/64] revert --- .github/docker/Dockerfile.centreon-collect-debian-bookworm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bookworm b/.github/docker/Dockerfile.centreon-collect-debian-bookworm index c23fcef80bc..1cb8f9b951a 100644 --- a/.github/docker/Dockerfile.centreon-collect-debian-bookworm +++ b/.github/docker/Dockerfile.centreon-collect-debian-bookworm @@ -6,6 +6,8 @@ RUN < Date: Thu, 18 Jan 2024 16:07:11 +0100 Subject: [PATCH 35/64] removed debian tooling --- .../docker/Dockerfile.centreon-collect-debian-bookworm | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bookworm b/.github/docker/Dockerfile.centreon-collect-debian-bookworm index 1cb8f9b951a..4a7f9907576 100644 --- a/.github/docker/Dockerfile.centreon-collect-debian-bookworm +++ b/.github/docker/Dockerfile.centreon-collect-debian-bookworm @@ -6,8 +6,6 @@ RUN < Date: Fri, 19 Jan 2024 11:51:49 +0100 Subject: [PATCH 36/64] added git for conan install command on dockerfile --- .github/docker/Dockerfile.centreon-collect-debian-bookworm | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bookworm b/.github/docker/Dockerfile.centreon-collect-debian-bookworm index 4a7f9907576..d0d88aeab7b 100644 --- a/.github/docker/Dockerfile.centreon-collect-debian-bookworm +++ b/.github/docker/Dockerfile.centreon-collect-debian-bookworm @@ -7,6 +7,7 @@ RUN < Date: Fri, 19 Jan 2024 13:34:04 +0100 Subject: [PATCH 37/64] removed packaging dependencies on debian dockerfile that are redundant with nfpm packaging --- .github/docker/Dockerfile.centreon-collect-debian-bullseye | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bullseye b/.github/docker/Dockerfile.centreon-collect-debian-bullseye index 1ae20485815..8f88ab94a65 100644 --- a/.github/docker/Dockerfile.centreon-collect-debian-bullseye +++ b/.github/docker/Dockerfile.centreon-collect-debian-bullseye @@ -7,6 +7,7 @@ RUN < Date: Fri, 19 Jan 2024 13:34:22 +0100 Subject: [PATCH 38/64] removed packaging dependencies on debian dockerfile that are redundant with nfpm packaging --- .github/docker/Dockerfile.centreon-collect-debian-bullseye | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bullseye b/.github/docker/Dockerfile.centreon-collect-debian-bullseye index 8f88ab94a65..67f3e397e0d 100644 --- a/.github/docker/Dockerfile.centreon-collect-debian-bullseye +++ b/.github/docker/Dockerfile.centreon-collect-debian-bullseye @@ -25,7 +25,6 @@ apt-get -y install cmake \ libperl-dev \ libgcrypt20-dev \ aptitude \ - pbuilder \ git-buildpackage \ fakeroot \ strace \ From 3c69e279c276ab3c421f11331db5a2d566937522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Chapron?= <34628915+sc979@users.noreply.github.com> Date: Fri, 26 Jan 2024 10:16:44 +0100 Subject: [PATCH 39/64] fix(ci): fix breaking changes on gha upload-artifact upgrade (#1025) * chore(deps): bump actions/upload-artifact from 3.1.3 to 4.0.0 (#1020) * chore(deps): bump actions/download-artifact from 3.0.2 to 4.1.0 (#1021) * refactor(ci): adapt robot-test with upgraded gha * fix * fix: yaml-lint error * force test to fail * upload the entire directory of reports * force multiple tests to fail * cleaning --------- Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: a-eljazouly --- .github/workflows/package-collect.yml | 2 +- .github/workflows/robot-test.yml | 57 +++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/.github/workflows/package-collect.yml b/.github/workflows/package-collect.yml index c3b7a82f152..1950b0c415a 100644 --- a/.github/workflows/package-collect.yml +++ b/.github/workflows/package-collect.yml @@ -175,7 +175,7 @@ jobs: # set condition to true if artifacts are needed - if: ${{ false }} name: Upload package artifacts - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 with: name: packages-${{ matrix.distrib }}-${{ matrix.arch }} path: ./*.${{ matrix.package_extension}} diff --git a/.github/workflows/robot-test.yml b/.github/workflows/robot-test.yml index 27d5bebf90a..9a28d5d20b1 100644 --- a/.github/workflows/robot-test.yml +++ b/.github/workflows/robot-test.yml @@ -157,11 +157,20 @@ jobs: cp tests/output.xml reports/$FILE_PREFIX-output.xml fi + - name: Replace / with - in the feature path + id: feature-path + if: always() + run: | + feature_name="${{ matrix.feature }}" + feature_name_with_dash="${feature_name//\//-}" + echo "Modified Feature Name: $feature_name_with_dash" + echo "feature_name_with_dash=$feature_name_with_dash" >> $GITHUB_OUTPUT + - name: Upload Test Results if: ${{ failure() }} - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 with: - name: reports-${{inputs.test_group_name}} + name: reports-${{inputs.test_group_name}}-${{ steps.feature-path.outputs.feature_name_with_dash }} path: reports retention-days: 1 @@ -174,10 +183,50 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Download Artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0 with: - name: reports-${{inputs.test_group_name}} + pattern: reports-${{inputs.test_group_name}}-* path: reports + merge-multiple: true + + - name: Upload the regrouped artifact + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + with: + name: reports-${{inputs.test_group_name}} + path: reports/ + retention-days: 1 + + - name: Delete separated artifacts + run: | + artifact_pattern="reports-${{inputs.test_group_name}}-" + TOKEN=${{ secrets.GITHUB_TOKEN }} + artifact_exist=true + while [ "$artifact_exist" = true ] ;do + artifact_exist=false + artifacts_response=$(curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${{ github.repository }}/actions/artifacts?per_page=100") + artifacts=$(echo $artifacts_response | jq -c '.artifacts[]') + echo "Those are the artifacts : $artifacts" + while read row; do + artifact_name=$(echo "$row" | jq -r '.name') + if [[ "$artifact_name" =~ ^.*"$artifact_pattern".* ]]; then + artifact_exist=true + echo "Deleting : $artifact_name" + artifact_id=$(echo "$row" | jq -r '.id') + curl -L \ + -X DELETE \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${{ github.repository }}/actions/artifacts/${artifact_id}" + fi + done <<< "$artifacts" + done + echo "End of Deleting" + shell: bash - uses: actions/setup-python@v4 with: From d243131cf3018204167025d5f82d1f9f6e21dd45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jan 2024 09:18:13 +0000 Subject: [PATCH 40/64] chore(deps): bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/robot-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/robot-test.yml b/.github/workflows/robot-test.yml index 9a28d5d20b1..26ff7d59699 100644 --- a/.github/workflows/robot-test.yml +++ b/.github/workflows/robot-test.yml @@ -228,7 +228,7 @@ jobs: echo "End of Deleting" shell: bash - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' From e503e10a00201fabfc85fdc2695140634415e82b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 10:53:21 +0000 Subject: [PATCH 41/64] chore(deps): bump actions/download-artifact from 3.0.2 to 4.1.1 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3.0.2 to 4.1.1. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/9bc31d5ccc31df68ecc42ccf4149144866c47d8a...6b208ae046db98c579e8a3aa621ab581ff575935) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/robot-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/robot-test.yml b/.github/workflows/robot-test.yml index 9a28d5d20b1..8023d420d70 100644 --- a/.github/workflows/robot-test.yml +++ b/.github/workflows/robot-test.yml @@ -183,7 +183,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Download Artifacts - uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0 + uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 with: pattern: reports-${{inputs.test_group_name}}-* path: reports From dcb6913d648d77e1e9270a647feee359c9a83977 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 09:37:10 +0000 Subject: [PATCH 42/64] chore(deps): bump actions/cache from 3.3.2 to 4.0.0 Bumps [actions/cache](https://github.com/actions/cache) from 3.3.2 to 4.0.0. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/704facf57e6136b1bc63b828d79edcd491f0ee84...13aacd865c20de90d75de3b17ebe84f7a17d57d2) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/robot-test.yml | 6 +++--- .github/workflows/veracode-analysis.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/robot-test.yml b/.github/workflows/robot-test.yml index 8023d420d70..af5542d1bb0 100644 --- a/.github/workflows/robot-test.yml +++ b/.github/workflows/robot-test.yml @@ -74,7 +74,7 @@ jobs: shell: bash - name: image to cache - uses: actions/cache/save@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + uses: actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: /tmp/${{inputs.image}} key: ${{inputs.image_test}} @@ -113,14 +113,14 @@ jobs: fetch-depth: 0 - name: Restore image - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: /tmp/${{inputs.image}} key: ${{inputs.image_test}} fail-on-cache-miss: true - name: Restore packages - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: ${{ inputs.package_cache_path }} key: ${{ inputs.package_cache_key }} diff --git a/.github/workflows/veracode-analysis.yml b/.github/workflows/veracode-analysis.yml index 43952f3436c..417cb381dba 100644 --- a/.github/workflows/veracode-analysis.yml +++ b/.github/workflows/veracode-analysis.yml @@ -110,7 +110,7 @@ jobs: tar cvzf "${{ inputs.module_name }}-${{ github.sha }}-${{ github.run_id }}-veracode-binary.tar.gz" build - name: Cache - uses: actions/cache/save@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + uses: actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: "${{ inputs.module_name }}-${{ github.sha }}-${{ github.run_id }}-veracode-binary.tar.gz" key: "${{ inputs.module_name }}-${{ github.sha }}-${{ github.run_id }}-veracode-binary" @@ -138,7 +138,7 @@ jobs: delete-on-promote: false - name: Get build binary - uses: actions/cache/restore@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: "${{ inputs.module_name }}-${{ github.sha }}-${{ github.run_id }}-veracode-binary.tar.gz" key: "${{ inputs.module_name }}-${{ github.sha }}-${{ github.run_id }}-veracode-binary" From 3d4296e8aecc7f7c31906ceba03a7cf2f5033df2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 09:37:19 +0000 Subject: [PATCH 43/64] chore(deps): bump actions/upload-artifact from 3.1.3 to 4.2.0 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.3 to 4.2.0. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/a8a3f3ad30e3422c9c7b888a15615d19a852ae32...694cdabd8bdb0f10b2cea11669e1bf5453eed0a6) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/package-collect.yml | 2 +- .github/workflows/robot-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package-collect.yml b/.github/workflows/package-collect.yml index 1950b0c415a..e9c0fc576d5 100644 --- a/.github/workflows/package-collect.yml +++ b/.github/workflows/package-collect.yml @@ -175,7 +175,7 @@ jobs: # set condition to true if artifacts are needed - if: ${{ false }} name: Upload package artifacts - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0 with: name: packages-${{ matrix.distrib }}-${{ matrix.arch }} path: ./*.${{ matrix.package_extension}} diff --git a/.github/workflows/robot-test.yml b/.github/workflows/robot-test.yml index af5542d1bb0..2952beb71a2 100644 --- a/.github/workflows/robot-test.yml +++ b/.github/workflows/robot-test.yml @@ -168,7 +168,7 @@ jobs: - name: Upload Test Results if: ${{ failure() }} - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0 with: name: reports-${{inputs.test_group_name}}-${{ steps.feature-path.outputs.feature_name_with_dash }} path: reports From cc3187cfba2250df7e4902a2aa4f8837af238597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Chapron?= <34628915+sc979@users.noreply.github.com> Date: Fri, 26 Jan 2024 10:40:03 +0100 Subject: [PATCH 44/64] fix(ci): skip sandbox analysis on PR jobs (#1051) --- .github/workflows/veracode-analysis.yml | 32 ++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/.github/workflows/veracode-analysis.yml b/.github/workflows/veracode-analysis.yml index 43952f3436c..91b8f9f13cb 100644 --- a/.github/workflows/veracode-analysis.yml +++ b/.github/workflows/veracode-analysis.yml @@ -27,9 +27,33 @@ on: required: true jobs: + routing: + name: Check before analysis + runs-on: ubuntu-latest + outputs: + development_stage: ${{ steps.routing-mode.outputs.development_stage }} + + steps: + - name: Set routing mode + id: routing-mode + run: | + DEVELOPMENT_STAGE="Development" + + ALLOWED_BRANCHES=(develop master dev-${{ inputs.major_version }}.x ${{ inputs.major_version }}.x) + for BRANCH in "${ALLOWED_BRANCHES[@]}"; do + if [[ "${{ github.ref_name }}" == "$BRANCH" ]] && [[ '${{ github.event_name }}' != 'pull_request' ]]; then + DEVELOPMENT_STAGE="Release" + fi + done + + echo "development_stage=$DEVELOPMENT_STAGE" >> $GITHUB_OUTPUT + cat $GITHUB_OUTPUT + build: name: Binary preparation runs-on: [self-hosted, collect] + needs: [routing] + if: needs.routing.outputs.development_stage != 'Development' container: image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/centreon-collect-alma9:${{ inputs.img_version }} credentials: @@ -115,13 +139,15 @@ jobs: path: "${{ inputs.module_name }}-${{ github.sha }}-${{ github.run_id }}-veracode-binary.tar.gz" key: "${{ inputs.module_name }}-${{ github.sha }}-${{ github.run_id }}-veracode-binary" - sandbox-scan: - needs: [build] + policy-scan: name: Sandbox scan + needs: [routing, build] + if: needs.routing.outputs.development_stage != 'Development' runs-on: ubuntu-latest steps: - name: Promote latest scan + # only develop will be promoted to policy scan if: github.ref_name == 'develop' env: VERACODE_API_ID: "${{ secrets.veracode_api_id }}" @@ -129,7 +155,7 @@ jobs: # Action forked as API calls hardcoded '.com' route uses: sc979/veracode-sandboxes-helper@cf67241c27cbe6405ad8705111121ece9a48c4ff # v0.2 - # Promote should not fail if sandbox was not found. + # Promote should not fail to trigger following sandbox scan. continue-on-error: true with: activity: "promote-latest-scan" From 717e531519aa91c01cafd8f43480ce9e8a5e1fe1 Mon Sep 17 00:00:00 2001 From: jean-christophe81 <98889244+jean-christophe81@users.noreply.github.com> Date: Fri, 26 Jan 2024 10:46:12 +0100 Subject: [PATCH 45/64] MON-23624 rrd files have group rw permissions (#978) REFS:MON-23624 --- broker/CMakeLists.txt | 3 ++- broker/rrd/src/creator.cc | 26 +++++++++++++++++++++----- broker/test/CMakeLists.txt | 3 ++- tests/broker-engine/rrd.robot | 4 ++++ tests/resources/Common.py | 15 +++++++++++++++ 5 files changed, 44 insertions(+), 7 deletions(-) diff --git a/broker/CMakeLists.txt b/broker/CMakeLists.txt index 59b6a56a106..ca52139b57a 100644 --- a/broker/CMakeLists.txt +++ b/broker/CMakeLists.txt @@ -491,7 +491,8 @@ target_link_libraries( ${CONAN_LIBS_PROTOBUF} "-Wl,--no-whole-archive" CONAN_PKG::spdlog - CONAN_PKG::grpc) + CONAN_PKG::grpc + stdc++fs) # Centreon Broker Watchdog option(WITH_CBWD "Build centreon broker watchdog." ON) diff --git a/broker/rrd/src/creator.cc b/broker/rrd/src/creator.cc index 4656ca6ddec..231336df3e4 100644 --- a/broker/rrd/src/creator.cc +++ b/broker/rrd/src/creator.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include "bbdo/storage/metric.hh" #include "com/centreon/broker/log_v2.hh" @@ -45,7 +46,8 @@ using namespace com::centreon::broker::rrd; */ creator::creator(std::string const& tmpl_path, uint32_t cache_size) : _cache_size(cache_size), _tmpl_path(tmpl_path) { - log_v2::rrd()->debug( + SPDLOG_LOGGER_DEBUG( + log_v2::rrd(), "RRD: file creator will maintain at most {} templates in '{}'", _cache_size, _tmpl_path); } @@ -105,7 +107,8 @@ void creator::create(std::string const& filename, if (it != _fds.end() && it->first.is_length_step_type_equal(info) && it->first.from <= from) { _duplicate(filename, it->second); - log_v2::rrd()->debug("reuse {} for {}", it->second.path, filename); + SPDLOG_LOGGER_DEBUG(log_v2::rrd(), "reuse {} for {}", it->second.path, + filename); } // Not in the cache, but we have enough space in the cache. // Create new entry. @@ -263,15 +266,28 @@ void creator::_open(std::string const& filename, // Debug message. argv[argc] = nullptr; - log_v2::rrd()->debug("RRD: opening file '{}' ({}, {}, {}, step 1, from {})", - filename, argv[0], argv[1], - (argv[2] ? argv[2] : "(null)"), from); + SPDLOG_LOGGER_DEBUG( + log_v2::rrd(), "RRD: create file '{}' ({}, {}, {}, step 1, from {})", + filename, argv[0], argv[1], (argv[2] ? argv[2] : "(null)"), from); // Create RRD file. rrd_clear_error(); if (rrd_create_r(filename.c_str(), 1, from, argc, argv)) throw exceptions::open("RRD: could not create file '{}: {}", filename, rrd_get_error()); + + // by default rrd_create_r create rw-r----- files group write is mandatory + // for rrdcached + std::error_code err; + std::filesystem::permissions( + filename, + std::filesystem::perms::group_read | std::filesystem::perms::group_write, + std::filesystem::perm_options::add, err); + if (err) { + SPDLOG_LOGGER_ERROR(log_v2::rrd(), + "RRD: fail to add access rights (660) to {}: {}", + filename, err.message()); + } } /** diff --git a/broker/test/CMakeLists.txt b/broker/test/CMakeLists.txt index ceeaf93f23a..50afdaa5b6c 100644 --- a/broker/test/CMakeLists.txt +++ b/broker/test/CMakeLists.txt @@ -144,7 +144,8 @@ target_link_libraries( CONAN_PKG::gtest CONAN_PKG::mariadb-connector-c CONAN_PKG::openssl - CONAN_PKG::grpc) + CONAN_PKG::grpc + stdc++fs) add_dependencies( ut_broker diff --git a/tests/broker-engine/rrd.robot b/tests/broker-engine/rrd.robot index 87045b922c4..a033f584692 100644 --- a/tests/broker-engine/rrd.robot +++ b/tests/broker-engine/rrd.robot @@ -384,6 +384,9 @@ BRRDRMU1 Should Be True ... ${result} ... Data before RRD rebuild contain alternatively the metric ID and 0. The expected average is metric_id / 2. + # 48 = 60(octal) + ${result} Has File Permissions ${VarRoot}/lib/centreon/metrics/${m}.rrd 48 + Should Be True ${result} ${VarRoot}/lib/centreon/metrics/${m}.rrd has not RW group permission END FOR ${index_id} IN @{index} @@ -429,6 +432,7 @@ RRD1 ${result} Find In Log With Timeout ${rrdLog} ${start} ${content1} 45 Should Not Be True ${result} Database did not receive command to rebuild metrics + *** Keywords *** Test Clean Stop Engine diff --git a/tests/resources/Common.py b/tests/resources/Common.py index 1903c3b6914..7edb6b1fed7 100644 --- a/tests/resources/Common.py +++ b/tests/resources/Common.py @@ -1484,3 +1484,18 @@ def get_uid(): def set_uid(user_id: int): os.setuid(user_id) + + +def has_file_permissions(path: str, permission: int): + """! test if file has permission passed in parameter + it does a AND with permission parameter + @param path path of the file + @permission mask to test file permission + @return True if the file has the requested permissions + """ + stat_res= os.stat(path) + if stat_res is None: + logger.console(f"fail to get permission of {path}") + return False + masked = stat_res.st_mode & permission + return masked == permission From 3e8a113a486aa2724f81b7a5d548e0b9ad437cde Mon Sep 17 00:00:00 2001 From: tuntoja <58987095+tuntoja@users.noreply.github.com> Date: Fri, 26 Jan 2024 11:23:01 +0100 Subject: [PATCH 46/64] fix(promote): add github_base_ref to promote job inputs (#1080) * fix(promote): add github_base_ref to promote job inputs * fix missing empty line --- .github/actions/promote-to-stable/action.yml | 5 ++++- .github/workflows/centreon-collect.yml | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/actions/promote-to-stable/action.yml b/.github/actions/promote-to-stable/action.yml index f61b16d27dc..115ef3668ba 100644 --- a/.github/actions/promote-to-stable/action.yml +++ b/.github/actions/promote-to-stable/action.yml @@ -22,6 +22,9 @@ inputs: repository_name: description: "The repository name" required: true + github_base_ref: + description: "Release base ref" + required: true runs: using: "composite" @@ -41,7 +44,7 @@ runs: # Cloud specific promote # delivery by default to onprem, override to internal if base branch is master - if [[ $GITHUB_BASE_REF == "master" ]]; then + if [[ ${{ inputs.github_base_ref }} == "master" ]]; then ROOT_REPO_PATH="rpm-standard-internal" else ROOT_REPO_PATH="rpm-standard" diff --git a/.github/workflows/centreon-collect.yml b/.github/workflows/centreon-collect.yml index b339f222923..dbf155d607e 100644 --- a/.github/workflows/centreon-collect.yml +++ b/.github/workflows/centreon-collect.yml @@ -188,3 +188,4 @@ jobs: minor_version: ${{ needs.get-version.outputs.patch }} stability: ${{ needs.get-version.outputs.stability }} repository_name: standard + github_base_ref: ${{ github.base_ref }} From 926273d71b44b946a8b846ce0005e2e91aafb46d Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Fri, 26 Jan 2024 12:01:40 +0100 Subject: [PATCH 47/64] add condition on os to use one version of setup-python over another --- .github/workflows/robot-test.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/robot-test.yml b/.github/workflows/robot-test.yml index 8a6004f8927..c213b3736db 100644 --- a/.github/workflows/robot-test.yml +++ b/.github/workflows/robot-test.yml @@ -228,7 +228,15 @@ jobs: echo "End of Deleting" shell: bash - - uses: actions/setup-python@v5 + # setup-python v5.0.0 relies on node20 which is not supported by el7 distributions + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + if: ${{ inputs.distrib == 'el7'}} + with: + python-version: '3.10' + + + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + if: ${{ inputs.distrib != 'el7' }} with: python-version: '3.10' From 08ac3f40b6559814d8ff1e39b6cc43d59793dc2f Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Fri, 26 Jan 2024 12:09:24 +0100 Subject: [PATCH 48/64] added deleteme file in broker to trigger pipelines --- broker/DELETEME | 1 + 1 file changed, 1 insertion(+) create mode 100644 broker/DELETEME diff --git a/broker/DELETEME b/broker/DELETEME new file mode 100644 index 00000000000..8c800dab277 --- /dev/null +++ b/broker/DELETEME @@ -0,0 +1 @@ +DELETEME \ No newline at end of file From 6c2ae5459e3766f8a449e83a720c70976a3805cc Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Fri, 26 Jan 2024 14:10:58 +0100 Subject: [PATCH 49/64] deleted DELETEME --- broker/DELETEME | 1 - 1 file changed, 1 deletion(-) delete mode 100644 broker/DELETEME diff --git a/broker/DELETEME b/broker/DELETEME deleted file mode 100644 index 8c800dab277..00000000000 --- a/broker/DELETEME +++ /dev/null @@ -1 +0,0 @@ -DELETEME \ No newline at end of file From d1cfff6c07dc967d0fe1e09398966f655bafd09f Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Fri, 26 Jan 2024 14:24:21 +0100 Subject: [PATCH 50/64] removed leftover space --- .github/workflows/robot-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/robot-test.yml b/.github/workflows/robot-test.yml index c213b3736db..028df065e64 100644 --- a/.github/workflows/robot-test.yml +++ b/.github/workflows/robot-test.yml @@ -234,7 +234,6 @@ jobs: with: python-version: '3.10' - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 if: ${{ inputs.distrib != 'el7' }} with: From f4a5394e89b93d01b75aec7d6e9c72ac8ffb2422 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 09:58:11 +0000 Subject: [PATCH 51/64] chore(deps): bump actions/upload-artifact from 4.0.0 to 4.3.0 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.0.0 to 4.3.0. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v4...26f96dfa697d77e81fd5907df203aa23a56210a8) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/package-collect.yml | 2 +- .github/workflows/robot-test.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/package-collect.yml b/.github/workflows/package-collect.yml index 027a2041ef8..e0c219eb331 100644 --- a/.github/workflows/package-collect.yml +++ b/.github/workflows/package-collect.yml @@ -184,7 +184,7 @@ jobs: # set condition to true if artifacts are needed - if: ${{ false }} name: Upload package artifacts - uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0 + uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 with: name: packages-${{ matrix.distrib }}-${{ matrix.arch }} path: ./*.${{ matrix.package_extension}} diff --git a/.github/workflows/robot-test.yml b/.github/workflows/robot-test.yml index 028df065e64..e2c6c2d8d01 100644 --- a/.github/workflows/robot-test.yml +++ b/.github/workflows/robot-test.yml @@ -168,7 +168,7 @@ jobs: - name: Upload Test Results if: ${{ failure() }} - uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0 + uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 with: name: reports-${{inputs.test_group_name}}-${{ steps.feature-path.outputs.feature_name_with_dash }} path: reports @@ -190,7 +190,7 @@ jobs: merge-multiple: true - name: Upload the regrouped artifact - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 with: name: reports-${{inputs.test_group_name}} path: reports/ From 9ac7cc3789d83fddf1dc25925f113f228e2fe5c9 Mon Sep 17 00:00:00 2001 From: jean-christophe81 <98889244+jean-christophe81@users.noreply.github.com> Date: Mon, 29 Jan 2024 14:13:53 +0100 Subject: [PATCH 52/64] Mon 33334 cbd memory leak jc 2 (#1076) * add malloc_trace library * move some parts of malloc-trace code * flush only orphan malloc and free * MON-33334 malloc_trace realloc name fix, tls memory-leak fix * function passed to gnutls must never throw * generate package * no generate package REFS:MON-33334 --- CMakeLists.txt | 1 + broker/tcp/src/tcp_connection.cc | 32 +-- .../tls/inc/com/centreon/broker/tls/params.hh | 11 +- .../tls/inc/com/centreon/broker/tls/stream.hh | 14 +- broker/tls/src/acceptor.cc | 93 ++------- broker/tls/src/connector.cc | 93 ++------- broker/tls/src/internal.cc | 46 +++-- broker/tls/src/params.cc | 4 +- broker/tls/src/stream.cc | 191 ++++++++++++------ malloc-trace/CMakeLists.txt | 2 +- malloc-trace/README.md | 2 +- .../centreon/malloc_trace/intrusive_map.hh | 1 - .../centreon/malloc_trace/orphan_container.hh | 2 +- malloc-trace/src/malloc_trace.cc | 6 +- malloc-trace/src/orphan_container.cc | 2 +- 15 files changed, 232 insertions(+), 268 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e9cbd44ba88..4953ec636ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -196,6 +196,7 @@ if (WITH_MALLOC_TRACE) add_subdirectory(malloc-trace) endif() + add_custom_target(test-broker COMMAND tests/ut_broker) add_custom_target(test-engine COMMAND tests/ut_engine) add_custom_target(test-clib COMMAND tests/ut_clib) diff --git a/broker/tcp/src/tcp_connection.cc b/broker/tcp/src/tcp_connection.cc index d5048c442ec..e21e3fcea90 100644 --- a/broker/tcp/src/tcp_connection.cc +++ b/broker/tcp/src/tcp_connection.cc @@ -1,20 +1,20 @@ /** -* Copyright 2020-2021 Centreon -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* For more information : contact@centreon.com -*/ + * Copyright 2020-2021 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ #include "com/centreon/broker/tcp/tcp_connection.hh" #include "com/centreon/broker/exceptions/connection_closed.hh" diff --git a/broker/tls/inc/com/centreon/broker/tls/params.hh b/broker/tls/inc/com/centreon/broker/tls/params.hh index d409f6d8550..cb7135b7623 100644 --- a/broker/tls/inc/com/centreon/broker/tls/params.hh +++ b/broker/tls/inc/com/centreon/broker/tls/params.hh @@ -21,10 +21,8 @@ #include +namespace com::centreon::broker::tls { -namespace com::centreon::broker { - -namespace tls { /** * @class params params.hh "com/centreon/broker/tls/params.hh" * @brief Configure parameters of a TLS connection (either incoming @@ -59,17 +57,16 @@ class params { params(params const& p) = delete; params& operator=(params const& p) = delete; virtual ~params(); - void apply(gnutls_session_t session); + void apply(gnutls_session_t session) const; void load(); void reset(); void set_cert(std::string const& cert, std::string const& key); void set_compression(bool compress = false); void set_trusted_ca(std::string const& ca_cert); void set_tls_hostname(std::string const& tls_hostname); - void validate_cert(gnutls_session_t session); + void validate_cert(gnutls_session_t session) const; }; -} // namespace tls -} +} // namespace com::centreon::broker::tls #endif // !CCB_TLS_PARAMS_HH diff --git a/broker/tls/inc/com/centreon/broker/tls/stream.hh b/broker/tls/inc/com/centreon/broker/tls/stream.hh index f4a28a311fb..8b59b375ae9 100644 --- a/broker/tls/inc/com/centreon/broker/tls/stream.hh +++ b/broker/tls/inc/com/centreon/broker/tls/stream.hh @@ -22,10 +22,10 @@ #include #include "com/centreon/broker/io/stream.hh" +#include "com/centreon/broker/tls/params.hh" -namespace com::centreon::broker { +namespace com::centreon::broker::tls { -namespace tls { /** * @class stream stream.hh "com/centreon/broker/tls/stream.hh" * @brief TLS wrapper of an underlying stream. @@ -38,11 +38,14 @@ namespace tls { class stream : public io::stream { std::vector _buffer; time_t _deadline; - gnutls_session_t* _session; + gnutls_session_t _session; public: - stream(gnutls_session_t* session); + stream(unsigned int session_flags); ~stream(); + + void init(const params& param); + stream(const stream&) = delete; stream& operator=(const stream&) = delete; bool read(std::shared_ptr& d, time_t deadline) override; @@ -51,8 +54,7 @@ class stream : public io::stream { int32_t stop() override { return 0; } long long write_encrypted(void const* buffer, long long size); }; -} // namespace tls -} +} // namespace com::centreon::broker::tls #endif // !CCB_TLS_STREAM_HH diff --git a/broker/tls/src/acceptor.cc b/broker/tls/src/acceptor.cc index 80c631b2aea..18455596d13 100644 --- a/broker/tls/src/acceptor.cc +++ b/broker/tls/src/acceptor.cc @@ -1,28 +1,26 @@ /** -* Copyright 2009-2013, 2021 Centreon -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* For more information : contact@centreon.com -*/ + * Copyright 2009-2013, 2021 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ #include "com/centreon/broker/tls/acceptor.hh" #include #include "com/centreon/broker/log_v2.hh" -#include "com/centreon/broker/tls/internal.hh" -#include "com/centreon/broker/tls/params.hh" #include "com/centreon/broker/tls/stream.hh" #include "com/centreon/exceptions/msg_fmt.hh" @@ -87,7 +85,7 @@ std::shared_ptr acceptor::open() { */ std::shared_ptr acceptor::open( const std::shared_ptr& lower) { - std::shared_ptr u; + std::shared_ptr u; if (lower) { int ret; @@ -98,61 +96,10 @@ std::shared_ptr acceptor::open( p.set_tls_hostname(_tls_hostname); p.load(); - gnutls_session_t* session(new gnutls_session_t); - try { - // Initialize the TLS session - log_v2::tls()->debug("TLS: initializing session"); - // GNUTLS_NONBLOCK was introduced in gnutls 2.99.3. -#ifdef GNUTLS_NONBLOCK - ret = gnutls_init(session, GNUTLS_SERVER | GNUTLS_NONBLOCK); -#else - ret = gnutls_init(session, GNUTLS_SERVER); -#endif // GNUTLS_NONBLOCK - if (ret != GNUTLS_E_SUCCESS) { - log_v2::tls()->error("TLS: cannot initialize session: {}", - gnutls_strerror(ret)); - throw msg_fmt("TLS: cannot initialize session: {}", - gnutls_strerror(ret)); - } - - // Apply TLS parameters. - p.apply(*session); - - // Create stream object. - u.reset(new stream(session)); - } catch (...) { - gnutls_deinit(*session); - delete session; - throw; - } + // Create stream object. + u = std::make_shared(GNUTLS_SERVER | GNUTLS_NONBLOCK); u->set_substream(lower); - - // Bind the TLS session with the stream from the lower layer. -#if GNUTLS_VERSION_NUMBER < 0x020C00 - gnutls_transport_set_lowat(*session, 0); -#endif // GNU TLS < 2.12.0 - gnutls_transport_set_pull_function(*session, pull_helper); - gnutls_transport_set_push_function(*session, push_helper); - gnutls_transport_set_ptr(*session, u.get()); - - // Perform the TLS handshake. - log_v2::tls()->debug("TLS: performing handshake"); - do { - ret = gnutls_handshake(*session); - } while (GNUTLS_E_AGAIN == ret || GNUTLS_E_INTERRUPTED == ret); - if (ret != GNUTLS_E_SUCCESS) { - log_v2::tls()->error("TLS: handshake failed: {}", gnutls_strerror(ret)); - throw msg_fmt("TLS: handshake failed: {} ", gnutls_strerror(ret)); - } - log_v2::tls()->debug("TLS: successful handshake"); - gnutls_protocol_t prot = gnutls_protocol_get_version(*session); - gnutls_cipher_algorithm_t ciph = gnutls_cipher_get(*session); - log_v2::tls()->debug("TLS: protocol and cipher {} {} used", - gnutls_protocol_get_name(prot), - gnutls_cipher_get_name(ciph)); - - // Check certificate. - p.validate_cert(*session); + u->init(p); } return u; diff --git a/broker/tls/src/connector.cc b/broker/tls/src/connector.cc index 010a262eed8..07e27119434 100644 --- a/broker/tls/src/connector.cc +++ b/broker/tls/src/connector.cc @@ -1,26 +1,24 @@ /** -* Copyright 2009-2013 Centreon -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* For more information : contact@centreon.com -*/ + * Copyright 2009-2013 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ #include "com/centreon/broker/tls/connector.hh" #include "com/centreon/broker/log_v2.hh" -#include "com/centreon/broker/tls/internal.hh" -#include "com/centreon/broker/tls/params.hh" #include "com/centreon/broker/tls/stream.hh" #include "com/centreon/exceptions/msg_fmt.hh" @@ -72,7 +70,7 @@ std::shared_ptr connector::open() { * @return Encrypted stream. */ std::shared_ptr connector::open(std::shared_ptr lower) { - std::shared_ptr u; + std::shared_ptr u; if (lower) { int ret; // Load parameters. @@ -82,61 +80,10 @@ std::shared_ptr connector::open(std::shared_ptr lower) { p.set_tls_hostname(_tls_hostname); p.load(); - gnutls_session_t* session(new gnutls_session_t); - try { - // Initialize the TLS session - log_v2::tls()->debug("TLS: initializing session"); -#ifdef GNUTLS_NONBLOCK - ret = gnutls_init(session, GNUTLS_CLIENT | GNUTLS_NONBLOCK); -#else - ret = gnutls_init(session, GNUTLS_CLIENT); -#endif // GNUTLS_NONBLOCK - if (ret != GNUTLS_E_SUCCESS) { - log_v2::tls()->error("TLS: cannot initialize session: {}", - gnutls_strerror(ret)); - throw msg_fmt("TLS: cannot initialize session: {} ", - gnutls_strerror(ret)); - } - - // Apply TLS parameters to the current session. - p.apply(*session); - - // Create stream object. - u.reset(new stream(session)); - } catch (...) { - gnutls_deinit(*session); - delete session; - throw; - } + // Create stream object. + u = std::make_shared(GNUTLS_CLIENT); u->set_substream(lower); - - // Bind the TLS session with the stream from the lower layer. -#if GNUTLS_VERSION_NUMBER < 0x020C00 - gnutls_transport_set_lowat(*session, 0); -#endif // GNU TLS < 2.12.0 - gnutls_transport_set_pull_function(*session, pull_helper); - gnutls_transport_set_push_function(*session, push_helper); - gnutls_transport_set_ptr(*session, u.get()); - - // Perform the TLS handshake. - log_v2::tls()->debug("TLS: performing handshake"); - do { - ret = gnutls_handshake(*session); - } while (GNUTLS_E_AGAIN == ret || GNUTLS_E_INTERRUPTED == ret); - if (ret != GNUTLS_E_SUCCESS) { - log_v2::tls()->error("TLS: handshake failed: {}", gnutls_strerror(ret)); - throw msg_fmt("TLS: handshake failed: {}", gnutls_strerror(ret)); - } - - log_v2::tls()->debug("TLS: successful handshake"); - gnutls_protocol_t prot = gnutls_protocol_get_version(*session); - gnutls_cipher_algorithm_t ciph = gnutls_cipher_get(*session); - log_v2::tls()->debug("TLS: protocol and cipher {} {} used", - gnutls_protocol_get_name(prot), - gnutls_cipher_get_name(ciph)); - - // Check certificate if necessary. - p.validate_cert(*session); + u->init(p); } return u; diff --git a/broker/tls/src/internal.cc b/broker/tls/src/internal.cc index e570792d8c5..5fc09236db6 100644 --- a/broker/tls/src/internal.cc +++ b/broker/tls/src/internal.cc @@ -1,20 +1,20 @@ /** -* Copyright 2009-2013,2017 Centreon -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* For more information : contact@centreon.com -*/ + * Copyright 2009-2013,2017 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ #include @@ -136,7 +136,12 @@ void tls::initialize() { * layer and give it to TLS for decoding. */ ssize_t tls::pull_helper(gnutls_transport_ptr_t ptr, void* data, size_t size) { - return static_cast(ptr)->read_encrypted(data, size); + try { + return static_cast(ptr)->read_encrypted(data, size); + } catch (const std::exception& e) { + SPDLOG_LOGGER_DEBUG(log_v2::tls(), "read failed: {}", e.what()); + return -1; + } } /** @@ -146,5 +151,10 @@ ssize_t tls::pull_helper(gnutls_transport_ptr_t ptr, void* data, size_t size) { ssize_t tls::push_helper(gnutls_transport_ptr_t ptr, void const* data, size_t size) { - return static_cast(ptr)->write_encrypted(data, size); + try { + return static_cast(ptr)->write_encrypted(data, size); + } catch (const std::exception& e) { + SPDLOG_LOGGER_DEBUG(log_v2::tls(), "write failed: {}", e.what()); + return -1; + } } diff --git a/broker/tls/src/params.cc b/broker/tls/src/params.cc index 81baa3cd555..6d633964c32 100644 --- a/broker/tls/src/params.cc +++ b/broker/tls/src/params.cc @@ -50,7 +50,7 @@ params::~params() { * * @param[out] session Object on which parameters will be applied. */ -void params::apply(gnutls_session_t session) { +void params::apply(gnutls_session_t session)const { // Set the encryption method (normal ciphers with anonymous // Diffie-Hellman and optionnally compression). int ret; @@ -221,7 +221,7 @@ void params::set_trusted_ca(std::string const& ca_cert) { * * @param[in] session Session on which checks will be performed. */ -void params::validate_cert(gnutls_session_t session) { +void params::validate_cert(gnutls_session_t session) const { if (!_ca.empty()) { int ret; uint32_t status; diff --git a/broker/tls/src/stream.cc b/broker/tls/src/stream.cc index 46708fc5533..b0528e8a555 100644 --- a/broker/tls/src/stream.cc +++ b/broker/tls/src/stream.cc @@ -1,22 +1,20 @@ /** -* Copyright 2009-2017 Centreon -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* For more information : contact@centreon.com -*/ - -#include "com/centreon/broker/tls/stream.hh" + * Copyright 2009-2017 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ #include @@ -25,6 +23,9 @@ #include "com/centreon/broker/log_v2.hh" #include "com/centreon/exceptions/msg_fmt.hh" +#include "com/centreon/broker/tls/internal.hh" +#include "com/centreon/broker/tls/stream.hh" + using namespace com::centreon::broker; using namespace com::centreon::broker::tls; using namespace com::centreon::exceptions; @@ -44,8 +45,56 @@ using namespace com::centreon::exceptions; * @param[in] sess TLS session, providing informations on the * encryption that should be used. */ -stream::stream(gnutls_session_t* sess) - : io::stream("TLS"), _deadline((time_t)-1), _session(sess) {} +stream::stream(unsigned int session_flags) + : io::stream("TLS"), _deadline((time_t)-1), _session(nullptr) { + SPDLOG_LOGGER_DEBUG(log_v2::tls(), "{:p} TLS: created", + static_cast(this)); + int ret = gnutls_init(&_session, session_flags); + if (ret != GNUTLS_E_SUCCESS) { + SPDLOG_LOGGER_ERROR(log_v2::tls(), "TLS: cannot initialize session: {}", + gnutls_strerror(ret)); + throw msg_fmt("TLS: cannot initialize session: {}", gnutls_strerror(ret)); + } +} + +/** + * @brief this mehtod initialize crypto of the stream + * + * @param param crypto params that will validate cert of the session + */ +void stream::init(const params& param) { + int ret; + param.apply(_session); + + // Bind the TLS session with the stream from the lower layer. +#if GNUTLS_VERSION_NUMBER < 0x020C00 + gnutls_transport_set_lowat(*session, 0); +#endif // GNU TLS < 2.12.0 + gnutls_transport_set_pull_function(_session, pull_helper); + gnutls_transport_set_push_function(_session, push_helper); + gnutls_transport_set_ptr(_session, this); + + // Perform the TLS handshake. + SPDLOG_LOGGER_DEBUG(log_v2::tls(), "{:p} TLS: performing handshake", + static_cast(this)); + do { + ret = gnutls_handshake(_session); + } while (GNUTLS_E_AGAIN == ret || GNUTLS_E_INTERRUPTED == ret); + if (ret != GNUTLS_E_SUCCESS) { + SPDLOG_LOGGER_ERROR(log_v2::tls(), "TLS: handshake failed: {}", + gnutls_strerror(ret)); + throw msg_fmt("TLS: handshake failed: {} ", gnutls_strerror(ret)); + } + SPDLOG_LOGGER_DEBUG(log_v2::tls(), "TLS: successful handshake"); + gnutls_protocol_t prot = gnutls_protocol_get_version(_session); + gnutls_cipher_algorithm_t ciph = gnutls_cipher_get(_session); + SPDLOG_LOGGER_DEBUG(log_v2::tls(), "TLS: protocol and cipher {} {} used", + gnutls_protocol_get_name(prot), + gnutls_cipher_get_name(ciph)); + + // Check certificate. + param.validate_cert(_session); +} /** * @brief Destructor. @@ -56,11 +105,11 @@ stream::stream(gnutls_session_t* sess) stream::~stream() { if (_session) { try { - _deadline = time(nullptr) + 30; // XXX : use connection timeout - gnutls_bye(*_session, GNUTLS_SHUT_RDWR); - gnutls_deinit(*_session); - delete (_session); - _session = nullptr; + SPDLOG_LOGGER_DEBUG(log_v2::tls(), "{:p} TLS: destroy session: {:p}", + static_cast(this), + static_cast(_session)); + gnutls_bye(_session, GNUTLS_SHUT_RDWR); + gnutls_deinit(_session); } // Ignore exception whatever the error might be. catch (...) { @@ -89,11 +138,11 @@ bool stream::read(std::shared_ptr& d, time_t deadline) { _deadline = deadline; std::shared_ptr buffer(new io::raw); buffer->resize(BUFSIZ); - int ret(gnutls_record_recv(*_session, buffer->data(), buffer->size())); + int ret(gnutls_record_recv(_session, buffer->data(), buffer->size())); if (ret < 0) { if ((ret != GNUTLS_E_INTERRUPTED) && (ret != GNUTLS_E_AGAIN)) { - log_v2::tls()->error("TLS: could not receive data: {}", - gnutls_strerror(ret)); + SPDLOG_LOGGER_ERROR(log_v2::tls(), "TLS: could not receive data: {}", + gnutls_strerror(ret)); throw msg_fmt("TLS: could not receive data: {} ", gnutls_strerror(ret)); } else return false; @@ -102,7 +151,7 @@ bool stream::read(std::shared_ptr& d, time_t deadline) { d = buffer; return true; } else { - log_v2::tls()->error("TLS session is terminated"); + SPDLOG_LOGGER_ERROR(log_v2::tls(), "TLS session is terminated"); throw msg_fmt("TLS session is terminated"); } return false; @@ -117,39 +166,45 @@ bool stream::read(std::shared_ptr& d, time_t deadline) { * @return Number of bytes actually read. */ long long stream::read_encrypted(void* buffer, long long size) { - // Read some data. - bool timed_out(false); - while (_buffer.empty()) { - std::shared_ptr d; - timed_out = !_substream->read(d, _deadline); - if (!timed_out && d && d->type() == io::raw::static_type()) { - io::raw* r(static_cast(d.get())); - _buffer.reserve(_buffer.size() + r->get_buffer().size()); - _buffer.insert(_buffer.end(), r->get_buffer().begin(), - r->get_buffer().end()); - //_buffer.append(r->data(), r->size()); - } else if (timed_out) - break; - } + try { + // Read some data. + bool timed_out(false); + while (_buffer.empty()) { + std::shared_ptr d; + timed_out = !_substream->read(d, _deadline); + if (!timed_out && d && d->type() == io::raw::static_type()) { + io::raw* r(static_cast(d.get())); + _buffer.reserve(_buffer.size() + r->get_buffer().size()); + _buffer.insert(_buffer.end(), r->get_buffer().begin(), + r->get_buffer().end()); + //_buffer.append(r->data(), r->size()); + } else if (timed_out) + break; + } - // Transfer data. - uint32_t rb(_buffer.size()); - if (!rb) { - if (timed_out) { - gnutls_transport_set_errno(*_session, EAGAIN); - return -1; + // Transfer data. + uint32_t rb(_buffer.size()); + if (!rb) { + if (timed_out) { + gnutls_transport_set_errno(_session, EAGAIN); + return -1; + } else { + return 0; + } + } else if (size >= rb) { + memcpy(buffer, _buffer.data(), rb); + _buffer.clear(); + return rb; } else { - return 0; + memcpy(buffer, _buffer.data(), size); + _buffer.erase(_buffer.begin(), _buffer.begin() + size); + //_buffer.remove(0, size); + return size; } - } else if (size >= rb) { - memcpy(buffer, _buffer.data(), rb); - _buffer.clear(); - return rb; - } else { - memcpy(buffer, _buffer.data(), size); - _buffer.erase(_buffer.begin(), _buffer.begin() + size); - //_buffer.remove(0, size); - return size; + } catch (const std::exception& e) { + SPDLOG_LOGGER_DEBUG(log_v2::tls(), "tls read fail: {}", e.what()); + gnutls_transport_set_errno(_session, EPIPE); + throw; } } @@ -172,10 +227,10 @@ int stream::write(std::shared_ptr const& d) { char const* ptr(packet->const_data()); int size(packet->size()); while (size > 0) { - int ret(gnutls_record_send(*_session, ptr, size)); + int ret(gnutls_record_send(_session, ptr, size)); if (ret < 0) { - log_v2::tls()->error("TLS: could not send data: {}", - gnutls_strerror(ret)); + SPDLOG_LOGGER_ERROR(log_v2::tls(), "TLS: could not send data: {}", + gnutls_strerror(ret)); throw msg_fmt("TLS: could not send data: {}", gnutls_strerror(ret)); } ptr += ret; @@ -199,9 +254,15 @@ long long stream::write_encrypted(void const* buffer, long long size) { std::vector tmp(const_cast(static_cast(buffer)), const_cast(static_cast(buffer)) + static_cast(size)); - log_v2::tls()->trace("tls write enc: {}", size); + SPDLOG_LOGGER_TRACE(log_v2::tls(), "tls write enc: {}", size); r->get_buffer() = std::move(tmp); - _substream->write(r); - _substream->flush(); + try { + _substream->write(r); + _substream->flush(); + } catch (const std::exception& e) { + SPDLOG_LOGGER_DEBUG(log_v2::tls(), "tls write fail: {}", e.what()); + gnutls_transport_set_errno(_session, EPIPE); + throw; + } return size; } diff --git a/malloc-trace/CMakeLists.txt b/malloc-trace/CMakeLists.txt index dd1954a6c6c..fe1a38c6518 100644 --- a/malloc-trace/CMakeLists.txt +++ b/malloc-trace/CMakeLists.txt @@ -39,4 +39,4 @@ target_include_directories(centreon-malloc-trace PRIVATE ${CMAKE_SOURCE_DIR}/common/inc ) -target_precompile_headers(centreon-malloc-trace PRIVATE "precomp_inc/precomp.hh") \ No newline at end of file +target_precompile_headers(centreon-malloc-trace PRIVATE "precomp_inc/precomp.hh") diff --git a/malloc-trace/README.md b/malloc-trace/README.md index 78e0e7b640c..812be313767 100644 --- a/malloc-trace/README.md +++ b/malloc-trace/README.md @@ -64,4 +64,4 @@ In this scripts, you will find in comments how to load output csv file in that t ### remove_malloc_free.py We store in output file malloc that aren't freed in the next minute, we also store orphan free. So if a malloc is dumped and it's corresponding free is operated two minutes later, the two are stored in output file. -The purpose of this script is to remove these malloc-free pairs. \ No newline at end of file +The purpose of this script is to remove these malloc-free pairs. diff --git a/malloc-trace/inc/com/centreon/malloc_trace/intrusive_map.hh b/malloc-trace/inc/com/centreon/malloc_trace/intrusive_map.hh index 0962f8f52b7..298c5be90bf 100644 --- a/malloc-trace/inc/com/centreon/malloc_trace/intrusive_map.hh +++ b/malloc-trace/inc/com/centreon/malloc_trace/intrusive_map.hh @@ -45,7 +45,6 @@ class intrusive_map { node_map _nodes; public: - ~intrusive_map() { _nodes.clear(); } const node_type* find(const key_type& key) const { diff --git a/malloc-trace/inc/com/centreon/malloc_trace/orphan_container.hh b/malloc-trace/inc/com/centreon/malloc_trace/orphan_container.hh index 3c0d76dc546..061c327d35c 100644 --- a/malloc-trace/inc/com/centreon/malloc_trace/orphan_container.hh +++ b/malloc-trace/inc/com/centreon/malloc_trace/orphan_container.hh @@ -25,7 +25,7 @@ namespace com::centreon::malloc_trace { -constexpr size_t max_backtrace_size = 10; +constexpr size_t max_backtrace_size = 15; /** * @brief information of free or malloc with stacktrace diff --git a/malloc-trace/src/malloc_trace.cc b/malloc-trace/src/malloc_trace.cc index 3a712d76d8e..38bf8ea2463 100644 --- a/malloc-trace/src/malloc_trace.cc +++ b/malloc-trace/src/malloc_trace.cc @@ -26,6 +26,7 @@ using namespace com::centreon::malloc_trace; + extern void* __libc_malloc(size_t size); pid_t gettid() __attribute__((weak)); @@ -138,8 +139,7 @@ static void* malloc(size_t size, const char* funct_name) { // if this thread is not yet dumping => store it if (have_to_dump) { if (_orphans) { - constexpr std::string_view _funct_name("malloc"); - _orphans->add_malloc(p, size, thread_id, _funct_name, + _orphans->add_malloc(p, size, thread_id, funct_name, boost::stacktrace::stacktrace(), 2); _orphans->flush_to_file(); } @@ -239,4 +239,4 @@ void free(void* p) { } _thread_dump_active.reset_dump_active(thread_id); } -} \ No newline at end of file +} diff --git a/malloc-trace/src/orphan_container.cc b/malloc-trace/src/orphan_container.cc index 45f7fce6b10..091a43eccda 100644 --- a/malloc-trace/src/orphan_container.cc +++ b/malloc-trace/src/orphan_container.cc @@ -269,7 +269,7 @@ void orphan_container::flush_to_file() { } _time_to_malloc.erase_and_dispose( _time_to_malloc.begin(), upper, [this](orphan_malloc* to_dispose) { - _malloc_allocator.deallocate(to_dispose); + _malloc_allocator.deallocate(to_dispose); }); } From 9afc361d386f0a65ac865159a94ebfdd5093aadc Mon Sep 17 00:00:00 2001 From: jean-christophe81 <98889244+jean-christophe81@users.noreply.github.com> Date: Tue, 30 Jan 2024 09:40:02 +0100 Subject: [PATCH 53/64] retention.dat is not deleted by collect install (#1007) REFS:MON-24741 --- packaging/centreon-collect.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packaging/centreon-collect.yaml b/packaging/centreon-collect.yaml index 6da569cfe3a..e1ab618aac6 100644 --- a/packaging/centreon-collect.yaml +++ b/packaging/centreon-collect.yaml @@ -30,12 +30,8 @@ contents: owner: centreon-engine group: centreon-engine - - src: "./files/empty_file" - dst: "/var/log/centreon-engine/retention.dat" - file_info: - mode: 0755 - owner: centreon-engine - group: centreon-engine + - dst: "/var/log/centreon-engine/retention.dat" + type: ghost - src: "./files/empty_file" dst: "/var/log/centreon-engine/status.dat" From a1a15e664d1772e54b3ba2f0a34603bff5caca62 Mon Sep 17 00:00:00 2001 From: jean-christophe81 <98889244+jean-christophe81@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:00:07 +0100 Subject: [PATCH 54/64] MON-33334 fix stats muxer memory-leak (#1087) * fix stats muxer memory-leak * no package generation REFS:MON-33334 --- broker/core/multiplexing/src/muxer.cc | 48 +++++++++++++++------------ 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/broker/core/multiplexing/src/muxer.cc b/broker/core/multiplexing/src/muxer.cc index 1e02aab8ee4..4ac2c1e7241 100644 --- a/broker/core/multiplexing/src/muxer.cc +++ b/broker/core/multiplexing/src/muxer.cc @@ -1,20 +1,20 @@ /** -* Copyright 2009-2013,2015-2017,2019-2021 Centreon -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* For more information : contact@centreon.com -*/ + * Copyright 2009-2013,2015-2017,2019-2021 Centreon + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For more information : contact@centreon.com + */ #include "com/centreon/broker/multiplexing/muxer.hh" @@ -195,15 +195,19 @@ std::shared_ptr muxer::create(std::string name, * Destructor. */ muxer::~muxer() noexcept { - stats::center::instance().unregister_muxer(_name); unsubscribe(); - std::lock_guard lock(_mutex); - SPDLOG_LOGGER_INFO(log_v2::core(), - "Destroying muxer {}: number of events in the queue: {}", - _name, _events_size); - _clean(); + { + std::lock_guard lock(_mutex); + SPDLOG_LOGGER_INFO(log_v2::core(), + "Destroying muxer {}: number of events in the queue: {}", + _name, _events_size); + _clean(); + } DEBUG( fmt::format("DESTRUCTOR muxer {:p} {}", static_cast(this), _name)); + // caution, unregister_muxer must be the last center method called at muxer + // destruction to avoid re create a muxer stat entry + stats::center::instance().unregister_muxer(_name); } /** From 0db41bea8b1c515c93c5fc422ea8df42e664fb98 Mon Sep 17 00:00:00 2001 From: schapron Date: Thu, 1 Feb 2024 11:39:03 +0100 Subject: [PATCH 55/64] try to ignore specific action --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3db635e65a3..7680b2dc048 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,3 +8,7 @@ updates: labels: - 'dependencies' - 'gha' + ignore: + - dependency-name: actions/setup-python + versions: + - 4.7.1 From d6193ffe4a3c7993d63cde82dde239c0fc9109f2 Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Thu, 1 Feb 2024 11:49:12 +0100 Subject: [PATCH 56/64] add quotes to dependency ignore --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7680b2dc048..f8f7c19fb11 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,6 @@ updates: - 'dependencies' - 'gha' ignore: - - dependency-name: actions/setup-python + - dependency-name: "actions/setup-python" versions: - 4.7.1 From abf251328806bc500a9aa9c4097d13bbde11063b Mon Sep 17 00:00:00 2001 From: Paul Oureib Date: Thu, 1 Feb 2024 11:51:23 +0100 Subject: [PATCH 57/64] Revert "try to ignore specific action" This reverts commit 0db41bea8b1c515c93c5fc422ea8df42e664fb98. --- .github/dependabot.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f8f7c19fb11..3db635e65a3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,7 +8,3 @@ updates: labels: - 'dependencies' - 'gha' - ignore: - - dependency-name: "actions/setup-python" - versions: - - 4.7.1 From a2c199d1d95a6cbe34ad9153c48c6be10b86c451 Mon Sep 17 00:00:00 2001 From: May <110405507+paul-oureib@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:32:44 +0100 Subject: [PATCH 58/64] feat(packaging): package for Ubuntu Jammy (#1097) Co-authored-by: Kevin Duret --- .github/actions/delivery/action.yml | 16 +++-- .github/actions/package/action.yml | 8 +-- .github/actions/promote-to-stable/action.yml | 2 +- .../docker/Dockerfile.centreon-collect-alma8 | 6 +- .../docker/Dockerfile.centreon-collect-alma9 | 6 +- .../Dockerfile.centreon-collect-alma9-test | 4 +- ...ockerfile.centreon-collect-debian-bookworm | 6 +- ...file.centreon-collect-debian-bookworm-test | 50 ++++++++++++++++ ...ockerfile.centreon-collect-debian-bullseye | 6 +- ...file.centreon-collect-debian-bullseye-test | 9 +-- .../Dockerfile.centreon-collect-mysql-alma9 | 6 +- ...ckerfile.centreon-collect-mysql-alma9-test | 4 +- .../Dockerfile.centreon-collect-ubuntu-jammy | 59 +++++++++++++++++++ .github/scripts/collect-prepare-test-robot.sh | 41 ++++--------- .github/scripts/collect-unit-tests.sh | 4 +- .github/workflows/centreon-collect.yml | 9 +-- .github/workflows/docker-builder.yml | 8 +++ .github/workflows/package-collect.yml | 7 ++- .github/workflows/robot-nightly.yml | 19 ++++-- 19 files changed, 196 insertions(+), 74 deletions(-) create mode 100644 .github/docker/Dockerfile.centreon-collect-debian-bookworm-test create mode 100644 .github/docker/Dockerfile.centreon-collect-ubuntu-jammy diff --git a/.github/actions/delivery/action.yml b/.github/actions/delivery/action.yml index 24ec39cbc6b..757034c9c60 100644 --- a/.github/actions/delivery/action.yml +++ b/.github/actions/delivery/action.yml @@ -31,12 +31,12 @@ runs: - name: Build name for DEB shell: bash - if: ${{ inputs.distrib == 'bullseye' || inputs.distrib == 'bookworm' }} + if: ${{ contains(fromJSON('["bullseye", "bookworm", "jammy"]' , inputs.distrib) }} run: | echo "extfile=deb" >> $GITHUB_ENV - name: Use cache files - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: ./*.${{ env.extfile }} key: ${{ inputs.cache_key }} @@ -46,7 +46,7 @@ runs: JF_URL: https://centreon.jfrog.io JF_ACCESS_TOKEN: ${{ inputs.artifactory_token }} - - if: ${{ startsWith(inputs.distrib, 'el') }} + - if: ${{ env.extfile == 'rpm' }} name: Publish RPMs run: | FILES="*.${{ env.extfile }}" @@ -89,7 +89,7 @@ runs: done shell: bash - - if: ${{ inputs.distrib == 'bullseye' || inputs.distrib == 'bookworm' }} + - if: ${{ env.extfile == 'deb' }} name: Publish DEBs run: | FILES="*.${{ env.extfile }}" @@ -113,6 +113,12 @@ runs: echo "[DEBUG] - Version: $VERSION" - jf rt upload "$FILE" "apt-standard-${{ inputs.version }}-${{ inputs.stability }}/pool/${{ inputs.module_name }}/" --deb "${{ inputs.distrib }}/main/$ARCH" + if [[ "${{ inputs.distrib }}" == "jammy" ]]; then + REPO_PREFIX="ubuntu" + else + REPO_PREFIX="apt" + fi + + jf rt upload "$FILE" "${REPO_PREFIX}-standard-${{ inputs.version }}-${{ inputs.stability }}/pool/${{ inputs.module_name }}/" --deb "${{ inputs.distrib }}/main/$ARCH" done shell: bash diff --git a/.github/actions/package/action.yml b/.github/actions/package/action.yml index 921e54dd9c9..b51c1ae496e 100644 --- a/.github/actions/package/action.yml +++ b/.github/actions/package/action.yml @@ -62,9 +62,9 @@ runs: else export DIST="" if [ "${{ inputs.stability }}" = "unstable" ] || [ "${{ inputs.stability }}" = "canary" ]; then - export RELEASE="$RELEASE-${{ inputs.distrib }}" - elif [ "${{ inputs.stability }}" = "testing" ]; then - export RELEASE="1-${{ inputs.distrib }}" + export RELEASE="$RELEASE~${{ inputs.distrib }}" + else + export RELEASE="1~${{ inputs.distrib }}" fi fi @@ -87,7 +87,7 @@ runs: shell: bash - name: Cache packages - uses: actions/cache/save@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + uses: actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: ./*.${{ inputs.package_extension }} key: ${{ inputs.cache_key }} diff --git a/.github/actions/promote-to-stable/action.yml b/.github/actions/promote-to-stable/action.yml index 59470cc983a..54eba4d3b13 100644 --- a/.github/actions/promote-to-stable/action.yml +++ b/.github/actions/promote-to-stable/action.yml @@ -78,7 +78,7 @@ runs: shell: bash - name: Promote DEB packages to stable - if: ${{ startsWith(inputs.distrib, 'bullseye') || startsWith(inputs.distrib, 'bookworm') }} + if: ${{ contains(fromJSON('["bullseye", "bookworm"]' , inputs.distrib) }} run: | echo "[DEBUG] - Major version: ${{ inputs.major_version }}" echo "[DEBUG] - Minor version: ${{ inputs.minor_version }}" diff --git a/.github/docker/Dockerfile.centreon-collect-alma8 b/.github/docker/Dockerfile.centreon-collect-alma8 index 5e09d3a516b..6fbfc962d31 100644 --- a/.github/docker/Dockerfile.centreon-collect-alma8 +++ b/.github/docker/Dockerfile.centreon-collect-alma8 @@ -2,7 +2,7 @@ ARG REGISTRY_URL FROM ${REGISTRY_URL}/almalinux:8 -RUN <> /etc/yum.conf && \ @@ -55,6 +55,8 @@ dnf install -y cmake \ dnf update libarchive +dnf clean all + pip3 install conan==1.61.0 --prefix=/usr --upgrade rm -rf ~/.conan/profiles/default @@ -62,7 +64,7 @@ EOF COPY conanfile.txt . -RUN <> /etc/yum.conf @@ -48,6 +48,8 @@ dnf --best install -y cmake \ nfpm \ sudo +dnf clean all + pip3 install conan==1.61.0 --prefix=/usr --upgrade rm -rf ~/.conan/profiles/default @@ -55,7 +57,7 @@ EOF COPY conanfile.txt . -RUN <> /etc/yum.conf @@ -38,6 +38,8 @@ dnf --best install -y \ psmisc \ sudo +dnf clean all + echo "install robot and dependencies" pip3 install -U robotframework robotframework-databaselibrary robotframework-httpctrl robotframework-examples pymysql python-dateutil psutil diff --git a/.github/docker/Dockerfile.centreon-collect-debian-bookworm b/.github/docker/Dockerfile.centreon-collect-debian-bookworm index d0d88aeab7b..1a3fea0b1e3 100644 --- a/.github/docker/Dockerfile.centreon-collect-debian-bookworm +++ b/.github/docker/Dockerfile.centreon-collect-debian-bookworm @@ -2,7 +2,7 @@ ARG REGISTRY_URL FROM ${REGISTRY_URL}/debian:bookworm -RUN <> /etc/yum.conf @@ -48,6 +48,8 @@ dnf --best install -y cmake \ nfpm \ sudo +dnf clean all + pip3 install conan==1.61.0 --prefix=/usr --upgrade rm -rf ~/.conan/profiles/default @@ -55,7 +57,7 @@ EOF COPY conanfile.txt . -RUN <> /etc/yum.conf @@ -38,6 +38,8 @@ dnf --best install -y gcc \ sudo \ iproute +dnf clean all + echo "install robot and dependencies" pip3 install -U robotframework robotframework-databaselibrary robotframework-httpctrl robotframework-examples pymysql python-dateutil psutil diff --git a/.github/docker/Dockerfile.centreon-collect-ubuntu-jammy b/.github/docker/Dockerfile.centreon-collect-ubuntu-jammy new file mode 100644 index 00000000000..8a85e5f7f65 --- /dev/null +++ b/.github/docker/Dockerfile.centreon-collect-ubuntu-jammy @@ -0,0 +1,59 @@ +ARG REGISTRY_URL + +FROM ${REGISTRY_URL}/ubuntu:jammy + +RUN bash -e < /tmp/centreon.sql @@ -58,7 +58,11 @@ cat resources/centreon.sql | sed "s/DBNameConf/centreon/g" > /tmp/centreon.sql mysql -u root_centreon -pcentreon < resources/centreon_storage.sql mysql -u root_centreon -pcentreon < /tmp/centreon.sql -cd tests +if [ $database_type == 'mysql' ]; then + killall -w mysqldtoto +else + killall -w mariadbd +fi if [ "$distrib" = "ALMALINUX" ]; then dnf groupinstall -y "Development Tools" @@ -68,28 +72,3 @@ else apt-get install -y build-essential apt-get install -y python3-dev fi - - -echo "########################## Install centreon collect ###########################" -cd .. -echo "Installation..." -if [ "$distrib" = "ALMALINUX" ]; then - dnf clean all - rm -f ./*-selinux-*.rpm # avoid to install selinux packages which are dependent to centreon-common-selinux - dnf install -y ./*.rpm -else - apt-get update -# apt-get install -y ./*.deb -fi - - -if [ $database_type == 'mysql' ]; then - killall -w mysqldtoto -else - killall -w mariadbd -fi - -cd .. - -\rm -rf * - diff --git a/.github/scripts/collect-unit-tests.sh b/.github/scripts/collect-unit-tests.sh index 6ccf25a4b2c..a9a95f8e173 100755 --- a/.github/scripts/collect-unit-tests.sh +++ b/.github/scripts/collect-unit-tests.sh @@ -7,7 +7,9 @@ mkdir /src/build cd /src/build/ conan install .. -s compiler.cppstd=17 -s compiler.libcxx=libstdc++11 --build=missing -if [ $(cat /etc/issue | awk '{print $1}') = "Debian" ] ; then + +OS_ID=$(cat /etc/issue | awk '{print $1}') +if [[ "$OS_ID" == "Debian" || "$OS_ID" == "Ubuntu" ]] ; then CXXFLAGS="-Wall -Wextra" cmake -G "Ninja" -DCMAKE_EXPORT_COMPILE_COMMANDS=On -DWITH_CENTREON_CLIB_INCLUDE_DIR=../clib/inc/ -DWITH_CENTREON_CLIB_LIBRARIES=centreon-clib/libcentreon_clib.so -DCMAKE_BUILD_TYPE=Debug -DWITH_PREFIX=/usr -DWITH_PREFIX_BIN=/usr/sbin -DWITH_USER_BROKER=centreon-broker -DWITH_USER_ENGINE=centreon-engine -DWITH_GROUP_BROKER=centreon-broker -DWITH_GROUP_ENGINE=centreon-engine -DWITH_TESTING=On -DWITH_PREFIX_MODULES=/usr/share/centreon/lib/centreon-broker -DWITH_PREFIX_CONF_BROKER=/etc/centreon-broker -DWITH_PREFIX_LIB_BROKER=/usr/lib64/nagios -DWITH_PREFIX_CONF_ENGINE=/etc/centreon-engine -DWITH_PREFIX_LIB_ENGINE=/usr/lib64/centreon-engine -DWITH_PREFIX_LIB_CLIB=/usr/lib64/ -DWITH_RW_DIR=/var/lib/centreon-engine/rw -DWITH_VAR_DIR=/var/log/centreon-engine -DWITH_MODULE_SIMU=On -DNG=On .. else CXXFLAGS="-Wall -Wextra" cmake3 -G "Ninja" -DCMAKE_EXPORT_COMPILE_COMMANDS=On -DWITH_CENTREON_CLIB_INCLUDE_DIR=../clib/inc/ -DWITH_CENTREON_CLIB_LIBRARIES=centreon-clib/libcentreon_clib.so -DCMAKE_BUILD_TYPE=Debug -DWITH_PREFIX=/usr -DWITH_PREFIX_BIN=/usr/sbin -DWITH_USER_BROKER=centreon-broker -DWITH_USER_ENGINE=centreon-engine -DWITH_GROUP_BROKER=centreon-broker -DWITH_GROUP_ENGINE=centreon-engine -DWITH_TESTING=On -DWITH_PREFIX_MODULES=/usr/share/centreon/lib/centreon-broker -DWITH_PREFIX_CONF_BROKER=/etc/centreon-broker -DWITH_PREFIX_LIB_BROKER=/usr/lib64/nagios -DWITH_PREFIX_CONF_ENGINE=/etc/centreon-engine -DWITH_PREFIX_LIB_ENGINE=/usr/lib64/centreon-engine -DWITH_PREFIX_LIB_CLIB=/usr/lib64/ -DWITH_RW_DIR=/var/lib/centreon-engine/rw -DWITH_VAR_DIR=/var/log/centreon-engine -DWITH_MODULE_SIMU=On -DNG=On .. diff --git a/.github/workflows/centreon-collect.yml b/.github/workflows/centreon-collect.yml index c2b2e6ad90e..7c9ae6aaeae 100644 --- a/.github/workflows/centreon-collect.yml +++ b/.github/workflows/centreon-collect.yml @@ -135,7 +135,7 @@ jobs: distrib: ${{ matrix.distrib }} version: ${{ needs.get-version.outputs.version }} artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} - cache_key: cache-${{ github.sha }}-rpm-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} + cache_key: ${{ github.run_id }}-${{ github.sha }}-rpm-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} stability: ${{ needs.get-version.outputs.stability }} deliver-deb: @@ -152,6 +152,8 @@ jobs: arch: arm64 - distrib: bookworm arch: amd64 + - distrib: jammy + arch: amd64 name: deliver ${{ matrix.distrib }} @@ -160,14 +162,13 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Publish DEB packages - if: ${{ !contains('bookworm', matrix.distrib) || !startsWith('22.', needs.get-version.outputs.version) || !startsWith('23.', needs.get-version.outputs.version) }} uses: ./.github/actions/delivery with: module_name: collect distrib: ${{ matrix.distrib }} version: ${{ needs.get-version.outputs.version }} artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} - cache_key: cache-${{ github.sha }}-deb-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} + cache_key: ${{ github.run_id }}-${{ github.sha }}-deb-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} stability: ${{ needs.get-version.outputs.stability }} promote: @@ -176,7 +177,7 @@ jobs: runs-on: [self-hosted, common] strategy: matrix: - distrib: [el8, el9, bullseye, bookworm] + distrib: [el8, el9, bullseye, bookworm, jammy] steps: - name: Checkout sources diff --git a/.github/workflows/docker-builder.yml b/.github/workflows/docker-builder.yml index 4b50dc9a602..6824ed50c02 100644 --- a/.github/workflows/docker-builder.yml +++ b/.github/workflows/docker-builder.yml @@ -59,6 +59,14 @@ jobs: dockerfile: centreon-collect-debian-bookworm image: centreon-collect-debian-bookworm tag: ${{ needs.get-version.outputs.img_version }} + - runner: collect + dockerfile: centreon-collect-debian-bookworm-test + image: centreon-collect-debian-bookworm-test + tag: ${{ needs.get-version.outputs.test_img_version }} + - runner: collect + dockerfile: centreon-collect-ubuntu-jammy + image: centreon-collect-ubuntu-jammy + tag: ${{ needs.get-version.outputs.img_version }} - runner: collect-arm64 dockerfile: centreon-collect-debian-bullseye image: centreon-collect-debian-bullseye-arm64 diff --git a/.github/workflows/package-collect.yml b/.github/workflows/package-collect.yml index e0c219eb331..621f2635e78 100644 --- a/.github/workflows/package-collect.yml +++ b/.github/workflows/package-collect.yml @@ -45,6 +45,11 @@ jobs: package_extension: deb runner: collect arch: amd64 + - image: centreon-collect-ubuntu-jammy + distrib: jammy + package_extension: deb + runner: collect + arch: amd64 - image: centreon-collect-debian-bullseye-arm64 distrib: bullseye package_extension: deb @@ -171,7 +176,7 @@ jobs: release: ${{ inputs.release }} arch: ${{ matrix.arch }} commit_hash: ${{ inputs.commit_hash }} - cache_key: cache-${{ github.sha }}-${{ matrix.package_extension}}-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} + cache_key: ${{ github.run_id }}-${{ github.sha }}-${{ matrix.package_extension}}-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} rpm_gpg_key: ${{ secrets.RPM_GPG_SIGNING_KEY }} rpm_gpg_signing_key_id: ${{ secrets.RPM_GPG_SIGNING_KEY_ID }} rpm_gpg_signing_passphrase: ${{ secrets.RPM_GPG_SIGNING_PASSPHRASE }} diff --git a/.github/workflows/robot-nightly.yml b/.github/workflows/robot-nightly.yml index 9c77b7b4826..13db94f259a 100644 --- a/.github/workflows/robot-nightly.yml +++ b/.github/workflows/robot-nightly.yml @@ -82,8 +82,16 @@ jobs: package_extension: deb arch: arm64 database_type: mariadb - test_group_name: robot_test-mariadb-el9-arm64 + test_group_name: robot_test-mariadb-bullseye-arm64 tests_params: '{}' + # uncomment once bbdo-protobuf.robot test is fixed + # - distrib: bookworm + # image: centreon-collect-debian-bookworm-test + # package_extension: deb + # arch: amd64 + # database_type: mariadb + # test_group_name: robot_test-mariadb-bookworm-amd64 + # tests_params: '{}' - distrib: el9 image: centreon-collect-alma9-test package_extension: rpm @@ -101,7 +109,7 @@ jobs: image: ${{ matrix.image }} image_test: ${{ matrix.image }}:${{ needs.get-version.outputs.test_img_version }} image_version: ${{ needs.get-version.outputs.img_version }} - package_cache_key: cache-${{ github.sha }}-${{ matrix.package_extension }}-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} + package_cache_key: ${{ github.run_id }}-${{ github.sha }}-${{ matrix.package_extension }}-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} package_cache_path: ./*.${{ matrix.package_extension}} database_type: ${{ matrix.database_type }} tests_params: ${{matrix.tests_params}} @@ -132,7 +140,7 @@ jobs: distrib: ${{ matrix.distrib }} version: ${{ needs.get-version.outputs.version }} artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} - cache_key: cache-${{ github.sha }}-rpm-centreon-collect-${{ matrix.distrib }}-amd64-${{ github.head_ref || github.ref_name }} + cache_key: ${{ github.run_id }}-${{ github.sha }}-rpm-centreon-collect-${{ matrix.distrib }}-amd64-${{ github.head_ref || github.ref_name }} stability: ${{ needs.get-version.outputs.stability }} deliver-deb: @@ -146,6 +154,8 @@ jobs: arch: amd64 - distrib: bookworm arch: amd64 + - distrib: jammy + arch: amd64 - distrib: bullseye arch: arm64 name: deliver ${{ matrix.distrib }} @@ -155,12 +165,11 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Publish DEB packages - if: ${{ !contains('bookworm', matrix.distrib) || !startsWith('22.', needs.get-version.outputs.version) || !startsWith('23.', needs.get-version.outputs.version) }} uses: ./.github/actions/delivery with: module_name: collect distrib: ${{ matrix.distrib }} version: ${{ needs.get-version.outputs.version }} artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} - cache_key: cache-${{ github.sha }}-deb-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} + cache_key: ${{ github.run_id }}-${{ github.sha }}-deb-centreon-collect-${{ matrix.distrib }}-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }} stability: ${{ needs.get-version.outputs.stability }} From 351102858909e2a9973eeff8fd56132cc43f871b Mon Sep 17 00:00:00 2001 From: tuntoja <58987095+tuntoja@users.noreply.github.com> Date: Fri, 2 Feb 2024 17:30:46 +0100 Subject: [PATCH 59/64] exclude paths from create jira version files (#1104) --- .github/workflows/release-collect.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release-collect.yml b/.github/workflows/release-collect.yml index f91fab7a47f..93f50c6fae1 100644 --- a/.github/workflows/release-collect.yml +++ b/.github/workflows/release-collect.yml @@ -10,6 +10,8 @@ on: - "[2-9][0-9].[0-9][0-9].x" paths: - "centreon-collect/**" + - "!centreon-collect/ci/**" + - "!centreon-collect/tests/**" workflow_dispatch: env: From 15e6ecda384cbd18f5e458f777e6d2962c597880 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 2 Feb 2024 19:09:14 +0100 Subject: [PATCH 60/64] fix(ci): fix delivery condition (#1108) --- .github/actions/delivery/action.yml | 2 +- .github/actions/promote-to-stable/action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/delivery/action.yml b/.github/actions/delivery/action.yml index 757034c9c60..66ed19e6db8 100644 --- a/.github/actions/delivery/action.yml +++ b/.github/actions/delivery/action.yml @@ -31,7 +31,7 @@ runs: - name: Build name for DEB shell: bash - if: ${{ contains(fromJSON('["bullseye", "bookworm", "jammy"]' , inputs.distrib) }} + if: ${{ contains(fromJSON('["bullseye", "bookworm", "jammy"]'), inputs.distrib) }} run: | echo "extfile=deb" >> $GITHUB_ENV diff --git a/.github/actions/promote-to-stable/action.yml b/.github/actions/promote-to-stable/action.yml index 54eba4d3b13..dfb7f6af133 100644 --- a/.github/actions/promote-to-stable/action.yml +++ b/.github/actions/promote-to-stable/action.yml @@ -78,7 +78,7 @@ runs: shell: bash - name: Promote DEB packages to stable - if: ${{ contains(fromJSON('["bullseye", "bookworm"]' , inputs.distrib) }} + if: ${{ contains(fromJSON('["bullseye", "bookworm"]'), inputs.distrib) }} run: | echo "[DEBUG] - Major version: ${{ inputs.major_version }}" echo "[DEBUG] - Minor version: ${{ inputs.minor_version }}" From 84de42b987deaf293d198b8796fc42ef46491fe6 Mon Sep 17 00:00:00 2001 From: jean-christophe81 <98889244+jean-christophe81@users.noreply.github.com> Date: Thu, 8 Feb 2024 11:04:40 +0100 Subject: [PATCH 61/64] lua_gc is called periodicaly in order to clean broker events owned by lua (#1119) --- .../com/centreon/broker/lua/broker_event.hh | 14 +++++- broker/lua/src/broker_event.cc | 47 ++++++++++++++++++- broker/lua/src/luabinding.cc | 1 + 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/broker/lua/inc/com/centreon/broker/lua/broker_event.hh b/broker/lua/inc/com/centreon/broker/lua/broker_event.hh index 7c5b0d07bac..16f94daa4e3 100644 --- a/broker/lua/inc/com/centreon/broker/lua/broker_event.hh +++ b/broker/lua/inc/com/centreon/broker/lua/broker_event.hh @@ -48,13 +48,25 @@ namespace lua { * */ class broker_event { + struct gc_info { + gc_info() : _broker_event_cpt(0), _last_full_gc(time(nullptr)) {} + + unsigned _broker_event_cpt; + time_t _last_full_gc; + }; + + static std::map _gc_info; + static std::mutex _gc_info_m; + + static int l_broker_event_destructor(lua_State* L); public: static void broker_event_reg(lua_State* L); static void create(lua_State* L, std::shared_ptr e); static void create_as_table(lua_State* L, const io::data& e); + static void lua_close(const lua_State* L); }; } // namespace lua -} +} // namespace com::centreon::broker #endif // !CCB_LUA_BROKER_EVENT_HH diff --git a/broker/lua/src/broker_event.cc b/broker/lua/src/broker_event.cc index 2c5644fde0c..84f639f2525 100644 --- a/broker/lua/src/broker_event.cc +++ b/broker/lua/src/broker_event.cc @@ -22,6 +22,7 @@ #include "com/centreon/broker/io/data.hh" #include "com/centreon/broker/io/protobuf.hh" #include "com/centreon/broker/mapping/entry.hh" +#include "com/centreon/broker/multiplexing/muxer.hh" #include "com/centreon/exceptions/msg_fmt.hh" using namespace com::centreon::broker; @@ -32,6 +33,8 @@ static void _write_item(lua_State* L, const google::protobuf::Message* p, const google::protobuf::FieldDescriptor* f); +std::map broker_event::_gc_info; +std::mutex broker_event::_gc_info_m; /** * The Lua broker_event constructor * @@ -48,6 +51,32 @@ void broker_event::create(lua_State* L, std::shared_ptr e) { luaL_getmetatable(L, "broker_event"); lua_setmetatable(L, -2); + bool have_to_gc_collect = false; + { + std::lock_guard l(_gc_info_m); + + /*In V2, lua stores only a userdata that contains a shared_ptr of event + * (16 bytes). So garbage collector don't see amount of memory used by + * events. + * So we need to call garbage collector ourselves to reduce memory + * consumption + * So we call at least gc every minute or + * at most every 10s if lua own more than + * com::centreon::broker::multiplexing::muxer::event_queue_max_size() events + * */ + time_t now = time(nullptr); + gc_info& gc_inf = _gc_info[L]; + if ((++gc_inf._broker_event_cpt > com::centreon::broker::multiplexing:: + muxer::event_queue_max_size() && + gc_inf._last_full_gc + 10 < now) || + (gc_inf._last_full_gc + 60 < now)) { + gc_inf._last_full_gc = now; + have_to_gc_collect = true; + } + } + if (have_to_gc_collect) { + lua_gc(L, LUA_GCCOLLECT, 0); + } } static void _message_to_table(lua_State* L, @@ -344,12 +373,18 @@ void broker_event::create_as_table(lua_State* L, const io::data& d) { * * @return 0 */ -static int l_broker_event_destructor(lua_State* L) { +int broker_event::l_broker_event_destructor(lua_State* L) { void* ptr = luaL_checkudata(L, 1, "broker_event"); if (ptr) { auto event = static_cast*>(ptr); event->reset(); + std::lock_guard l(_gc_info_m); + + gc_info& gc_inf = _gc_info[L]; + if (gc_inf._broker_event_cpt > 0) { + --gc_inf._broker_event_cpt; + } } return 0; } @@ -758,3 +793,13 @@ void broker_event::broker_event_reg(lua_State* L) { lua_setglobal(L, name); } + +/** + * @brief when a lua_State is closed we clean _gc_info + * + * @param The Lua interpreter as a lua_State* + */ +void broker_event::lua_close(const lua_State* L) { + std::lock_guard l(_gc_info_m); + _gc_info.erase(L); +} diff --git a/broker/lua/src/luabinding.cc b/broker/lua/src/luabinding.cc index a2ac8750d1e..c47d7ac3a96 100644 --- a/broker/lua/src/luabinding.cc +++ b/broker/lua/src/luabinding.cc @@ -91,6 +91,7 @@ int32_t luabinding::stop() { if (_L) { retval = flush(); lua_close(_L); + broker_event::lua_close(_L); _L = nullptr; } return retval; From 8f13d071d460cb05f1c7d4b17e9450e964e89540 Mon Sep 17 00:00:00 2001 From: sdepassio <114986849+sdepassio@users.noreply.github.com> Date: Fri, 9 Feb 2024 11:17:45 +0100 Subject: [PATCH 62/64] fix(gorgone): package last version of libzmq (#1115) * fix(ci): fix delivery condition (#1108) * add github action for libzmq * fix * fix * update * use centreon-collect github actions for delivery + add bullseye arm64 * try to fix path problem * fix * update * Update .github/workflows/zmq-lib.yml Co-authored-by: Kevin Duret * Update .github/workflows/zmq-lib.yml Co-authored-by: Kevin Duret * Update .github/workflows/zmq-lib.yml Co-authored-by: Kevin Duret * Update .github/workflows/zmq-lib.yml Co-authored-by: Kevin Duret * Update .github/workflows/zmq-lib.yml Co-authored-by: Kevin Duret * Update .github/workflows/zmq-lib.yml Co-authored-by: Kevin Duret --------- Co-authored-by: Kevin Duret --- .github/workflows/zmq-lib.yml | 207 ++++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 .github/workflows/zmq-lib.yml diff --git a/.github/workflows/zmq-lib.yml b/.github/workflows/zmq-lib.yml new file mode 100644 index 00000000000..4f82df8e509 --- /dev/null +++ b/.github/workflows/zmq-lib.yml @@ -0,0 +1,207 @@ +name: libzmq + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/zmq-lib.yml' + push: + branches: + - develop + - dev-[2-9][0-9].[0-9][0-9].x + - master + - "[2-9][0-9].[0-9][0-9].x" + paths: + - '.github/workflows/zmq-lib.yml' + +jobs: + get-version: + uses: ./.github/workflows/get-version.yml + + package-rpm: + needs: [get-version] + + strategy: + fail-fast: false + matrix: + include: + - image: packaging-alma8 + distrib: el8 + arch: amd64 + - image: packaging-alma9 + distrib: el9 + arch: amd64 + + runs-on: ubuntu-22.04 + + container: + image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:${{ needs.get-version.outputs.version }} + credentials: + username: ${{ secrets.DOCKER_REGISTRY_ID }} + password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + + name: package ${{ matrix.distrib }} + + steps: + - name: package rpm + run: | + dnf install -y wget rpmdevtools rpmlint epel-release + dnf config-manager --set-enabled crb || true # alma 9 + dnf config-manager --set-enabled powertools || true # alma 8 + dnf install -y asciidoc autoconf automake gcc gcc-c++ glib2-devel libbsd-devel libtool make xmlto + cd /github/home + wget -O - https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.5.tar.gz|tar zxvf - + mkdir -p /github/home/rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} + cp libzmq-4.3.5/packaging/redhat/zeromq.spec /github/home/rpmbuild/SPECS/ + wget https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz -O /github/home/rpmbuild/SOURCES/zeromq-4.3.5.tar.gz + rpmbuild -bb /github/home/rpmbuild/SPECS/zeromq.spec + cd - + mv /github/home/rpmbuild/RPMS/x86_64/*.rpm ./ + shell: bash + + - name: cache rpm + uses: actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + with: + path: ./*.rpm + key: ${{ github.run_id }}-${{ github.sha }}-rpm-libzmq-${{ matrix.distrib }}-${{ matrix.arch }} + + package-deb: + needs: [get-version] + + strategy: + fail-fast: false + matrix: + include: + - image: packaging-bullseye + distrib: bullseye + runner: ubuntu-22.04 + arch: amd64 + - image: packaging-bookworm + distrib: bookworm + runner: ubuntu-22.04 + arch: amd64 + - image: packaging-nfpm-jammy + distrib: jammy + runner: ubuntu-22.04 + arch: amd64 + - image: packaging-bullseye-arm64 + distrib: bullseye + runner: ["self-hosted", "collect-arm64"] + arch: arm64 + + runs-on: ${{ matrix.runner }} + + container: + image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:${{ needs.get-version.outputs.version }} + credentials: + username: ${{ secrets.DOCKER_REGISTRY_ID }} + password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + + name: package ${{ matrix.distrib }} ${{ matrix.arch }} + + steps: + - name: package deb + run: | + apt-get update + apt-get install -y dpkg-dev wget + apt-get install -y debhelper dh-autoreconf libkrb5-dev libnorm-dev libpgm-dev libsodium-dev libunwind8-dev libnss3-dev libgnutls28-dev libbsd-dev pkg-config asciidoc xmlto + wget -O - https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz|tar zxvf - + cd zeromq-4.3.5 + ./configure + make + make install + cd .. + wget -O - https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.5.tar.gz|tar zxvf - + cd libzmq-4.3.5 + apt-get install -y debhelper dh-autoreconf libkrb5-dev libnorm-dev libpgm-dev libsodium-dev libunwind8-dev libnss3-dev libgnutls28-dev libbsd-dev pkg-config asciidoc xmlto + ln -s packaging/debian + dpkg-buildpackage -us -uc -nc + cd .. + shell: bash + + - name: cache deb + uses: actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + with: + path: ./*.deb + key: ${{ github.run_id }}-${{ github.sha }}-deb-libzmq-${{ matrix.distrib }}-${{ matrix.arch }} + + deliver-rpm: + if: ${{ contains(fromJson('["testing", "unstable"]'), needs.get-version.outputs.stability) }} + needs: [get-version, package-rpm] + environment: ${{ needs.get-version.outputs.environment }} + runs-on: [self-hosted, common] + strategy: + matrix: + include: + - distrib: el8 + arch: amd64 + - distrib: el9 + arch: amd64 + + name: deliver ${{ matrix.distrib }} + + steps: + - name: Publish RPM packages + uses: ./.github/actions/delivery + with: + module_name: libzmq + distrib: ${{ matrix.distrib }} + version: ${{ needs.get-version.outputs.version }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.run_id }}-${{ github.sha }}-rpm-libzmq-${{ matrix.distrib }}-${{ matrix.arch }} + stability: ${{ needs.get-version.outputs.stability }} + + deliver-deb: + if: ${{ contains(fromJson('["testing", "unstable"]'), needs.get-version.outputs.stability) }} + needs: [get-version, package-deb] + environment: ${{ needs.get-version.outputs.environment }} + runs-on: [self-hosted, common] + strategy: + matrix: + include: + - distrib: bullseye + arch: amd64 + - distrib: bullseye + arch: arm64 + - distrib: bookworm + arch: amd64 + - distrib: jammy + arch: amd64 + + name: deliver ${{ matrix.distrib }} + + steps: + - name: Publish DEB packages + uses: ./.github/actions/delivery + with: + module_name: libzmq + distrib: ${{ matrix.distrib }} + version: ${{ needs.get-version.outputs.version }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.run_id }}-${{ github.sha }}-deb-libzmq-${{ matrix.distrib }}-${{ matrix.arch }} + stability: ${{ needs.get-version.outputs.stability }} + + promote: + needs: [get-version] + if: ${{ contains(fromJson('["stable"]'), needs.get-version.outputs.stability) && github.event_name != 'workflow_dispatch' }} + runs-on: [self-hosted, common] + strategy: + matrix: + distrib: [el8, el9, bullseye, bookworm, jammy] + + steps: + - name: Promote ${{ matrix.distrib }} to stable + uses: ./.github/actions/promote-to-stable + with: + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + module_name: libzmq + distrib: ${{ matrix.distrib }} + major_version: ${{ needs.get-version.outputs.version }} + minor_version: ${{ needs.get-version.outputs.patch }} + stability: ${{ needs.get-version.outputs.stability }} + repository_name: standard + github_base_ref: ${{ github.base_ref }} From fe085c6b6233d56c0704542c50ef6b254d689420 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 9 Feb 2024 12:02:33 +0100 Subject: [PATCH 63/64] fix(ci): fix delivery of libzmq (#1130) * fix(ci): fix delivery of libzmq * fix(ci): fix delivery of libzmq --- .github/workflows/{zmq-lib.yml => libzmq.yml} | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) rename .github/workflows/{zmq-lib.yml => libzmq.yml} (94%) diff --git a/.github/workflows/zmq-lib.yml b/.github/workflows/libzmq.yml similarity index 94% rename from .github/workflows/zmq-lib.yml rename to .github/workflows/libzmq.yml index 4f82df8e509..2f1971043d0 100644 --- a/.github/workflows/zmq-lib.yml +++ b/.github/workflows/libzmq.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: pull_request: paths: - - '.github/workflows/zmq-lib.yml' + - '.github/workflows/libzmq.yml' push: branches: - develop @@ -16,7 +16,7 @@ on: - master - "[2-9][0-9].[0-9][0-9].x" paths: - - '.github/workflows/zmq-lib.yml' + - '.github/workflows/libzmq.yml' jobs: get-version: @@ -145,6 +145,9 @@ jobs: name: deliver ${{ matrix.distrib }} steps: + - name: Checkout sources + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Publish RPM packages uses: ./.github/actions/delivery with: @@ -175,6 +178,9 @@ jobs: name: deliver ${{ matrix.distrib }} steps: + - name: Checkout sources + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Publish DEB packages uses: ./.github/actions/delivery with: @@ -194,6 +200,9 @@ jobs: distrib: [el8, el9, bullseye, bookworm, jammy] steps: + - name: Checkout sources + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Promote ${{ matrix.distrib }} to stable uses: ./.github/actions/promote-to-stable with: From cf7a6cbe9dcd9736a37a12a61ae784a9fb78717f Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Mon, 12 Feb 2024 10:11:48 +0100 Subject: [PATCH 64/64] fix(ci): add distrib name in libzmq deb package (#1133) * fix(ci): add distrib name in libzmq deb package * update version in changelog * update * replace UNRELEASED by distrib name * fix(ci): add distrib name in libzmq deb package * fix(ci): add distrib name in libzmq deb package * fix(ci): add distrib name in libzmq deb package * fix(ci): add distrib name in libzmq deb package * fix(ci): add distrib name in libzmq deb package * fix(ci): add distrib name in libzmq deb package --------- Co-authored-by: Sophie Depassio --- .github/workflows/libzmq.yml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/libzmq.yml b/.github/workflows/libzmq.yml index 2f1971043d0..4fa4309a73d 100644 --- a/.github/workflows/libzmq.yml +++ b/.github/workflows/libzmq.yml @@ -29,10 +29,10 @@ jobs: fail-fast: false matrix: include: - - image: packaging-alma8 + - image: packaging-nfpm-alma8 distrib: el8 arch: amd64 - - image: packaging-alma9 + - image: packaging-nfpm-alma9 distrib: el9 arch: amd64 @@ -52,15 +52,18 @@ jobs: dnf install -y wget rpmdevtools rpmlint epel-release dnf config-manager --set-enabled crb || true # alma 9 dnf config-manager --set-enabled powertools || true # alma 8 - dnf install -y asciidoc autoconf automake gcc gcc-c++ glib2-devel libbsd-devel libtool make xmlto + dnf install -y asciidoc autoconf automake gcc gcc-c++ glib2-devel libbsd-devel libtool make rpm-build xmlto + cd /github/home - wget -O - https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.5.tar.gz|tar zxvf - + wget -O - https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.5.tar.gz | tar zxvf - mkdir -p /github/home/rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} cp libzmq-4.3.5/packaging/redhat/zeromq.spec /github/home/rpmbuild/SPECS/ wget https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz -O /github/home/rpmbuild/SOURCES/zeromq-4.3.5.tar.gz rpmbuild -bb /github/home/rpmbuild/SPECS/zeromq.spec cd - + mv /github/home/rpmbuild/RPMS/x86_64/*.rpm ./ + rm -f zeromq-debugsource-*.rpm libzmq5-debuginfo-*.rpm shell: bash - name: cache rpm @@ -76,11 +79,11 @@ jobs: fail-fast: false matrix: include: - - image: packaging-bullseye + - image: packaging-nfpm-bullseye distrib: bullseye runner: ubuntu-22.04 arch: amd64 - - image: packaging-bookworm + - image: packaging-nfpm-bookworm distrib: bookworm runner: ubuntu-22.04 arch: amd64 @@ -107,20 +110,24 @@ jobs: - name: package deb run: | apt-get update - apt-get install -y dpkg-dev wget - apt-get install -y debhelper dh-autoreconf libkrb5-dev libnorm-dev libpgm-dev libsodium-dev libunwind8-dev libnss3-dev libgnutls28-dev libbsd-dev pkg-config asciidoc xmlto - wget -O - https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz|tar zxvf - + apt-get install -y debhelper dh-autoreconf dpkg-dev libkrb5-dev libnorm-dev libpgm-dev libsodium-dev libunwind8-dev libnss3-dev libgnutls28-dev libbsd-dev pkg-config asciidoc wget xmlto + wget -O - https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz | tar zxvf - + cd zeromq-4.3.5 ./configure make make install cd .. - wget -O - https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.5.tar.gz|tar zxvf - + + wget -O - https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.5.tar.gz | tar zxvf - cd libzmq-4.3.5 - apt-get install -y debhelper dh-autoreconf libkrb5-dev libnorm-dev libpgm-dev libsodium-dev libunwind8-dev libnss3-dev libgnutls28-dev libbsd-dev pkg-config asciidoc xmlto ln -s packaging/debian + sed -Ei 's/([0-9]+.[0-9]+.[0-9]+-[0-9]+.[0-9]+)/\1~${{ matrix.distrib }}/' debian/changelog + sed -Ei 's/UNRELEASED/${{ matrix.distrib }}/' debian/changelog dpkg-buildpackage -us -uc -nc cd .. + + rm -f libzmq5-dbg_*.deb shell: bash - name: cache deb @@ -175,7 +182,7 @@ jobs: - distrib: jammy arch: amd64 - name: deliver ${{ matrix.distrib }} + name: deliver ${{ matrix.distrib }} ${{ matrix.arch }} steps: - name: Checkout sources