From 15e6ecda384cbd18f5e458f777e6d2962c597880 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 2 Feb 2024 19:09:14 +0100 Subject: [PATCH 1/5] fix(ci): fix delivery condition (#1108) --- .github/actions/delivery/action.yml | 2 +- .github/actions/promote-to-stable/action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/delivery/action.yml b/.github/actions/delivery/action.yml index 757034c9c60..66ed19e6db8 100644 --- a/.github/actions/delivery/action.yml +++ b/.github/actions/delivery/action.yml @@ -31,7 +31,7 @@ runs: - name: Build name for DEB shell: bash - if: ${{ contains(fromJSON('["bullseye", "bookworm", "jammy"]' , inputs.distrib) }} + if: ${{ contains(fromJSON('["bullseye", "bookworm", "jammy"]'), inputs.distrib) }} run: | echo "extfile=deb" >> $GITHUB_ENV diff --git a/.github/actions/promote-to-stable/action.yml b/.github/actions/promote-to-stable/action.yml index 54eba4d3b13..dfb7f6af133 100644 --- a/.github/actions/promote-to-stable/action.yml +++ b/.github/actions/promote-to-stable/action.yml @@ -78,7 +78,7 @@ runs: shell: bash - name: Promote DEB packages to stable - if: ${{ contains(fromJSON('["bullseye", "bookworm"]' , inputs.distrib) }} + if: ${{ contains(fromJSON('["bullseye", "bookworm"]'), inputs.distrib) }} run: | echo "[DEBUG] - Major version: ${{ inputs.major_version }}" echo "[DEBUG] - Minor version: ${{ inputs.minor_version }}" From 84de42b987deaf293d198b8796fc42ef46491fe6 Mon Sep 17 00:00:00 2001 From: jean-christophe81 <98889244+jean-christophe81@users.noreply.github.com> Date: Thu, 8 Feb 2024 11:04:40 +0100 Subject: [PATCH 2/5] lua_gc is called periodicaly in order to clean broker events owned by lua (#1119) --- .../com/centreon/broker/lua/broker_event.hh | 14 +++++- broker/lua/src/broker_event.cc | 47 ++++++++++++++++++- broker/lua/src/luabinding.cc | 1 + 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/broker/lua/inc/com/centreon/broker/lua/broker_event.hh b/broker/lua/inc/com/centreon/broker/lua/broker_event.hh index 7c5b0d07bac..16f94daa4e3 100644 --- a/broker/lua/inc/com/centreon/broker/lua/broker_event.hh +++ b/broker/lua/inc/com/centreon/broker/lua/broker_event.hh @@ -48,13 +48,25 @@ namespace lua { * */ class broker_event { + struct gc_info { + gc_info() : _broker_event_cpt(0), _last_full_gc(time(nullptr)) {} + + unsigned _broker_event_cpt; + time_t _last_full_gc; + }; + + static std::map _gc_info; + static std::mutex _gc_info_m; + + static int l_broker_event_destructor(lua_State* L); public: static void broker_event_reg(lua_State* L); static void create(lua_State* L, std::shared_ptr e); static void create_as_table(lua_State* L, const io::data& e); + static void lua_close(const lua_State* L); }; } // namespace lua -} +} // namespace com::centreon::broker #endif // !CCB_LUA_BROKER_EVENT_HH diff --git a/broker/lua/src/broker_event.cc b/broker/lua/src/broker_event.cc index 2c5644fde0c..84f639f2525 100644 --- a/broker/lua/src/broker_event.cc +++ b/broker/lua/src/broker_event.cc @@ -22,6 +22,7 @@ #include "com/centreon/broker/io/data.hh" #include "com/centreon/broker/io/protobuf.hh" #include "com/centreon/broker/mapping/entry.hh" +#include "com/centreon/broker/multiplexing/muxer.hh" #include "com/centreon/exceptions/msg_fmt.hh" using namespace com::centreon::broker; @@ -32,6 +33,8 @@ static void _write_item(lua_State* L, const google::protobuf::Message* p, const google::protobuf::FieldDescriptor* f); +std::map broker_event::_gc_info; +std::mutex broker_event::_gc_info_m; /** * The Lua broker_event constructor * @@ -48,6 +51,32 @@ void broker_event::create(lua_State* L, std::shared_ptr e) { luaL_getmetatable(L, "broker_event"); lua_setmetatable(L, -2); + bool have_to_gc_collect = false; + { + std::lock_guard l(_gc_info_m); + + /*In V2, lua stores only a userdata that contains a shared_ptr of event + * (16 bytes). So garbage collector don't see amount of memory used by + * events. + * So we need to call garbage collector ourselves to reduce memory + * consumption + * So we call at least gc every minute or + * at most every 10s if lua own more than + * com::centreon::broker::multiplexing::muxer::event_queue_max_size() events + * */ + time_t now = time(nullptr); + gc_info& gc_inf = _gc_info[L]; + if ((++gc_inf._broker_event_cpt > com::centreon::broker::multiplexing:: + muxer::event_queue_max_size() && + gc_inf._last_full_gc + 10 < now) || + (gc_inf._last_full_gc + 60 < now)) { + gc_inf._last_full_gc = now; + have_to_gc_collect = true; + } + } + if (have_to_gc_collect) { + lua_gc(L, LUA_GCCOLLECT, 0); + } } static void _message_to_table(lua_State* L, @@ -344,12 +373,18 @@ void broker_event::create_as_table(lua_State* L, const io::data& d) { * * @return 0 */ -static int l_broker_event_destructor(lua_State* L) { +int broker_event::l_broker_event_destructor(lua_State* L) { void* ptr = luaL_checkudata(L, 1, "broker_event"); if (ptr) { auto event = static_cast*>(ptr); event->reset(); + std::lock_guard l(_gc_info_m); + + gc_info& gc_inf = _gc_info[L]; + if (gc_inf._broker_event_cpt > 0) { + --gc_inf._broker_event_cpt; + } } return 0; } @@ -758,3 +793,13 @@ void broker_event::broker_event_reg(lua_State* L) { lua_setglobal(L, name); } + +/** + * @brief when a lua_State is closed we clean _gc_info + * + * @param The Lua interpreter as a lua_State* + */ +void broker_event::lua_close(const lua_State* L) { + std::lock_guard l(_gc_info_m); + _gc_info.erase(L); +} diff --git a/broker/lua/src/luabinding.cc b/broker/lua/src/luabinding.cc index a2ac8750d1e..c47d7ac3a96 100644 --- a/broker/lua/src/luabinding.cc +++ b/broker/lua/src/luabinding.cc @@ -91,6 +91,7 @@ int32_t luabinding::stop() { if (_L) { retval = flush(); lua_close(_L); + broker_event::lua_close(_L); _L = nullptr; } return retval; From 8f13d071d460cb05f1c7d4b17e9450e964e89540 Mon Sep 17 00:00:00 2001 From: sdepassio <114986849+sdepassio@users.noreply.github.com> Date: Fri, 9 Feb 2024 11:17:45 +0100 Subject: [PATCH 3/5] fix(gorgone): package last version of libzmq (#1115) * fix(ci): fix delivery condition (#1108) * add github action for libzmq * fix * fix * update * use centreon-collect github actions for delivery + add bullseye arm64 * try to fix path problem * fix * update * Update .github/workflows/zmq-lib.yml Co-authored-by: Kevin Duret * Update .github/workflows/zmq-lib.yml Co-authored-by: Kevin Duret * Update .github/workflows/zmq-lib.yml Co-authored-by: Kevin Duret * Update .github/workflows/zmq-lib.yml Co-authored-by: Kevin Duret * Update .github/workflows/zmq-lib.yml Co-authored-by: Kevin Duret * Update .github/workflows/zmq-lib.yml Co-authored-by: Kevin Duret --------- Co-authored-by: Kevin Duret --- .github/workflows/zmq-lib.yml | 207 ++++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 .github/workflows/zmq-lib.yml diff --git a/.github/workflows/zmq-lib.yml b/.github/workflows/zmq-lib.yml new file mode 100644 index 00000000000..4f82df8e509 --- /dev/null +++ b/.github/workflows/zmq-lib.yml @@ -0,0 +1,207 @@ +name: libzmq + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/zmq-lib.yml' + push: + branches: + - develop + - dev-[2-9][0-9].[0-9][0-9].x + - master + - "[2-9][0-9].[0-9][0-9].x" + paths: + - '.github/workflows/zmq-lib.yml' + +jobs: + get-version: + uses: ./.github/workflows/get-version.yml + + package-rpm: + needs: [get-version] + + strategy: + fail-fast: false + matrix: + include: + - image: packaging-alma8 + distrib: el8 + arch: amd64 + - image: packaging-alma9 + distrib: el9 + arch: amd64 + + runs-on: ubuntu-22.04 + + container: + image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:${{ needs.get-version.outputs.version }} + credentials: + username: ${{ secrets.DOCKER_REGISTRY_ID }} + password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + + name: package ${{ matrix.distrib }} + + steps: + - name: package rpm + run: | + dnf install -y wget rpmdevtools rpmlint epel-release + dnf config-manager --set-enabled crb || true # alma 9 + dnf config-manager --set-enabled powertools || true # alma 8 + dnf install -y asciidoc autoconf automake gcc gcc-c++ glib2-devel libbsd-devel libtool make xmlto + cd /github/home + wget -O - https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.5.tar.gz|tar zxvf - + mkdir -p /github/home/rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} + cp libzmq-4.3.5/packaging/redhat/zeromq.spec /github/home/rpmbuild/SPECS/ + wget https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz -O /github/home/rpmbuild/SOURCES/zeromq-4.3.5.tar.gz + rpmbuild -bb /github/home/rpmbuild/SPECS/zeromq.spec + cd - + mv /github/home/rpmbuild/RPMS/x86_64/*.rpm ./ + shell: bash + + - name: cache rpm + uses: actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + with: + path: ./*.rpm + key: ${{ github.run_id }}-${{ github.sha }}-rpm-libzmq-${{ matrix.distrib }}-${{ matrix.arch }} + + package-deb: + needs: [get-version] + + strategy: + fail-fast: false + matrix: + include: + - image: packaging-bullseye + distrib: bullseye + runner: ubuntu-22.04 + arch: amd64 + - image: packaging-bookworm + distrib: bookworm + runner: ubuntu-22.04 + arch: amd64 + - image: packaging-nfpm-jammy + distrib: jammy + runner: ubuntu-22.04 + arch: amd64 + - image: packaging-bullseye-arm64 + distrib: bullseye + runner: ["self-hosted", "collect-arm64"] + arch: arm64 + + runs-on: ${{ matrix.runner }} + + container: + image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:${{ needs.get-version.outputs.version }} + credentials: + username: ${{ secrets.DOCKER_REGISTRY_ID }} + password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + + name: package ${{ matrix.distrib }} ${{ matrix.arch }} + + steps: + - name: package deb + run: | + apt-get update + apt-get install -y dpkg-dev wget + apt-get install -y debhelper dh-autoreconf libkrb5-dev libnorm-dev libpgm-dev libsodium-dev libunwind8-dev libnss3-dev libgnutls28-dev libbsd-dev pkg-config asciidoc xmlto + wget -O - https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz|tar zxvf - + cd zeromq-4.3.5 + ./configure + make + make install + cd .. + wget -O - https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.5.tar.gz|tar zxvf - + cd libzmq-4.3.5 + apt-get install -y debhelper dh-autoreconf libkrb5-dev libnorm-dev libpgm-dev libsodium-dev libunwind8-dev libnss3-dev libgnutls28-dev libbsd-dev pkg-config asciidoc xmlto + ln -s packaging/debian + dpkg-buildpackage -us -uc -nc + cd .. + shell: bash + + - name: cache deb + uses: actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + with: + path: ./*.deb + key: ${{ github.run_id }}-${{ github.sha }}-deb-libzmq-${{ matrix.distrib }}-${{ matrix.arch }} + + deliver-rpm: + if: ${{ contains(fromJson('["testing", "unstable"]'), needs.get-version.outputs.stability) }} + needs: [get-version, package-rpm] + environment: ${{ needs.get-version.outputs.environment }} + runs-on: [self-hosted, common] + strategy: + matrix: + include: + - distrib: el8 + arch: amd64 + - distrib: el9 + arch: amd64 + + name: deliver ${{ matrix.distrib }} + + steps: + - name: Publish RPM packages + uses: ./.github/actions/delivery + with: + module_name: libzmq + distrib: ${{ matrix.distrib }} + version: ${{ needs.get-version.outputs.version }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.run_id }}-${{ github.sha }}-rpm-libzmq-${{ matrix.distrib }}-${{ matrix.arch }} + stability: ${{ needs.get-version.outputs.stability }} + + deliver-deb: + if: ${{ contains(fromJson('["testing", "unstable"]'), needs.get-version.outputs.stability) }} + needs: [get-version, package-deb] + environment: ${{ needs.get-version.outputs.environment }} + runs-on: [self-hosted, common] + strategy: + matrix: + include: + - distrib: bullseye + arch: amd64 + - distrib: bullseye + arch: arm64 + - distrib: bookworm + arch: amd64 + - distrib: jammy + arch: amd64 + + name: deliver ${{ matrix.distrib }} + + steps: + - name: Publish DEB packages + uses: ./.github/actions/delivery + with: + module_name: libzmq + distrib: ${{ matrix.distrib }} + version: ${{ needs.get-version.outputs.version }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.run_id }}-${{ github.sha }}-deb-libzmq-${{ matrix.distrib }}-${{ matrix.arch }} + stability: ${{ needs.get-version.outputs.stability }} + + promote: + needs: [get-version] + if: ${{ contains(fromJson('["stable"]'), needs.get-version.outputs.stability) && github.event_name != 'workflow_dispatch' }} + runs-on: [self-hosted, common] + strategy: + matrix: + distrib: [el8, el9, bullseye, bookworm, jammy] + + steps: + - name: Promote ${{ matrix.distrib }} to stable + uses: ./.github/actions/promote-to-stable + with: + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + module_name: libzmq + distrib: ${{ matrix.distrib }} + major_version: ${{ needs.get-version.outputs.version }} + minor_version: ${{ needs.get-version.outputs.patch }} + stability: ${{ needs.get-version.outputs.stability }} + repository_name: standard + github_base_ref: ${{ github.base_ref }} From fe085c6b6233d56c0704542c50ef6b254d689420 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Fri, 9 Feb 2024 12:02:33 +0100 Subject: [PATCH 4/5] fix(ci): fix delivery of libzmq (#1130) * fix(ci): fix delivery of libzmq * fix(ci): fix delivery of libzmq --- .github/workflows/{zmq-lib.yml => libzmq.yml} | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) rename .github/workflows/{zmq-lib.yml => libzmq.yml} (94%) diff --git a/.github/workflows/zmq-lib.yml b/.github/workflows/libzmq.yml similarity index 94% rename from .github/workflows/zmq-lib.yml rename to .github/workflows/libzmq.yml index 4f82df8e509..2f1971043d0 100644 --- a/.github/workflows/zmq-lib.yml +++ b/.github/workflows/libzmq.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: pull_request: paths: - - '.github/workflows/zmq-lib.yml' + - '.github/workflows/libzmq.yml' push: branches: - develop @@ -16,7 +16,7 @@ on: - master - "[2-9][0-9].[0-9][0-9].x" paths: - - '.github/workflows/zmq-lib.yml' + - '.github/workflows/libzmq.yml' jobs: get-version: @@ -145,6 +145,9 @@ jobs: name: deliver ${{ matrix.distrib }} steps: + - name: Checkout sources + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Publish RPM packages uses: ./.github/actions/delivery with: @@ -175,6 +178,9 @@ jobs: name: deliver ${{ matrix.distrib }} steps: + - name: Checkout sources + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Publish DEB packages uses: ./.github/actions/delivery with: @@ -194,6 +200,9 @@ jobs: distrib: [el8, el9, bullseye, bookworm, jammy] steps: + - name: Checkout sources + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Promote ${{ matrix.distrib }} to stable uses: ./.github/actions/promote-to-stable with: From cf7a6cbe9dcd9736a37a12a61ae784a9fb78717f Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Mon, 12 Feb 2024 10:11:48 +0100 Subject: [PATCH 5/5] fix(ci): add distrib name in libzmq deb package (#1133) * fix(ci): add distrib name in libzmq deb package * update version in changelog * update * replace UNRELEASED by distrib name * fix(ci): add distrib name in libzmq deb package * fix(ci): add distrib name in libzmq deb package * fix(ci): add distrib name in libzmq deb package * fix(ci): add distrib name in libzmq deb package * fix(ci): add distrib name in libzmq deb package * fix(ci): add distrib name in libzmq deb package --------- Co-authored-by: Sophie Depassio --- .github/workflows/libzmq.yml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/libzmq.yml b/.github/workflows/libzmq.yml index 2f1971043d0..4fa4309a73d 100644 --- a/.github/workflows/libzmq.yml +++ b/.github/workflows/libzmq.yml @@ -29,10 +29,10 @@ jobs: fail-fast: false matrix: include: - - image: packaging-alma8 + - image: packaging-nfpm-alma8 distrib: el8 arch: amd64 - - image: packaging-alma9 + - image: packaging-nfpm-alma9 distrib: el9 arch: amd64 @@ -52,15 +52,18 @@ jobs: dnf install -y wget rpmdevtools rpmlint epel-release dnf config-manager --set-enabled crb || true # alma 9 dnf config-manager --set-enabled powertools || true # alma 8 - dnf install -y asciidoc autoconf automake gcc gcc-c++ glib2-devel libbsd-devel libtool make xmlto + dnf install -y asciidoc autoconf automake gcc gcc-c++ glib2-devel libbsd-devel libtool make rpm-build xmlto + cd /github/home - wget -O - https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.5.tar.gz|tar zxvf - + wget -O - https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.5.tar.gz | tar zxvf - mkdir -p /github/home/rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} cp libzmq-4.3.5/packaging/redhat/zeromq.spec /github/home/rpmbuild/SPECS/ wget https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz -O /github/home/rpmbuild/SOURCES/zeromq-4.3.5.tar.gz rpmbuild -bb /github/home/rpmbuild/SPECS/zeromq.spec cd - + mv /github/home/rpmbuild/RPMS/x86_64/*.rpm ./ + rm -f zeromq-debugsource-*.rpm libzmq5-debuginfo-*.rpm shell: bash - name: cache rpm @@ -76,11 +79,11 @@ jobs: fail-fast: false matrix: include: - - image: packaging-bullseye + - image: packaging-nfpm-bullseye distrib: bullseye runner: ubuntu-22.04 arch: amd64 - - image: packaging-bookworm + - image: packaging-nfpm-bookworm distrib: bookworm runner: ubuntu-22.04 arch: amd64 @@ -107,20 +110,24 @@ jobs: - name: package deb run: | apt-get update - apt-get install -y dpkg-dev wget - apt-get install -y debhelper dh-autoreconf libkrb5-dev libnorm-dev libpgm-dev libsodium-dev libunwind8-dev libnss3-dev libgnutls28-dev libbsd-dev pkg-config asciidoc xmlto - wget -O - https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz|tar zxvf - + apt-get install -y debhelper dh-autoreconf dpkg-dev libkrb5-dev libnorm-dev libpgm-dev libsodium-dev libunwind8-dev libnss3-dev libgnutls28-dev libbsd-dev pkg-config asciidoc wget xmlto + wget -O - https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz | tar zxvf - + cd zeromq-4.3.5 ./configure make make install cd .. - wget -O - https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.5.tar.gz|tar zxvf - + + wget -O - https://github.com/zeromq/libzmq/archive/refs/tags/v4.3.5.tar.gz | tar zxvf - cd libzmq-4.3.5 - apt-get install -y debhelper dh-autoreconf libkrb5-dev libnorm-dev libpgm-dev libsodium-dev libunwind8-dev libnss3-dev libgnutls28-dev libbsd-dev pkg-config asciidoc xmlto ln -s packaging/debian + sed -Ei 's/([0-9]+.[0-9]+.[0-9]+-[0-9]+.[0-9]+)/\1~${{ matrix.distrib }}/' debian/changelog + sed -Ei 's/UNRELEASED/${{ matrix.distrib }}/' debian/changelog dpkg-buildpackage -us -uc -nc cd .. + + rm -f libzmq5-dbg_*.deb shell: bash - name: cache deb @@ -175,7 +182,7 @@ jobs: - distrib: jammy arch: amd64 - name: deliver ${{ matrix.distrib }} + name: deliver ${{ matrix.distrib }} ${{ matrix.arch }} steps: - name: Checkout sources