From 80b90749e793aaa4f44d5a85cca6a4d218992385 Mon Sep 17 00:00:00 2001 From: pkippes <144150042+pkippes@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:50:59 +0100 Subject: [PATCH 1/6] chore(release) bump collect to 23.04.16 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f09610b19f6..f0fb63da2d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -116,7 +116,7 @@ endif() # Version. set(COLLECT_MAJOR 23) set(COLLECT_MINOR 04) -set(COLLECT_PATCH 15) +set(COLLECT_PATCH 16) set(COLLECT_VERSION "${COLLECT_MAJOR}.${COLLECT_MINOR}.${COLLECT_PATCH}") add_definitions(-DCENTREON_CONNECTOR_VERSION=\"${COLLECT_VERSION}\") From 4d76b99f6657ed81110e80a496885ba90ab17696 Mon Sep 17 00:00:00 2001 From: pkippes <144150042+pkippes@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:51:43 +0100 Subject: [PATCH 2/6] chore(release) bump collect to 23.04.16 --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index 401a5d36575..900e7898621 100644 --- a/.version +++ b/.version @@ -1,2 +1,2 @@ MAJOR=23.04 -MINOR=15 +MINOR=16 From e8a1b98fd59b80894d656068e8b385d927dc6e87 Mon Sep 17 00:00:00 2001 From: tuntoja <58987095+tuntoja@users.noreply.github.com> Date: Thu, 31 Oct 2024 13:52:55 +0100 Subject: [PATCH 3/6] enh(ci): update pull and push jobs in workflows (#1799) (#1807) --- .github/workflows/centreon-collect.yml | 4 ++-- .github/workflows/docker-builder.yml | 4 ++-- .github/workflows/gorgone.yml | 8 ++++---- .github/workflows/libzmq.yml | 8 ++++---- .github/workflows/lua-curl.yml | 4 ++-- .github/workflows/package-collect.yml | 12 ++++++------ .github/workflows/robot-nightly.yml | 8 ++++---- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/centreon-collect.yml b/.github/workflows/centreon-collect.yml index f15e5008ed9..592f34c29c7 100644 --- a/.github/workflows/centreon-collect.yml +++ b/.github/workflows/centreon-collect.yml @@ -71,8 +71,8 @@ jobs: uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 with: registry: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }} - username: ${{ secrets.DOCKER_REGISTRY_ID }} - password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} - name: Test ${{ matrix.image }} uses: ./.github/actions/runner-docker diff --git a/.github/workflows/docker-builder.yml b/.github/workflows/docker-builder.yml index 9dfd004c05d..9fc2bc52d9f 100644 --- a/.github/workflows/docker-builder.yml +++ b/.github/workflows/docker-builder.yml @@ -77,8 +77,8 @@ jobs: uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 with: registry: ${{ vars.DOCKER_PROXY_REGISTRY_URL }} - username: ${{ secrets.DOCKER_REGISTRY_ID }} - password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 diff --git a/.github/workflows/gorgone.yml b/.github/workflows/gorgone.yml index 9cd4782f0b5..1f3874fcac8 100644 --- a/.github/workflows/gorgone.yml +++ b/.github/workflows/gorgone.yml @@ -52,8 +52,8 @@ jobs: veracode_api_id: ${{ secrets.VERACODE_API_ID_GORG }} veracode_api_key: ${{ secrets.VERACODE_API_KEY_GORG }} veracode_srcclr_token: ${{ secrets.VERACODE_SRCCLR_TOKEN }} - docker_registry_id: ${{ secrets.DOCKER_REGISTRY_ID }} - docker_registry_passwd: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + docker_registry_id: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + docker_registry_passwd: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} package: needs: [get-environment] @@ -87,8 +87,8 @@ jobs: commit_hash: ${{ github.sha }} cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-${{ matrix.distrib }} secrets: - registry_username: ${{ secrets.DOCKER_REGISTRY_ID }} - registry_password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + registry_username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + registry_password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} registry_private_username: ${{ secrets.HARBOR_RPM_GPG_SIGNING_REPO_USERNAME }} registry_private_token: ${{ secrets.HARBOR_RPM_GPG_SIGNING_REPO_TOKEN }} diff --git a/.github/workflows/libzmq.yml b/.github/workflows/libzmq.yml index b3bbc15b58d..a72942def05 100644 --- a/.github/workflows/libzmq.yml +++ b/.github/workflows/libzmq.yml @@ -41,8 +41,8 @@ jobs: container: image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:${{ needs.get-environment.outputs.major_version }} credentials: - username: ${{ secrets.DOCKER_REGISTRY_ID }} - password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} name: package ${{ matrix.distrib }} @@ -93,8 +93,8 @@ jobs: container: image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:${{ needs.get-environment.outputs.major_version }} credentials: - username: ${{ secrets.DOCKER_REGISTRY_ID }} - password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} name: package ${{ matrix.distrib }} ${{ matrix.arch }} diff --git a/.github/workflows/lua-curl.yml b/.github/workflows/lua-curl.yml index 889f112c908..96f48d668fc 100644 --- a/.github/workflows/lua-curl.yml +++ b/.github/workflows/lua-curl.yml @@ -59,8 +59,8 @@ jobs: container: image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:${{ needs.get-environment.outputs.img_version }} credentials: - username: ${{ secrets.DOCKER_REGISTRY_ID }} - password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} name: package ${{ matrix.distrib }} ${{ matrix.arch }} diff --git a/.github/workflows/package-collect.yml b/.github/workflows/package-collect.yml index 07d2b389af4..541939cd169 100644 --- a/.github/workflows/package-collect.yml +++ b/.github/workflows/package-collect.yml @@ -43,8 +43,8 @@ jobs: uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 with: registry: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }} - username: ${{ secrets.DOCKER_REGISTRY_ID }} - password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} - name: package rpm ${{ matrix.image }} run: | @@ -64,8 +64,8 @@ jobs: image_name: rpm-signing image_version: latest registry_url: docker.centreon.com/centreon-private - registry_username: ${{ secrets.DOCKER_REGISTRY_ID }} - registry_password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + registry_username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + registry_password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} registry_private_username: ${{ secrets.HARBOR_RPM_GPG_SIGNING_REPO_USERNAME }} registry_private_token: ${{ secrets.HARBOR_RPM_GPG_SIGNING_REPO_TOKEN }} @@ -108,8 +108,8 @@ jobs: uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 with: registry: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }} - username: ${{ secrets.DOCKER_REGISTRY_ID }} - password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} - name: debmake ${{ matrix.image }} uses: ./centreon-collect/.github/actions/runner-docker diff --git a/.github/workflows/robot-nightly.yml b/.github/workflows/robot-nightly.yml index 79fffed6b2c..a0c365af0c8 100644 --- a/.github/workflows/robot-nightly.yml +++ b/.github/workflows/robot-nightly.yml @@ -42,8 +42,8 @@ jobs: veracode_api_id: ${{ secrets.VERACODE_API_ID_COLL }} veracode_api_key: ${{ secrets.VERACODE_API_KEY_COLL }} veracode_srcclr_token: ${{ secrets.VERACODE_SRCCLR_TOKEN }} - docker_registry_id: ${{ secrets.DOCKER_REGISTRY_ID }} - docker_registry_passwd: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + docker_registry_id: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + docker_registry_passwd: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} package: needs: [get-environment] @@ -83,8 +83,8 @@ jobs: database_type: ${{ matrix.database_type }} test_group_name: ${{matrix.test_group_name}} secrets: - registry_username: ${{ secrets.DOCKER_REGISTRY_ID }} - registry_password: ${{ secrets.DOCKER_REGISTRY_PASSWD }} + registry_username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + registry_password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} deliver-rpm: if: ${{ contains(fromJson('["unstable"]'), needs.get-environment.outputs.stability) }} From e0b2664d9c51bbacf5174a39190589afe1fb180f Mon Sep 17 00:00:00 2001 From: jean-christophe81 <98889244+jean-christophe81@users.noreply.github.com> Date: Wed, 6 Nov 2024 08:58:42 +0100 Subject: [PATCH 4/6] fix robot tests (#1831) --- .github/workflows/robot-test.yml | 11 ++++++++++- tests/broker-engine/delete-poller.robot | 2 +- tests/resources/resources.robot | 6 +++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/robot-test.yml b/.github/workflows/robot-test.yml index 76395ade5b0..8a959da6f25 100644 --- a/.github/workflows/robot-test.yml +++ b/.github/workflows/robot-test.yml @@ -114,11 +114,20 @@ jobs: cp tests/output.xml reports/$FILE_PREFIX-output.xml fi + - name: Replace / with - in the feature path + id: feature-path + if: always() + run: | + feature_name="${{ matrix.feature }}" + feature_name_with_dash="${feature_name//\//-}" + echo "Modified Feature Name: $feature_name_with_dash" + echo "feature_name_with_dash=$feature_name_with_dash" >> $GITHUB_OUTPUT + - name: Upload Test Results if: ${{ failure() }} uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 with: - name: reports-${{inputs.test_group_name}} + name: reports-${{inputs.test_group_name}}-${{ steps.feature-path.outputs.feature_name_with_dash }} path: reports retention-days: 1 diff --git a/tests/broker-engine/delete-poller.robot b/tests/broker-engine/delete-poller.robot index 79b4aae8d39..4fbd042a01a 100644 --- a/tests/broker-engine/delete-poller.robot +++ b/tests/broker-engine/delete-poller.robot @@ -669,4 +669,4 @@ EBDP8 *** Keywords *** Ctn Clear Instances Connect To Database pymysql ${DBName} ${DBUser} ${DBPass} ${DBHost} ${DBPort} - ${output} Query DELETE FROM instances + Execute SQL String DELETE FROM instances diff --git a/tests/resources/resources.robot b/tests/resources/resources.robot index 739d5f2780d..9017d7ef2a7 100644 --- a/tests/resources/resources.robot +++ b/tests/resources/resources.robot @@ -212,7 +212,7 @@ Ctn Stop Engine Broker And Save Logs Ctn Get Engine Pid [Arguments] ${process_alias} ${pid}= Get Process Id ${process_alias} - [Return] ${pid} + RETURN ${pid} Ctn Reload Engine ${count}= Ctn Get Engines Count @@ -234,7 +234,7 @@ Ctn Check Connections ${pid2}= Get Process Id b2 Log to console Check Connection 5670 ${pid1} ${pid2} ${retval}= Ctn Check Connection 5670 ${pid1} ${pid2} - [Return] ${retval} + RETURN ${retval} Ctn Disable Eth Connection On Port [Arguments] ${port} @@ -283,7 +283,7 @@ Ctn Wait Or Dump And Kill Process Run Process gcore -o ${ENGINE_LOG}/config0/gcore_${process_name} ${pid} ${result}= Wait For Process ${process_name} timeout=1s on_timeout=kill END - [Return] ${result} + RETURN ${result} Ctn Clear Metrics Connect To Database pymysql ${DBName} ${DBUser} ${DBPass} ${DBHost} ${DBPort} From 0f2848893b76a34bbc5ecbc7d22b363b6ff1bdd4 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Wed, 6 Nov 2024 11:44:11 +0100 Subject: [PATCH 5/6] fix(ci): remove remaining release type condition (#1842) --- .github/actions/delivery/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/delivery/action.yml b/.github/actions/delivery/action.yml index 9b6328723e9..663b1f35549 100644 --- a/.github/actions/delivery/action.yml +++ b/.github/actions/delivery/action.yml @@ -146,7 +146,7 @@ runs: echo "[DEBUG] - stability: ${{ inputs.stability }}" # Make sure all required inputs are NOT empty - if [[ -z "${{ inputs.module_name }}" || -z "${{ inputs.distrib }}" || -z ${{ inputs.stability }} || -z ${{ inputs.major_version }} || -z "${{ inputs.is_cloud }}" || -z "${{ inputs.release_type }}" ]]; then + if [[ -z "${{ inputs.module_name }}" || -z "${{ inputs.distrib }}" || -z ${{ inputs.stability }} || -z ${{ inputs.major_version }} || -z "${{ inputs.is_cloud }}" ]]; then echo "Some mandatory inputs are empty, please check the logs." exit 1 fi From 7cf73e306c1b628aaec77dc037cddeec9a187177 Mon Sep 17 00:00:00 2001 From: David Boucher Date: Fri, 22 Nov 2024 14:45:37 +0100 Subject: [PATCH 6/6] fix(broker): mariadb connection with errno0 (#1888) * fix(broker/mysql): mysql_connection had an issue with last mariadb server REFS: MON-153666 --- broker/core/src/mysql_connection.cc | 27 ++-- .../services-and-bulk-stmt.robot | 143 ++++++++++++++++++ tests/resources/Broker.py | 98 ++++++++++++ 3 files changed, 259 insertions(+), 9 deletions(-) diff --git a/broker/core/src/mysql_connection.cc b/broker/core/src/mysql_connection.cc index cd6f96557ec..0f0502411a1 100644 --- a/broker/core/src/mysql_connection.cc +++ b/broker/core/src/mysql_connection.cc @@ -17,6 +17,7 @@ */ #include +#include #include "com/centreon/broker/config/applier/init.hh" #include "com/centreon/broker/log_v2.hh" @@ -463,18 +464,26 @@ void mysql_connection::_statement(mysql_task* t) { "mysql_connection {:p}: execute statement {} attempt {}: {}", static_cast(this), task->statement_id, attempts, query); if (mysql_stmt_execute(stmt)) { - std::string err_msg( - fmt::format("{} errno={} {}", mysql_error::msg[task->error_code], - ::mysql_errno(_conn), ::mysql_stmt_error(stmt))); - SPDLOG_LOGGER_ERROR(log_v2::sql(), - "connection fail to execute statement {:p}: {}", - static_cast(this), err_msg); - if (_server_error(::mysql_stmt_errno(stmt))) { + int32_t err_code = ::mysql_stmt_errno(stmt); + std::string err_msg(fmt::format("{} errno={} {}", + mysql_error::msg[task->error_code], + err_code, ::mysql_stmt_error(stmt))); + if (err_code == 0) { + SPDLOG_LOGGER_ERROR( + log_v2::sql(), + "mysql_connection: errno=0, so we simulate a server error 1213"); + err_code = CR_SERVER_LOST; + } else { + SPDLOG_LOGGER_ERROR(log_v2::sql(), + "connection fail to execute statement {:p}: {}", + static_cast(this), err_msg); + } + if (_server_error(err_code)) { set_error_message(err_msg); break; } - if (mysql_stmt_errno(stmt) != 1213 && - mysql_stmt_errno(stmt) != 1205) // Dead Lock error + if (err_code != ER_LOCK_DEADLOCK && + err_code != ER_LOCK_WAIT_TIMEOUT) // Dead Lock error attempts = MAX_ATTEMPTS; if (mysql_commit(_conn)) { diff --git a/tests/broker-engine/services-and-bulk-stmt.robot b/tests/broker-engine/services-and-bulk-stmt.robot index 786048f1dd7..3c65bea4c69 100644 --- a/tests/broker-engine/services-and-bulk-stmt.robot +++ b/tests/broker-engine/services-and-bulk-stmt.robot @@ -65,6 +65,7 @@ EBBPS1 IF "${output}" == "((0,),)" BREAK END Should Be Equal As Strings ${output} ((0,),) + Disconnect From Database FOR ${i} IN RANGE ${1000} Ctn Process Service Check Result host_1 service_${i+1} 2 warning${i} @@ -102,6 +103,7 @@ EBBPS1 IF "${output}" == "((0,),)" BREAK END Should Be Equal As Strings ${output} ((0,),) + Disconnect From Database EBBPS2 [Documentation] 1000 service check results are sent to the poller. The test is done with the unified_sql stream, no service status is lost, we find the 1000 results in the database: table services. @@ -148,6 +150,7 @@ EBBPS2 IF "${output}" == "((0,),)" BREAK END Should Be Equal As Strings ${output} ((0,),) + Disconnect From Database FOR ${i} IN RANGE ${1000} Ctn Process Service Check Result host_1 service_${i+1} 2 critical${i} @@ -184,6 +187,7 @@ EBBPS2 IF "${output}" == "((0,),)" BREAK END Should Be Equal As Strings ${output} ((0,),) + Disconnect From Database EBMSSM [Documentation] 1000 services are configured with 100 metrics each. The rrd output is removed from the broker configuration. GetSqlManagerStats is called to measure writes into data_bin. @@ -230,6 +234,7 @@ EBMSSM Sleep 1s END Should Be True ${output[0][0]} >= 100000 + Disconnect From Database EBPS2 [Documentation] 1000 services are configured with 20 metrics each. The rrd output is removed from the broker configuration to avoid to write too many rrd files. While metrics are written in bulk, the database is stopped. This must not crash broker. @@ -449,6 +454,144 @@ Services_and_bulks_${id} ... 2 150 +EBMSSMDBD + [Documentation] 1000 services are configured with 100 metrics each. + ... The rrd output is removed from the broker configuration. + ... While metrics are written in the database, we stop the database and then restart it. + ... Broker must recover its connection to the database and continue to write metrics. + [Tags] broker engine unified_sql MON-153323 + Ctn Clear Metrics + Ctn Config Engine ${1} ${1} ${1000} + # We want all the services to be passive to avoid parasite checks during our test. + Ctn Set Services Passive ${0} service_.* + Ctn Config Broker central + Ctn Config Broker rrd + Ctn Config Broker module ${1} + Ctn Config BBDO3 1 + Ctn Broker Config Log central core error + Ctn Broker Config Log central tcp error + Ctn Broker Config Log central sql debug + Ctn Config Broker Sql Output central unified_sql + Ctn Config Broker Remove Rrd Output central + Ctn Clear Retention + ${start} Get Current Date + Ctn Start Broker + Ctn Start Engine + + Ctn Wait For Engine To Be Ready ${start} 1 + + ${start} Ctn Get Round Current Date + # Let's wait for one "INSERT INTO data_bin" to appear in stats. + Log To Console Many service checks with 100 metrics each are processed. + FOR ${i} IN RANGE ${1000} + Ctn Process Service Check Result With Metrics host_1 service_${i+1} 1 warning${i} 100 + END + + Log To Console We wait for at least one metric to be written in the database. + # Let's wait for all force checks to be in the storage database. + Connect To Database pymysql ${DBName} ${DBUser} ${DBPass} ${DBHost} ${DBPort} + FOR ${i} IN RANGE ${500} + ${output} Query + ... SELECT COUNT(s.last_check) FROM metrics m LEFT JOIN index_data i ON m.index_id = i.id LEFT JOIN services s ON s.host_id = i.host_id AND s.service_id = i.service_id WHERE metric_name LIKE "metric_%%" AND s.last_check >= ${start} + IF ${output[0][0]} >= 1 BREAK + Sleep 1s + END + Disconnect From Database + + Log To Console Let's start some database manipulation... + ${start} Get Current Date + + FOR ${i} IN RANGE ${3} + Ctn Stop Mysql + Sleep 10s + Ctn Start Mysql + ${content} Create List could not insert data in data_bin + ${result} Ctn Find In Log With Timeout ${centralLog} ${start} ${content} 10 + Log To Console ${result} + END + +EBMSSMPART + [Documentation] 1000 services are configured with 100 metrics each. + ... The rrd output is removed from the broker configuration. + ... The data_bin table is configured with two partitions p1 and p2 such + ... that p1 contains old data and p2 contains current data. + ... While metrics are written in the database, we remove the p2 partition. + ... Once the p2 partition is recreated, broker must recover its connection + ... to the database and continue to write metrics. + ... To check that last point, we force a last service check and we check + ... that its metrics are written in the database. + [Tags] broker engine unified_sql MON-153323 + Ctn Clear Metrics + Ctn Config Engine ${1} ${1} ${1000} + # We want all the services to be passive to avoid parasite checks during our test. + Ctn Set Services Passive ${0} service_.* + Ctn Config Broker central + Ctn Config Broker rrd + Ctn Config Broker module ${1} + Ctn Config BBDO3 1 + Ctn Broker Config Log central core error + Ctn Broker Config Log central tcp error + Ctn Broker Config Log central sql trace + Ctn Config Broker Sql Output central unified_sql + Ctn Config Broker Remove Rrd Output central + Ctn Clear Retention + + Ctn Prepare Partitions For Data Bin + ${start} Get Current Date + Ctn Start Broker + Ctn Start Engine + + Ctn Wait For Engine To Be Ready ${start} 1 + + ${start} Ctn Get Round Current Date + # Let's wait for one "INSERT INTO data_bin" to appear in stats. + Log To Console Many service checks with 100 metrics each are processed. + FOR ${i} IN RANGE ${1000} + Ctn Process Service Check Result With Metrics host_1 service_${i+1} 1 warning${i} 100 + END + + Log To Console We wait for at least one metric to be written in the database. + # Let's wait for all force checks to be in the storage database. + Connect To Database pymysql ${DBName} ${DBUser} ${DBPass} ${DBHost} ${DBPort} + FOR ${i} IN RANGE ${500} + ${output} Query + ... SELECT COUNT(s.last_check) FROM metrics m LEFT JOIN index_data i ON m.index_id = i.id LEFT JOIN services s ON s.host_id = i.host_id AND s.service_id = i.service_id WHERE metric_name LIKE "metric_%%" AND s.last_check >= ${start} + IF ${output[0][0]} >= 1 BREAK + Sleep 1s + END + Disconnect From Database + + Log To Console Let's start some database manipulation... + Ctn Remove P2 From Data Bin + ${start} Get Current Date + + ${content} Create List errno= + FOR ${i} IN RANGE ${6} + ${result} Ctn Find In Log With Timeout ${centralLog} ${start} ${content} 10 + IF ${result} BREAK + END + + Log To Console Let's recreate the p2 partition... + Ctn Add P2 To Data Bin + + ${start} Ctn Get Round Current Date + Ctn Process Service Check Result With Metrics host_1 service_1 0 Last Output OK 100 + + Log To Console Let's wait for the last service check to be in the database... + Connect To Database pymysql ${DBName} ${DBUser} ${DBPass} ${DBHost} ${DBPort} + FOR ${i} IN RANGE ${120} + ${output} Query SELECT count(*) FROM data_bin WHERE ctime >= ${start} - 10 + Log To Console ${output} + IF ${output[0][0]} >= 100 BREAK + Sleep 1s + END + Log To Console ${output} + Should Be True ${output[0][0]} >= 100 + Disconnect From Database + + Ctn Init Data Bin Without Partition + + *** Keywords *** Ctn Test Clean Ctn Stop Engine diff --git a/tests/resources/Broker.py b/tests/resources/Broker.py index 6ec19d4dedf..10ec5c81460 100755 --- a/tests/resources/Broker.py +++ b/tests/resources/Broker.py @@ -2566,3 +2566,101 @@ def ctn_broker_get_ba(port: int, ba_id: int, output_file: str, timeout=TIMEOUT): except: logger.console("gRPC server not ready") return res + + +def ctn_prepare_partitions_for_data_bin(): + """ + Create two partitions for the data_bin table. + The first one named p1 contains data with ctime older than now - 60. + The second one named p2 contains data with ctime older than now + 3600. + """ + connection = pymysql.connect(host=DB_HOST, + user=DB_USER, + password=DB_PASS, + database=DB_NAME_STORAGE, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor) + + now = int(time.time()) + before = now - 60 + after = now + 3600 + with connection: + with connection.cursor() as cursor: + cursor.execute("DROP TABLE IF EXISTS data_bin") + sql = f"""CREATE TABLE `data_bin` ( + `id_metric` int(11) DEFAULT NULL, + `ctime` int(11) DEFAULT NULL, + `value` float DEFAULT NULL, + `status` enum('0','1','2','3','4') DEFAULT NULL, + KEY `index_metric` (`id_metric`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 + PARTITION BY RANGE (`ctime`) +(PARTITION `p1` VALUES LESS THAN ({before}) ENGINE = InnoDB, + PARTITION `p2` VALUES LESS THAN ({after}) ENGINE = InnoDB)""" + cursor.execute(sql) + connection.commit() + + +def ctn_remove_p2_from_data_bin(): + """ + Remove the partition p2 from the data_bin table. + """ + connection = pymysql.connect(host=DB_HOST, + user=DB_USER, + password=DB_PASS, + database=DB_NAME_STORAGE, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor) + + with connection: + with connection.cursor() as cursor: + cursor.execute("ALTER TABLE data_bin DROP PARTITION p2") + connection.commit() + + +def ctn_add_p2_to_data_bin(): + """ + Add the partition p2 the the data_bin table. + """ + connection = pymysql.connect(host=DB_HOST, + user=DB_USER, + password=DB_PASS, + database=DB_NAME_STORAGE, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor) + + after = int(time.time()) + 3600 + with connection: + with connection.cursor() as cursor: + cursor.execute( + f"ALTER TABLE data_bin ADD PARTITION (PARTITION p2 VALUES LESS THAN ({after}))") + connection.commit() + + +def ctn_init_data_bin_without_partition(): + """ + Recreate the data_bin table without partition. + """ + connection = pymysql.connect(host=DB_HOST, + user=DB_USER, + password=DB_PASS, + database=DB_NAME_STORAGE, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor) + + now = int(time.time()) + before = now - 60 + after = now + 3600 + with connection: + with connection.cursor() as cursor: + cursor.execute("DROP TABLE IF EXISTS data_bin") + sql = f"""CREATE TABLE `data_bin` ( + `id_metric` int(11) DEFAULT NULL, + `ctime` int(11) DEFAULT NULL, + `value` float DEFAULT NULL, + `status` enum('0','1','2','3','4') DEFAULT NULL, + KEY `index_metric` (`id_metric`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1""" + cursor.execute(sql) + connection.commit() +