diff --git a/apps/mysql.go b/apps/mysql.go index 6221884679..2297375f89 100644 --- a/apps/mysql.go +++ b/apps/mysql.go @@ -175,7 +175,37 @@ var mysqlLegacyReplicationQueries = []sqlReceiverQuery{ metric_name: "mysql.replica.time_behind_source", value_column: "Seconds_Behind_Master", unit: "s", - description: "This field is an indication of how “late” the replica is.", + description: "This field is an indication of how late the replica is.", + data_type: "sum", + monotonic: "false", + value_type: "int", + attribute_columns: []string{}, + static_attributes: map[string]string{}, + }, + }, + }, + // SHOW SLAVE STATUS is deprecated in MySQL 8.0.23, so we need to also somehow query SHOW REPLICA STATUS instead for later version support. + // The downside of this approach is now the receiver always has to make a query that will fail, which is not ideal. + // We should be open to other suggestions if there is an alternative to get this data. + { + query: `SHOW REPLICA STATUS`, + metrics: []sqlReceiverMetric{ + { + metric_name: "mysql.replica.sql_delay", + value_column: "SQL_Delay", + unit: "s", + description: "The number of seconds that the replica must lag the source.", + data_type: "sum", + monotonic: "false", + value_type: "int", + attribute_columns: []string{}, + static_attributes: map[string]string{}, + }, + { + metric_name: "mysql.replica.time_behind_source", + value_column: "Seconds_Behind_Source", + unit: "s", + description: "This field is an indication of how late the replica is.", data_type: "sum", monotonic: "false", value_type: "int", diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/otel.yaml index 444bf8976f..966f9328b5 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux-gpu/otel.yaml @@ -578,7 +578,7 @@ receivers: value_type: int - attribute_columns: [] data_type: sum - description: This field is an indication of how “late” the replica is. + description: This field is an indication of how late the replica is. metric_name: mysql.replica.time_behind_source monotonic: "false" static_attributes: {} @@ -586,6 +586,26 @@ receivers: value_column: Seconds_Behind_Master value_type: int sql: SHOW SLAVE STATUS + - metrics: + - attribute_columns: [] + data_type: sum + description: The number of seconds that the replica must lag the source. + metric_name: mysql.replica.sql_delay + monotonic: "false" + static_attributes: {} + unit: s + value_column: SQL_Delay + value_type: int + - attribute_columns: [] + data_type: sum + description: This field is an indication of how late the replica is. + metric_name: mysql.replica.time_behind_source + monotonic: "false" + static_attributes: {} + unit: s + value_column: Seconds_Behind_Source + value_type: int + sql: SHOW REPLICA STATUS service: pipelines: metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/otel.yaml index a36f5526e6..e9bb9e50bc 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/linux/otel.yaml @@ -547,7 +547,7 @@ receivers: value_type: int - attribute_columns: [] data_type: sum - description: This field is an indication of how “late” the replica is. + description: This field is an indication of how late the replica is. metric_name: mysql.replica.time_behind_source monotonic: "false" static_attributes: {} @@ -555,6 +555,26 @@ receivers: value_column: Seconds_Behind_Master value_type: int sql: SHOW SLAVE STATUS + - metrics: + - attribute_columns: [] + data_type: sum + description: The number of seconds that the replica must lag the source. + metric_name: mysql.replica.sql_delay + monotonic: "false" + static_attributes: {} + unit: s + value_column: SQL_Delay + value_type: int + - attribute_columns: [] + data_type: sum + description: This field is an indication of how late the replica is. + metric_name: mysql.replica.time_behind_source + monotonic: "false" + static_attributes: {} + unit: s + value_column: Seconds_Behind_Source + value_type: int + sql: SHOW REPLICA STATUS service: pipelines: metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/otel.yaml index e25be934f7..dfac3eafac 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows-2012/otel.yaml @@ -622,7 +622,7 @@ receivers: value_type: int - attribute_columns: [] data_type: sum - description: This field is an indication of how “late” the replica is. + description: This field is an indication of how late the replica is. metric_name: mysql.replica.time_behind_source monotonic: "false" static_attributes: {} @@ -630,6 +630,26 @@ receivers: value_column: Seconds_Behind_Master value_type: int sql: SHOW SLAVE STATUS + - metrics: + - attribute_columns: [] + data_type: sum + description: The number of seconds that the replica must lag the source. + metric_name: mysql.replica.sql_delay + monotonic: "false" + static_attributes: {} + unit: s + value_column: SQL_Delay + value_type: int + - attribute_columns: [] + data_type: sum + description: This field is an indication of how late the replica is. + metric_name: mysql.replica.time_behind_source + monotonic: "false" + static_attributes: {} + unit: s + value_column: Seconds_Behind_Source + value_type: int + sql: SHOW REPLICA STATUS windowsperfcounters/iis: collection_interval: 60s perfcounters: diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/otel.yaml index e25be934f7..dfac3eafac 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql/golden/windows/otel.yaml @@ -622,7 +622,7 @@ receivers: value_type: int - attribute_columns: [] data_type: sum - description: This field is an indication of how “late” the replica is. + description: This field is an indication of how late the replica is. metric_name: mysql.replica.time_behind_source monotonic: "false" static_attributes: {} @@ -630,6 +630,26 @@ receivers: value_column: Seconds_Behind_Master value_type: int sql: SHOW SLAVE STATUS + - metrics: + - attribute_columns: [] + data_type: sum + description: The number of seconds that the replica must lag the source. + metric_name: mysql.replica.sql_delay + monotonic: "false" + static_attributes: {} + unit: s + value_column: SQL_Delay + value_type: int + - attribute_columns: [] + data_type: sum + description: This field is an indication of how late the replica is. + metric_name: mysql.replica.time_behind_source + monotonic: "false" + static_attributes: {} + unit: s + value_column: Seconds_Behind_Source + value_type: int + sql: SHOW REPLICA STATUS windowsperfcounters/iis: collection_interval: 60s perfcounters: diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/otel.yaml index 0cf6baf0a7..ee60d992be 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux-gpu/otel.yaml @@ -578,7 +578,7 @@ receivers: value_type: int - attribute_columns: [] data_type: sum - description: This field is an indication of how “late” the replica is. + description: This field is an indication of how late the replica is. metric_name: mysql.replica.time_behind_source monotonic: "false" static_attributes: {} @@ -586,6 +586,26 @@ receivers: value_column: Seconds_Behind_Master value_type: int sql: SHOW SLAVE STATUS + - metrics: + - attribute_columns: [] + data_type: sum + description: The number of seconds that the replica must lag the source. + metric_name: mysql.replica.sql_delay + monotonic: "false" + static_attributes: {} + unit: s + value_column: SQL_Delay + value_type: int + - attribute_columns: [] + data_type: sum + description: This field is an indication of how late the replica is. + metric_name: mysql.replica.time_behind_source + monotonic: "false" + static_attributes: {} + unit: s + value_column: Seconds_Behind_Source + value_type: int + sql: SHOW REPLICA STATUS service: pipelines: metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/otel.yaml index 5043f3a6a5..cdeb215ea1 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/linux/otel.yaml @@ -547,7 +547,7 @@ receivers: value_type: int - attribute_columns: [] data_type: sum - description: This field is an indication of how “late” the replica is. + description: This field is an indication of how late the replica is. metric_name: mysql.replica.time_behind_source monotonic: "false" static_attributes: {} @@ -555,6 +555,26 @@ receivers: value_column: Seconds_Behind_Master value_type: int sql: SHOW SLAVE STATUS + - metrics: + - attribute_columns: [] + data_type: sum + description: The number of seconds that the replica must lag the source. + metric_name: mysql.replica.sql_delay + monotonic: "false" + static_attributes: {} + unit: s + value_column: SQL_Delay + value_type: int + - attribute_columns: [] + data_type: sum + description: This field is an indication of how late the replica is. + metric_name: mysql.replica.time_behind_source + monotonic: "false" + static_attributes: {} + unit: s + value_column: Seconds_Behind_Source + value_type: int + sql: SHOW REPLICA STATUS service: pipelines: metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/otel.yaml index 2ec84b7b9a..eb865641d2 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows-2012/otel.yaml @@ -622,7 +622,7 @@ receivers: value_type: int - attribute_columns: [] data_type: sum - description: This field is an indication of how “late” the replica is. + description: This field is an indication of how late the replica is. metric_name: mysql.replica.time_behind_source monotonic: "false" static_attributes: {} @@ -630,6 +630,26 @@ receivers: value_column: Seconds_Behind_Master value_type: int sql: SHOW SLAVE STATUS + - metrics: + - attribute_columns: [] + data_type: sum + description: The number of seconds that the replica must lag the source. + metric_name: mysql.replica.sql_delay + monotonic: "false" + static_attributes: {} + unit: s + value_column: SQL_Delay + value_type: int + - attribute_columns: [] + data_type: sum + description: This field is an indication of how late the replica is. + metric_name: mysql.replica.time_behind_source + monotonic: "false" + static_attributes: {} + unit: s + value_column: Seconds_Behind_Source + value_type: int + sql: SHOW REPLICA STATUS windowsperfcounters/iis: collection_interval: 60s perfcounters: diff --git a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/otel.yaml index 2ec84b7b9a..eb865641d2 100644 --- a/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-receiver_mysql_missing_endpoint/golden/windows/otel.yaml @@ -622,7 +622,7 @@ receivers: value_type: int - attribute_columns: [] data_type: sum - description: This field is an indication of how “late” the replica is. + description: This field is an indication of how late the replica is. metric_name: mysql.replica.time_behind_source monotonic: "false" static_attributes: {} @@ -630,6 +630,26 @@ receivers: value_column: Seconds_Behind_Master value_type: int sql: SHOW SLAVE STATUS + - metrics: + - attribute_columns: [] + data_type: sum + description: The number of seconds that the replica must lag the source. + metric_name: mysql.replica.sql_delay + monotonic: "false" + static_attributes: {} + unit: s + value_column: SQL_Delay + value_type: int + - attribute_columns: [] + data_type: sum + description: This field is an indication of how late the replica is. + metric_name: mysql.replica.time_behind_source + monotonic: "false" + static_attributes: {} + unit: s + value_column: Seconds_Behind_Source + value_type: int + sql: SHOW REPLICA STATUS windowsperfcounters/iis: collection_interval: 60s perfcounters: diff --git a/integration_test/third_party_apps_test/applications/mysql/centos_rhel/install b/integration_test/third_party_apps_test/applications/mysql/centos_rhel/install index 5e66f63f26..0af9a69fdc 100644 --- a/integration_test/third_party_apps_test/applications/mysql/centos_rhel/install +++ b/integration_test/third_party_apps_test/applications/mysql/centos_rhel/install @@ -5,7 +5,7 @@ VERSION_ID=${VERSION_ID%%.*} sudo rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 sudo rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2022 -sudo yum -y install https://repo.mysql.com/mysql80-community-release-el${VERSION_ID}.rpm +sudo yum -y install https://repo.mysql.com/mysql84-community-release-el${VERSION_ID}.rpm if [ ${VERSION_ID} == 8 ]; then sudo yum -y module disable mysql @@ -14,6 +14,8 @@ sudo yum -y install mysql-community-server echo "default-authentication-plugin=mysql_native_password" | sudo tee -a /etc/my.cnf +# !!!Even though everything appears to install OK, service will not start ATM +# For now not an issue until we actually enable mysql8.4 tests for RHEL family sudo service mysqld start password=$(sudo grep -oP '(?<=temporary password is generated for root@localhost: ).*$' /var/log/mysqld.log) diff --git a/integration_test/third_party_apps_test/applications/mysql/debian_ubuntu/install b/integration_test/third_party_apps_test/applications/mysql/debian_ubuntu/install index 4690c8290c..649de5b2e4 100644 --- a/integration_test/third_party_apps_test/applications/mysql/debian_ubuntu/install +++ b/integration_test/third_party_apps_test/applications/mysql/debian_ubuntu/install @@ -3,8 +3,8 @@ set -e sudo apt update sudo apt install -y wget -wget --no-verbose https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb -sudo dpkg -i mysql-apt-config_0.8.29-1_all.deb +wget --no-verbose https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb +sudo dpkg -i mysql-apt-config_0.8.32-1_all.deb sudo apt update sudo debconf-set-selections <<< 'mysql-community-server mysql-server/default-auth-override select Use Legacy Authentication Method (Retain MySQL 5.x Compatibility)' diff --git a/integration_test/third_party_apps_test/applications/mysql/exercise b/integration_test/third_party_apps_test/applications/mysql/exercise index 9343a2cf5e..29bb29034d 100644 --- a/integration_test/third_party_apps_test/applications/mysql/exercise +++ b/integration_test/third_party_apps_test/applications/mysql/exercise @@ -30,7 +30,7 @@ sudo mysql --defaults-extra-file=config-user -S /var/run/mysqld/mysql2.sock -Bse sudo mysqldump --defaults-extra-file=config-user -S /var/run/mysqld/mysql2.sock --all-databases --source-data > /tmp/dbdump.sql # Capture status to seed replica -raw_status=$(sudo mysql --defaults-extra-file=config-user -S /var/run/mysqld/mysql2.sock -Bse "SHOW MASTER STATUS;") +raw_status=$(sudo mysql --defaults-extra-file=config-user -S /var/run/mysqld/mysql2.sock -Bse "SHOW BINARY LOG STATUS;") read -r logfile logpos <<< $raw_status # Dump data from source into replica @@ -46,4 +46,4 @@ sudo mysql --defaults-extra-file=config-user -Bse "SET GLOBAL long_query_time = sudo mysql --defaults-extra-file=config-user -Bse "SET GLOBAL slow_query_log = 1" sudo mysql --defaults-extra-file=config-user -Bse "SET GLOBAL general_log = 'ON'" -sudo mysql --defaults-extra-file=config-user -Bse "select table_catalog, table_schema, table_name from information_schema.tables" \ No newline at end of file +sudo mysql --defaults-extra-file=config-user -Bse "select table_catalog, table_schema, table_name from information_schema.tables" diff --git a/integration_test/third_party_apps_test/applications/mysql/metadata.yaml b/integration_test/third_party_apps_test/applications/mysql/metadata.yaml index 08322f765a..f4f390d7cf 100644 --- a/integration_test/third_party_apps_test/applications/mysql/metadata.yaml +++ b/integration_test/third_party_apps_test/applications/mysql/metadata.yaml @@ -33,10 +33,22 @@ minimum_supported_agent_version: supported_operating_systems: linux platforms_to_skip: # MySQL is not currently supported on various distros. + - ml-images:common-gpu-debian-11-py310 + - debian-cloud:debian-11 - debian-cloud:debian-12 - debian-cloud:debian-12-arm64 + - rocky-linux-cloud:rocky-linux-8 + - rocky-linux-cloud:rocky-linux-9 + - rocky-linux-cloud:rocky-linux-9-arm64 + - ubuntu-os-cloud:ubuntu-2004-lts + - ubuntu-os-cloud:ubuntu-2004-lts-arm64 + - ubuntu-os-cloud:ubuntu-2204-lts + - ubuntu-os-cloud:ubuntu-2204-lts-arm64 + - ubuntu-os-cloud:ubuntu-2404-lts-amd64 + - ubuntu-os-cloud:ubuntu-2404-lts-arm64 + - suse-cloud:sles-12 - suse-cloud:sles-15-arm64 -supported_app_version: ["5.7", "8.0"] +supported_app_version: ["5.7", "8.0", "8.4"] expected_metrics: - type: workload.googleapis.com/mysql.buffer_pool_data_pages value_type: INT64 diff --git a/integration_test/third_party_apps_test/applications/mysql/sles/install b/integration_test/third_party_apps_test/applications/mysql/sles/install index d619965cc5..e462cd7a73 100644 --- a/integration_test/third_party_apps_test/applications/mysql/sles/install +++ b/integration_test/third_party_apps_test/applications/mysql/sles/install @@ -3,52 +3,40 @@ set -e source /etc/os-release SUSE_VERSION="${VERSION_ID%%.*}" -if [[ "${ID}" == opensuse-leap && "${VERSION_ID}" == 15.[01] ]]; then - if [[ "${VERSION_ID}" == 15.0 ]]; then - sudo zypper modifyrepo --disable openSUSE-Leap-Cloud-Tools - elif [[ "${VERSION_ID}" == 15.1 ]]; then - sudo zypper modifyrepo --disable openSUSE-Leap-devel-languages-python-aws - fi - sudo zypper -n refresh - - sudo zypper -n install mariadb -else - mysql_repo_pkg_name='' - if [[ "${SUSE_VERSION}" == 12 ]]; then - mysql_repo_pkg_name=mysql80-community-release-sles12.rpm - elif [[ "${SUSE_VERSION}" == 15 ]]; then - mysql_repo_pkg_name=mysql80-community-release-sl15.rpm - fi - +if [[ "${ID}" == sles ]]; then # Installation followed in: https://dev.mysql.com/doc/mysql-sles-repo-quick-guide/en/ sudo rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 sudo rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2022 - sudo zypper -n install https://dev.mysql.com/get/${mysql_repo_pkg_name} + sudo zypper -n install https://dev.mysql.com/get/mysql84-community-release-sl15.rpm sudo zypper -n install mysql-community-server fi -if [[ "${SUSE_VERSION}" == 12 ]]; then - sudo zypper -n install libmysqlclient18 -elif [[ "${SUSE_VERSION}" == 15 ]]; then +if [[ "${SUSE_VERSION}" == 15 ]]; then sudo zypper -n install libmariadb3 fi +## Default configuration +cat < /dev/null +[mysqld] +mysql_native_password=ON +datadir=/var/lib/mysql +socket=/var/lib/mysql/mysql.sock + +log-error=/var/log/mysql/mysqld.log +pid-file=/var/run/mysql/mysqld.pid +EOF + sudo service mysql start -if [[ "${ID}" == opensuse-leap && "${VERSION_ID}" == 15.[01] ]]; then - # mariadb default root password is empty. - mysql -u root --skip-password -Bse "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'Ss123%321'; FLUSH PRIVILEGES;" -else - password="$(sudo grep -oP '(?<=temporary password is generated for root@localhost: ).*$' /var/log/mysql/mysqld.log)" +password="$(sudo grep -oP '(?<=temporary password is generated for root@localhost: ).*$' /var/log/mysql/mysqld.log)" - cat <config-user - [client] - user=root - password='$password' +cat <config-user +[client] +user=root +password='$password' EOF - mysql --defaults-extra-file=config-user -Bse "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'Ss123%321'; FLUSH PRIVILEGES;" --connect-expired-password -fi +mysql --defaults-extra-file=config-user -Bse "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'Ss123%321'; FLUSH PRIVILEGES;" --connect-expired-password # default socket path is different from originally set default (from debian/ubuntu), so we need to create a symlink sudo mkdir /var/run/mysqld/ diff --git a/integration_test/third_party_apps_test/applications/mysql5.7/metadata.yaml b/integration_test/third_party_apps_test/applications/mysql5.7/metadata.yaml index 81733ceea7..6ea3daa44f 100644 --- a/integration_test/third_party_apps_test/applications/mysql5.7/metadata.yaml +++ b/integration_test/third_party_apps_test/applications/mysql5.7/metadata.yaml @@ -55,7 +55,7 @@ platforms_to_skip: - ubuntu-os-cloud:ubuntu-2204-lts-arm64 - ubuntu-os-cloud:ubuntu-2404-lts-amd64 - ubuntu-os-cloud:ubuntu-2404-lts-arm64 -supported_app_version: ["5.7", "8.0"] +supported_app_version: ["5.7", "8.0", "8.4"] expected_metrics: - type: workload.googleapis.com/mysql.buffer_pool_data_pages value_type: INT64 diff --git a/integration_test/third_party_apps_test/applications/mysql8.0/centos_rhel/install b/integration_test/third_party_apps_test/applications/mysql8.0/centos_rhel/install new file mode 100644 index 0000000000..5e66f63f26 --- /dev/null +++ b/integration_test/third_party_apps_test/applications/mysql8.0/centos_rhel/install @@ -0,0 +1,60 @@ +set -e + +source /etc/os-release +VERSION_ID=${VERSION_ID%%.*} + +sudo rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 +sudo rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2022 +sudo yum -y install https://repo.mysql.com/mysql80-community-release-el${VERSION_ID}.rpm + +if [ ${VERSION_ID} == 8 ]; then + sudo yum -y module disable mysql +fi +sudo yum -y install mysql-community-server + +echo "default-authentication-plugin=mysql_native_password" | sudo tee -a /etc/my.cnf + +sudo service mysqld start + +password=$(sudo grep -oP '(?<=temporary password is generated for root@localhost: ).*$' /var/log/mysqld.log) + +cat < config-user +[client] +user=root +password='$password' +EOF + +mysql --defaults-extra-file=config-user -Bse "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'Ss123%321'; FLUSH PRIVILEGES;" --connect-expired-password + +# default socket path is different from originally set default (from debian/ubuntu), so we need to create a symlink +sudo ln -s /var/lib/mysql/mysql.sock /var/run/mysqld/mysqld.sock + +sudo mkdir /var/log/mysql +sudo chown mysql:mysql /var/log/mysql + +# Required to allow mysql to accept connections on 3307 +if [ ${ID} == rocky ]; then + sudo yum install -y policycoreutils-python-utils +else + sudo yum install -y policycoreutils-python +fi +sudo semanage port --add --type mysqld_port_t --proto tcp 3307 + +# set up replication source to validate replica metrics +# set main target (replica) to use binary logging for replication to work +sudo tee -a /etc/my.cnf >/dev/null </dev/null < /dev/null << EOF +metrics: + receivers: + mysql: + type: mysql + username: root + password: Ss123%321 + service: + pipelines: + mysql: + receivers: + - mysql +logging: + receivers: + mysql_error: + type: mysql_error + mysql_general: + type: mysql_general + mysql_slow: + type: mysql_slow + service: + pipelines: + mysql: + receivers: + - mysql_error + - mysql_general + - mysql_slow +EOF + +sudo service google-cloud-ops-agent restart +sleep 60 diff --git a/integration_test/third_party_apps_test/applications/mysql8.0/exercise b/integration_test/third_party_apps_test/applications/mysql8.0/exercise new file mode 100644 index 0000000000..9343a2cf5e --- /dev/null +++ b/integration_test/third_party_apps_test/applications/mysql8.0/exercise @@ -0,0 +1,49 @@ +set -e + +cat < config-user +[client] +user=root +password='Ss123%321' +EOF + +# initialize and start replation source +sudo mysqld --defaults-group-suffix=2 --initialize +nohup sudo mysqld --defaults-group-suffix=2 2>/dev/null >/dev/null config-user-2 +[client] +user=root +password='$password' +EOF + +sudo mysql --defaults-extra-file=config-user-2 -S /var/run/mysqld/mysql2.sock -Bse "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'Ss123%321'; FLUSH PRIVILEGES;" --connect-expired-password + +# Create replication user +sudo mysql --defaults-extra-file=config-user -S /var/run/mysqld/mysql2.sock -Bse "CREATE USER 'repl'@'localhost' IDENTIFIED WITH mysql_native_password BY 'password';" +sudo mysql --defaults-extra-file=config-user -S /var/run/mysqld/mysql2.sock -Bse "GRANT REPLICATION SLAVE ON *.* TO 'repl'@'localhost';" + +# Dump data from replica source +sudo mysqldump --defaults-extra-file=config-user -S /var/run/mysqld/mysql2.sock --all-databases --source-data > /tmp/dbdump.sql + +# Capture status to seed replica +raw_status=$(sudo mysql --defaults-extra-file=config-user -S /var/run/mysqld/mysql2.sock -Bse "SHOW MASTER STATUS;") +read -r logfile logpos <<< $raw_status + +# Dump data from source into replica +sudo mysql --defaults-extra-file=config-user < /tmp/dbdump.sql + +# Configure replication +sudo mysql --defaults-extra-file=config-user -Bse "STOP REPLICA;" +sudo mysql --defaults-extra-file=config-user -Bse "CHANGE REPLICATION SOURCE TO SOURCE_HOST='localhost', SOURCE_USER='repl', SOURCE_PORT=3307, SOURCE_PASSWORD='password', SOURCE_LOG_FILE='$logfile', SOURCE_LOG_POS=$logpos;" +sudo mysql --defaults-extra-file=config-user -Bse "START REPLICA;" + +sudo mysql --defaults-extra-file=config-user -Bse "SET GLOBAL log_slow_extra = 'ON'" +sudo mysql --defaults-extra-file=config-user -Bse "SET GLOBAL long_query_time = 0" +sudo mysql --defaults-extra-file=config-user -Bse "SET GLOBAL slow_query_log = 1" +sudo mysql --defaults-extra-file=config-user -Bse "SET GLOBAL general_log = 'ON'" + +sudo mysql --defaults-extra-file=config-user -Bse "select table_catalog, table_schema, table_name from information_schema.tables" \ No newline at end of file diff --git a/integration_test/third_party_apps_test/applications/mysql8.0/features.yaml b/integration_test/third_party_apps_test/applications/mysql8.0/features.yaml new file mode 100644 index 0000000000..ac777917e4 --- /dev/null +++ b/integration_test/third_party_apps_test/applications/mysql8.0/features.yaml @@ -0,0 +1,17 @@ +features: +- feature: receivers:mysql + module: metrics + key: "[0].enabled" + value: true +- feature: receivers:mysql_error + module: logging + key: "[0].enabled" + value: true +- feature: receivers:mysql_general + module: logging + key: "[1].enabled" + value: true +- feature: receivers:mysql_slow + module: logging + key: "[2].enabled" + value: true diff --git a/integration_test/third_party_apps_test/applications/mysql8.0/metadata.yaml b/integration_test/third_party_apps_test/applications/mysql8.0/metadata.yaml new file mode 100644 index 0000000000..fb27489f1e --- /dev/null +++ b/integration_test/third_party_apps_test/applications/mysql8.0/metadata.yaml @@ -0,0 +1,457 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +public_url: "https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/third-party/mysql" +app_url: "http://www.mysql.com/" +short_name: MySQL +long_name: MySQL +logo_path: /images/launcher/logos/mysql.png # supplied by google technical writer +description: |- + The MySQL integration collects performance metrics related to + InnoDB, the buffer pool, and various other operations. It also collects general, + error, and slow-query logs and parses them into a JSON payload. Error logs are + parsed for their error code and subsystem. Slow-query logs are parsed into + key-value pairs that describe the performance of a query, including query time + and rows examined. +configure_integration: |- + The `mysql` receiver connects by default to a local MySQL + server using a Unix socket and Unix authentication as the `root` user. +minimum_supported_agent_version: + metrics: 2.32.0 + logging: 2.5.0 +supported_operating_systems: linux +platforms_to_skip: + # MySQL8.0 is not currently supported on various distros. + - debian-cloud:debian-12 + - debian-cloud:debian-12-arm64 + - suse-cloud:sles-15-arm64 +supported_app_version: ["5.7", "8.0", "8.4"] +expected_metrics: + - type: workload.googleapis.com/mysql.buffer_pool_data_pages + value_type: INT64 + kind: GAUGE + monitored_resources: [gce_instance] + labels: + - name: status + value_regex: dirty|clean + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.buffer_pool_limit + value_type: INT64 + kind: GAUGE + monitored_resources: [gce_instance] + labels: + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.buffer_pool_operations + value_type: INT64 + kind: CUMULATIVE + monitored_resources: [gce_instance] + labels: + - name: operation + value_regex: read_ahead|read_ahead_evicted|read_ahead_rnd|read_requests|reads|wait_free|write_requests + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.buffer_pool_page_flushes + value_type: INT64 + kind: CUMULATIVE + monitored_resources: [gce_instance] + labels: + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.buffer_pool_pages + value_type: DOUBLE + kind: GAUGE + monitored_resources: [gce_instance] + labels: + - name: kind + value_regex: data|free|misc + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.buffer_pool_size + value_type: DOUBLE + kind: GAUGE + monitored_resources: [gce_instance] + labels: + - name: kind + value_regex: clean|dirty + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.commands + value_type: INT64 + kind: CUMULATIVE + monitored_resources: [gce_instance] + labels: + - name: command + value_regex: delete|insert|select|update + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.double_writes + value_type: INT64 + kind: CUMULATIVE + monitored_resources: [gce_instance] + labels: + - name: kind + value_regex: writes|pages_written + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.handlers + value_type: INT64 + kind: CUMULATIVE + monitored_resources: [gce_instance] + labels: + - name: kind + value_regex: commit|delete|discover|lock|mrr_init|prepare|read_first|read_key|read_last|read_next|read_prev|read_rnd|read_rnd_next|rollback|savepoint|savepoint_rollback|update|write + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.locks + value_type: INT64 + kind: CUMULATIVE + monitored_resources: [gce_instance] + labels: + - name: kind + value_regex: immediate|waited + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.log_operations + value_type: INT64 + kind: CUMULATIVE + monitored_resources: [gce_instance] + labels: + - name: operation + value_regex: requests|waits|writes + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.operations + value_type: INT64 + kind: CUMULATIVE + monitored_resources: [gce_instance] + labels: + - name: operation + value_regex: fsyncs|reads|writes + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.page_operations + value_type: INT64 + kind: CUMULATIVE + monitored_resources: [gce_instance] + labels: + - name: operation + value_regex: created|read|written + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.replica.time_behind_source + value_type: INT64 + kind: GAUGE + monitored_resources: [gce_instance] + labels: + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.replica.sql_delay + value_type: INT64 + kind: GAUGE + monitored_resources: [gce_instance] + labels: + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.row_locks + value_type: INT64 + kind: CUMULATIVE + monitored_resources: [gce_instance] + labels: + - name: kind + value_regex: time|waits + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.row_operations + value_type: INT64 + kind: CUMULATIVE + monitored_resources: [gce_instance] + labels: + - name: operation + value_regex: deleted|inserted|read|updated + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.sorts + value_type: INT64 + kind: CUMULATIVE + monitored_resources: [gce_instance] + labels: + - name: kind + value_regex: merge_passes|range|rows|scan + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + - type: workload.googleapis.com/mysql.threads + value_type: DOUBLE + kind: GAUGE + monitored_resources: [gce_instance] + labels: + - name: kind + value_regex: cached|connected|created|running + - name: instrumentation_source + value_regex: agent.googleapis.com/mysql + representative: true +expected_logs: + - log_name: mysql_error + fields: + - name: jsonPayload.level + value_regex: System + type: string + description: 'Log entry level' + optional: true + - name: jsonPayload.message + value_regex: .*InnoDB initialization has started.* + type: string + description: 'Log message' + - name: jsonPayload.subsystem + value_regex: InnoDB + type: string + description: 'MySQL subsystem where the log originated' + - name: jsonPayload.tid + type: number + description: 'Thread ID where the log originated' + - name: jsonPayload.errorCode + type: string + description: 'MySQL error code associated with the log' + - name: severity + type: string + description: '' + - log_name: mysql_general + fields: + - name: jsonPayload.message + value_regex: .*select table_catalog, table_schema, table_name from information_schema.tables.* + type: string + description: 'Log of the database action' + - name: jsonPayload.command + value_regex: Query + type: string + description: 'Type of database action being logged' + - name: jsonPayload.tid + type: number + description: 'Thread ID where the log originated' + - name: severity + type: string + description: '' + - log_name: mysql_slow + fields: + - name: jsonPayload.message + value_regex: (?s).*select table_catalog, table_schema, table_name from information_schema.tables.* # The (?s) part will make the . match with newline as well. See https://github.com/google/re2/blob/main/doc/syntax.txt#L65,L68 + type: string + description: 'Full text of the query' + - name: jsonPayload.user + value_regex: root + type: string + description: 'User that executed the query' + - name: jsonPayload.database + value_regex: root + type: string + description: 'Database where the query was executed' + - name: jsonPayload.host + value_regex: localhost + type: string + description: 'Host of the database instance' + - name: jsonPayload.ipAddress + type: string + description: 'Address of the database instance' + optional: true + - name: jsonPayload.tid + type: number + description: 'Thread ID where the query was logged' + - name: jsonPayload.queryTime + type: number + description: 'The statement execution time in seconds' + - name: jsonPayload.lockTime + type: number + description: 'The time to acquire locks in seconds' + - name: jsonPayload.rowsSent + type: number + description: 'The number of rows sent to the client' + - name: jsonPayload.rowsExamined + type: number + description: 'The number of rows examined by the server layer' + - name: jsonPayload.errorNumber + type: number + description: 'The statement error number, or 0 if no error occurred' + notes: + - &log_slow_extra_note + These fields are only provided if the `log_slow_extra` system variable is set to `'ON'`. This variable is available as of MySQL 8.0.14. + - name: jsonPayload.killed + type: number + description: 'If the statement was terminated, the error number indicating why, or 0 if the statement terminated normally' + notes: + - *log_slow_extra_note + - name: jsonPayload.bytesReceived + type: number + description: 'The number of bytes received from all clients' + notes: + - *log_slow_extra_note + - name: jsonPayload.bytesSent + type: number + description: 'The number of bytes sent to all clients' + notes: + - *log_slow_extra_note + - name: jsonPayload.readFirst + type: number + description: 'The number of times the first entry in an index was read' + notes: + - *log_slow_extra_note + - name: jsonPayload.readLast + type: number + description: 'The number of requests to read the last key in an index' + notes: + - *log_slow_extra_note + - name: jsonPayload.readKey + type: number + description: 'The number of requests to read a row based on a key' + notes: + - *log_slow_extra_note + - name: jsonPayload.readNext + type: number + description: 'The number of requests to read the next row in key order' + notes: + - *log_slow_extra_note + - name: jsonPayload.readPrev + type: number + description: 'The number of requests to read the previous row in key order' + notes: + - *log_slow_extra_note + - name: jsonPayload.readRnd + type: number + description: 'The number of requests to read a row based on a fixed position' + notes: + - *log_slow_extra_note + - name: jsonPayload.readRndNext + type: number + description: 'The number of requests to read the next row in the data file' + notes: + - *log_slow_extra_note + - name: jsonPayload.sortMergePasses + type: number + description: 'The number of merge passes that the sort algorithm has had to do' + notes: + - *log_slow_extra_note + - name: jsonPayload.sortRangeCount + type: number + description: 'The number of sorts that were done using ranges' + notes: + - *log_slow_extra_note + - name: jsonPayload.sortRows + type: number + description: 'The number of sorted rows' + notes: + - *log_slow_extra_note + - name: jsonPayload.sortScanCount + type: number + description: 'The number of sorts that were done by scanning the table' + notes: + - *log_slow_extra_note + - name: jsonPayload.createdTmpDiskTables + type: number + description: 'The number of internal on-disk temporary tables created by the server' + notes: + - *log_slow_extra_note + - name: jsonPayload.createdTmpTables + type: number + description: 'The number of internal temporary tables created by the server' + notes: + - *log_slow_extra_note + - name: jsonPayload.startTime + type: string + description: 'The statement execution start time' + notes: + - *log_slow_extra_note + - name: jsonPayload.endTime + type: string + description: 'The statement execution end time' + notes: + - *log_slow_extra_note + - name: severity + type: string + description: '' + - name: jsonPayload.level + type: string + description: Log entry level + optional: true + notes: + - For information about these fields, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.0/en/slow-query-log.html). +configuration_options: + logs: + - type: mysql_error + fields: + - name: type + default: null + description: This value must be `mysql_error`. + - name: include_paths + default: '[/var/log/mysqld.log, /var/log/mysql/mysqld.log, /var/log/mysql/error.log]' + description: A list of filesystem paths to read by tailing each file. A wild card (`*`) can be used in the paths; for example, `/var/log/mysql/*.log`. + - name: exclude_paths + default: null + description: A list of filesystem path patterns to exclude from the set matched by `include_paths`. + - name: record_log_file_path + default: false + description: If set to `true`, then the path to the specific file from which the log record was obtained appears in the output log entry as the value of the `agent.googleapis.com/log_file_path` label. When using a wildcard, only the path of the file from which the record was obtained is recorded. + - name: wildcard_refresh_interval + default: 60s + description: The interval at which wildcard file paths in `include_paths` are refreshed. Given as a [time duration](https://pkg.go.dev/time#ParseDuration), for example `30s` or `2m`. This property might be useful under high logging throughputs where log files are rotated faster than the default interval. + - type: mysql_general + fields: + - name: type + default: null + description: This value must be `mysql_general`. + - name: include_paths + default: '[/var/lib/mysql/${HOSTNAME}.log]' + description: A list of filesystem paths to read by tailing each file. A wild card (`*`) can be used in the paths. + - name: exclude_paths + default: null + description: A list of filesystem path patterns to exclude from the set matched by `include_paths`. + - name: record_log_file_path + default: false + description: If set to `true`, then the path to the specific file from which the log record was obtained appears in the output log entry as the value of the `agent.googleapis.com/log_file_path` label. When using a wildcard, only the path of the file from which the record was obtained is recorded. + - name: wildcard_refresh_interval + default: 60s + description: The interval at which wildcard file paths in `include_paths` are refreshed. Given as a [time duration](https://pkg.go.dev/time#ParseDuration), for example `30s` or `2m`. This property might be useful under high logging throughputs where log files are rotated faster than the default interval. + - type: mysql_slow + fields: + - name: type + default: null + description: This value must be `mysql_slow`. + - name: include_paths + default: '[/var/lib/mysql/${HOSTNAME}-slow.log]' + description: A list of filesystem paths to read by tailing each file. A wild card (`*`) can be used in the paths. + - name: exclude_paths + default: null + description: A list of filesystem path patterns to exclude from the set matched by `include_paths`. + - name: record_log_file_path + default: false + description: If set to `true`, then the path to the specific file from which the log record was obtained appears in the output log entry as the value of the `agent.googleapis.com/log_file_path` label. When using a wildcard, only the path of the file from which the record was obtained is recorded. + - name: wildcard_refresh_interval + default: 60s + description: The interval at which wildcard file paths in `include_paths` are refreshed. Given as a [time duration](https://pkg.go.dev/time#ParseDuration), for example `30s` or `2m`. This property might be useful under high logging throughputs where log files are rotated faster than the default interval. + metrics: + - type: mysql + fields: + - name: type + default: null + description: This value must be `mysql`. + - name: endpoint + default: /var/run/mysqld/mysqld.sock + description: The `hostname:port` or Unix socket path starting with `/` used to connect to the MySQL server. + - name: username + default: root + description: The username used to connect to the server. + - name: password + default: null + description: The password used to connect to the server. + - name: collection_interval + default: 60s + description: A [time duration](https://pkg.go.dev/time#ParseDuration) value, such as `30s` or `5m`. diff --git a/integration_test/third_party_apps_test/applications/mysql8.0/sles/install b/integration_test/third_party_apps_test/applications/mysql8.0/sles/install new file mode 100644 index 0000000000..8e0200732f --- /dev/null +++ b/integration_test/third_party_apps_test/applications/mysql8.0/sles/install @@ -0,0 +1,80 @@ +set -e + +source /etc/os-release +SUSE_VERSION="${VERSION_ID%%.*}" + +if [[ "${ID}" == opensuse-leap && "${VERSION_ID}" == 15.[01] ]]; then + if [[ "${VERSION_ID}" == 15.0 ]]; then + sudo zypper modifyrepo --disable openSUSE-Leap-Cloud-Tools + elif [[ "${VERSION_ID}" == 15.1 ]]; then + sudo zypper modifyrepo --disable openSUSE-Leap-devel-languages-python-aws + fi + sudo zypper -n refresh + + sudo zypper -n install mariadb +else + mysql_repo_pkg_name='' + if [[ "${SUSE_VERSION}" == 12 ]]; then + mysql_repo_pkg_name=mysql80-community-release-sles12.rpm + elif [[ "${SUSE_VERSION}" == 15 ]]; then + mysql_repo_pkg_name=mysql84-community-release-sl15.rpm + fi + + # Installation followed in: https://dev.mysql.com/doc/mysql-sles-repo-quick-guide/en/ + sudo rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 + sudo rpm --import https://repo.mysql.com/RPM-GPG-KEY-mysql-2022 + sudo zypper -n install https://dev.mysql.com/get/${mysql_repo_pkg_name} + if [[ "${SUSE_VERSION}" == 15 ]]; then + sudo zypper modifyrepo -d mysql-8.4-lts-community + sudo zypper modifyrepo -e mysql80-community + fi + sudo zypper -n install mysql-community-server +fi + +if [[ "${SUSE_VERSION}" == 12 ]]; then + sudo zypper -n install libmysqlclient18 +elif [[ "${SUSE_VERSION}" == 15 ]]; then + sudo zypper -n install libmariadb3 +fi + +sudo service mysql start + +if [[ "${ID}" == opensuse-leap && "${VERSION_ID}" == 15.[01] ]]; then + # mariadb default root password is empty. + mysql -u root --skip-password -Bse "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'Ss123%321'; FLUSH PRIVILEGES;" +else + password="$(sudo grep -oP '(?<=temporary password is generated for root@localhost: ).*$' /var/log/mysql/mysqld.log)" + + cat <config-user + [client] + user=root + password='$password' +EOF + + mysql --defaults-extra-file=config-user -Bse "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'Ss123%321'; FLUSH PRIVILEGES;" --connect-expired-password +fi + +# default socket path is different from originally set default (from debian/ubuntu), so we need to create a symlink +sudo mkdir /var/run/mysqld/ +sudo chown mysql:mysql /var/run/mysqld/ +sudo ln -s /var/lib/mysql/mysql.sock /var/run/mysqld/mysqld.sock +sudo chown mysql:mysql /var/run/mysqld/mysqld.sock + +# set up replication source to validate replica metrics +# set main target (replica) to use binary logging for replication to work +sudo tee -a /etc/my.cnf >/dev/null <