PMM-7 Update pg_query_go

.github/workflows/agent.yml

@@ -26,15 +26,15 @@ jobs:
     strategy:
       matrix:
         images:
-          - { mysql: 'mysql:5.6', mongo: 'mongo:4.2', postgres: 'postgres:10', pmm_server: 'percona/pmm-server:2.0.0' }
-          - { mysql: 'mysql:5.7', mongo: 'mongo:4.4', postgres: 'postgres:11', pmm_server: 'percona/pmm-server:2.0.1' }
-          - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:12', pmm_server: 'percona/pmm-server:2.1.0' }
-          - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:13', pmm_server: 'percona/pmm-server:2.2.0' }
+          - { mysql: 'mysql:5.6', mongo: 'mongo:4.2', postgres: 'postgres:13', pmm_server: 'percona/pmm-server:2.0.0' }
+          - { mysql: 'mysql:5.7', mongo: 'mongo:4.4', postgres: 'postgres:14', pmm_server: 'percona/pmm-server:2.0.1' }
+          - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:15', pmm_server: 'percona/pmm-server:2.1.0' }
+          - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:16', pmm_server: 'percona/pmm-server:2.2.0' }
 
           # Percona + latest PMM Server release
-          - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.2', postgres: 'perconalab/percona-distribution-postgresql:11', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
-          - { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:12.8-pg_stat', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
-          - { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:13.5-pg_stat', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
+          - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.2', postgres: 'perconalab/percona-distribution-postgresql:14', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
+          - { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:15', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
+          - { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:16', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'percona/pmm-server:2' }
 
           # MariaDB (only 3 latest GA versions)
           #   + older supported MongoDB versions
@@ -44,9 +44,9 @@ jobs:
           # - MYSQL_IMAGE=mariadb:10.0
           # - MYSQL_IMAGE=mariadb:10.1
 
-          - { mysql: 'mariadb:10.2', mongo: 'mongo:4.4',                          postgres: 'postgres:9.4', pmm_server: 'perconalab/pmm-server:dev-latest' }
-          - { mysql: 'mariadb:10.3', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'postgres:9.5', pmm_server: 'perconalab/pmm-server:dev-latest' }
-          - { mysql: 'mariadb:10.4',                                              postgres: 'postgres:9.6', pmm_server: 'perconalab/pmm-server:dev-latest' }
+          - { mysql: 'mariadb:10.2', mongo: 'mongo:4.4',                          postgres: 'postgres:10', pmm_server: 'perconalab/pmm-server:dev-latest' }
+          - { mysql: 'mariadb:10.3', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'postgres:11', pmm_server: 'perconalab/pmm-server:dev-latest' }
+          - { mysql: 'mariadb:10.4',                                              postgres: 'postgres:12', pmm_server: 'perconalab/pmm-server:dev-latest' }
 
     continue-on-error: true
 

agent/agents/postgres/parser/parser.go

@@ -21,7 +21,7 @@ import (
 	"sort"
 	"strings"
 
-	pgquery "github.com/pganalyze/pg_query_go/v2"
+	pgquery "github.com/pganalyze/pg_query_go/v5"
 	"github.com/pkg/errors"
 )
 

agent/agents/postgres/parser/testdata/query019.json

@@ -1,3 +1,7 @@
 {
-  "error": "error on parsing sql query: subquery in FROM must have an alias"
+  "tables": [
+    "credit_card_wo_complaints",
+    "without_complaints"
+  ],
+  "error": ""
 }

agent/agents/postgres/pgstatmonitor/stat_monitor_cache.go

@@ -21,7 +21,7 @@ import (
 	"time"
 
 	"github.com/AlekSi/pointer"
-	pgquery "github.com/pganalyze/pg_query_go/v2"
+	pgquery "github.com/pganalyze/pg_query_go/v5"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/reform.v1"

agent/agents/postgres/pgstatstatements/pgstatstatements_test.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -128,16 +129,6 @@ func TestPGStatStatementsQAN(t *testing.T) {
 	engineVersion := tests.PostgreSQLVersion(t, sqlDB)
 	var digests map[string]string // digest_text/fingerprint to digest/query_id
 	switch engineVersion {
-	case "9.4":
-		digests = map[string]string{
-			selectAllCities:     "3239586867",
-			selectAllCitiesLong: "2745128652",
-		}
-	case "9.5", "9.6":
-		digests = map[string]string{
-			selectAllCities:     "3994135135",
-			selectAllCitiesLong: "2677760328",
-		}
 	case "10":
 		digests = map[string]string{
 			selectAllCities:     "2229807896",
@@ -163,6 +154,16 @@ func TestPGStatStatementsQAN(t *testing.T) {
 			selectAllCities:     "5991662752016701281",
 			selectAllCitiesLong: "-3564720362103294944",
 		}
+	case "15":
+		digests = map[string]string{
+			selectAllCities:     "5991662752016701281",
+			selectAllCitiesLong: "-3564720362103294944",
+		}
+	case "16":
+		digests = map[string]string{
+			selectAllCities:     "9094455616937907056",
+			selectAllCitiesLong: "-8264367755446145090",
+		}
 	default:
 		t.Log("Unhandled version, assuming dummy digests.")
 		digests = map[string]string{
@@ -282,9 +283,16 @@ func TestPGStatStatementsQAN(t *testing.T) {
 		t.Logf("Actual:\n%s", tests.FormatBuckets(buckets))
 		require.Len(t, buckets, 1)
 
+		expectedSharedBlksHitSum := float32(1007)
+		engineVersionInt, err := strconv.Atoi(engineVersion)
+		assert.NoError(t, err)
+		if engineVersionInt >= 14 {
+			expectedSharedBlksHitSum = 32
+		}
+
 		actual := buckets[0]
 		assert.InDelta(t, 0, actual.Common.MQueryTimeSum, 0.09)
-		assert.InDelta(t, 1010, actual.Postgresql.MSharedBlksHitSum+actual.Postgresql.MSharedBlksReadSum, 3)
+		assert.InDelta(t, expectedSharedBlksHitSum+3, actual.Postgresql.MSharedBlksHitSum+actual.Postgresql.MSharedBlksReadSum, 3)
 		assert.InDelta(t, 1.5, actual.Postgresql.MSharedBlksHitCnt+actual.Postgresql.MSharedBlksReadCnt, 0.5)
 		expected := &agentpb.MetricsBucket{
 			Common: &agentpb.MetricsBucket_Common{
@@ -329,7 +337,7 @@ func TestPGStatStatementsQAN(t *testing.T) {
 		actual = buckets[0]
 		assert.InDelta(t, 0, actual.Common.MQueryTimeSum, 0.09)
 		assert.InDelta(t, 0, actual.Postgresql.MBlkReadTimeCnt, 1)
-		assert.InDelta(t, 1007, actual.Postgresql.MSharedBlksHitSum, 2)
+		assert.InDelta(t, expectedSharedBlksHitSum, actual.Postgresql.MSharedBlksHitSum, 2)
 		expected = &agentpb.MetricsBucket{
 			Common: &agentpb.MetricsBucket_Common{
 				Fingerprint:         selectAllCitiesLong,
@@ -349,7 +357,7 @@ func TestPGStatStatementsQAN(t *testing.T) {
 			Postgresql: &agentpb.MetricsBucket_PostgreSQL{
 				MBlkReadTimeCnt:   actual.Postgresql.MBlkReadTimeCnt,
 				MBlkReadTimeSum:   actual.Postgresql.MBlkReadTimeSum,
-				MSharedBlksHitCnt: 1,
+				MSharedBlksHitCnt: actual.Postgresql.MSharedBlksHitCnt,
 				MSharedBlksHitSum: actual.Postgresql.MSharedBlksHitSum,
 				MRowsCnt:          1,
 				MRowsSum:          499,
@@ -397,16 +405,9 @@ func TestPGStatStatementsQAN(t *testing.T) {
 		t.Logf("Actual:\n%s", tests.FormatBuckets(buckets))
 		require.Len(t, buckets, 1)
 
-		var fingerprint string
-		switch engineVersion {
-		case "9.4", "9.5", "9.6":
-			fingerprint = fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES (?, ?, ?, ?)`, tableName)
+		fingerprint := fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES ($1, $2, $3, $4)`, tableName)
 
-		default:
-			fingerprint = fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES ($1, $2, $3, $4)`, tableName)
-		}
 		actual := buckets[0]
-		assert.NotZero(t, actual.Postgresql.MBlkReadTimeSum)
 		expected := &agentpb.MetricsBucket{
 			Common: &agentpb.MetricsBucket_Common{
 				Queryid:             actual.Common.Queryid,
@@ -424,8 +425,10 @@ func TestPGStatStatementsQAN(t *testing.T) {
 				MQueryTimeSum:       actual.Common.MQueryTimeSum,
 			},
 			Postgresql: &agentpb.MetricsBucket_PostgreSQL{
-				MBlkReadTimeCnt:       float32(n),
+				MBlkReadTimeCnt:       actual.Postgresql.MBlkReadTimeCnt,
 				MBlkReadTimeSum:       actual.Postgresql.MBlkReadTimeSum,
+				MBlkWriteTimeCnt:      actual.Postgresql.MBlkWriteTimeCnt,
+				MBlkWriteTimeSum:      actual.Postgresql.MBlkWriteTimeSum,
 				MSharedBlksReadCnt:    actual.Postgresql.MSharedBlksReadCnt,
 				MSharedBlksReadSum:    actual.Postgresql.MSharedBlksReadSum,
 				MSharedBlksWrittenCnt: actual.Postgresql.MSharedBlksWrittenCnt,
@@ -438,6 +441,7 @@ func TestPGStatStatementsQAN(t *testing.T) {
 				MRowsSum:              float32(n),
 			},
 		}
+		assert.InDelta(t, n, actual.Postgresql.MBlkWriteTimeCnt+actual.Postgresql.MBlkReadTimeCnt, 1)
 		tests.AssertBucketsEqual(t, expected, actual)
 		assert.LessOrEqual(t, actual.Postgresql.MBlkReadTimeSum, actual.Common.MQueryTimeSum)
 	})

agent/runner/actions/postgresql_query_show_action_test.go

@@ -46,14 +46,12 @@ func TestPostgreSQLQueryShow(t *testing.T) {
 
 		b, err := a.Run(ctx)
 		require.NoError(t, err)
-		assert.LessOrEqual(t, 22000, len(b))
-		assert.LessOrEqual(t, len(b), 33668)
+		assert.GreaterOrEqual(t, len(b), 22000)
 
 		data, err := agentpb.UnmarshalActionQueryResult(b)
 		require.NoError(t, err)
 		t.Log(spew.Sdump(data))
-		assert.LessOrEqual(t, 200, len(data))
-		assert.LessOrEqual(t, len(data), 358)
+		assert.GreaterOrEqual(t, len(data), 200)
 
 		var found int
 		for _, m := range data {

go.mod

@@ -62,7 +62,7 @@ require (
 	github.com/percona/go-mysql v0.0.0-20210427141028-73d29c6da78c
 	github.com/percona/percona-toolkit v3.2.1+incompatible
 	github.com/percona/promconfig v0.2.5
-	github.com/pganalyze/pg_query_go/v2 v2.2.0
+	github.com/pganalyze/pg_query_go/v5 v5.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.13.6
 	github.com/pmezard/go-difflib v1.0.0

go.sum

@@ -656,8 +656,8 @@ github.com/percona/percona-toolkit v3.2.1+incompatible h1:5jLvtZKcu9fDmaLRB8qA4b
 github.com/percona/percona-toolkit v3.2.1+incompatible/go.mod h1:netQWdWMaF1cnmwiIS+i5uyaqNXz46yNeM6HKkR6yeI=
 github.com/percona/promconfig v0.2.5 h1:f/HN/CbECQs7d9RIB6MKVkuXstsrsqEDxRvf6yigquc=
 github.com/percona/promconfig v0.2.5/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg=
-github.com/pganalyze/pg_query_go/v2 v2.2.0 h1:OW+reH+ZY7jdEuPyuLGlf1m7dLbE+fDudKXhLs0Ttpk=
-github.com/pganalyze/pg_query_go/v2 v2.2.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA=
+github.com/pganalyze/pg_query_go/v5 v5.1.0 h1:MlxQqHZnvA3cbRQYyIrjxEjzo560P6MyTgtlaf3pmXg=
+github.com/pganalyze/pg_query_go/v5 v5.1.0/go.mod h1:FsglvxidZsVN+Ltw3Ai6nTgPVcK2BPukH3jCDEqc1Ug=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pierrec/lz4 v2.3.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
@@ -1190,6 +1190,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=