Merge branch 'master' into master

.github/scripts/format-go-code.sh

@@ -6,12 +6,15 @@ FILEPATH="$1"
 
 gofmt -s -w "$FILEPATH"
 
-# https://github.com/rinchsan/gosimports
+# https://github.com/daixiang0/gci
 if [[ "$FILEPATH" == *"tests/slo/"* ]]
 then
-  gosimports -local slo -w "$FILEPATH"
+  gci write --skip-generated -s standard -s default -s "prefix(slo)" "$FILEPATH"
+elif [[ "$FILEPATH" == *"examples/"* ]]
+then
+  gci write --skip-generated -s standard -s default -s "prefix(examples)" "$FILEPATH"
 else
-  gosimports -local github.com/ydb-platform/ydb-go-sdk/v3 -w "$FILEPATH"
+  gci write --skip-generated -s standard -s default -s "prefix(github.com/ydb-platform/ydb-go-sdk/v3)" "$FILEPATH"
 fi
 
 

.github/workflows/breaking.yml

@@ -11,6 +11,9 @@ jobs:
       group: broken-changes-${{ github.ref }}
       cancel-in-progress: true
     runs-on: ubuntu-latest
+    permissions: 
+      pull-requests: write
+      contents: read
     steps:
       - name: Install Go
         uses: actions/setup-go@v3
@@ -22,6 +25,8 @@ jobs:
         run: test -e ~/go/bin/gorelease || go install golang.org/x/exp/cmd/gorelease@latest
       - name: Check broken API changes
         run: gorelease -base=$GITHUB_BASE_REF 2>&1 > changes.txt | true
+      - name: Print API changes
+        run: cat changes.txt
       - name: Comment Report
         if: always()
         uses: marocchino/sticky-pull-request-comment@v2

.github/workflows/check-codegen.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        go-version: [1.20.x, 1.21.x]
+        go-version: [1.21.x, 1.22.x]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Checkout
@@ -32,13 +32,18 @@ jobs:
     - name: Build
       run: |
         go install ./internal/cmd/gtrace
-        go install go.uber.org/mock/mockgen@892b665398ecece7c7a900d2a2f2fa3dac07e4c4        
+        go install ./internal/cmd/gstack
+        go install go.uber.org/mock/[email protected]
 
     - name: Clean and re-generate *_gtrace.go files
       run: |
         rm -f ./trace/*_gtrace.go
         go generate ./trace
         go generate ./...
 
+    - name: Re-generate stack.FunctionID calls
+      run: |
+        gstack .
+
     - name: Check repository diff
       run: bash ./.github/scripts/check-work-copy-equals-to-committed.sh "code-generation not equal with committed"

.github/workflows/examples.yml

@@ -15,11 +15,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ydb-version: [ 22.5, 23.1, 23.2 ]
-        application: [ native, database_sql, gorm, xorm ]
+        ydb-version: [ 23.3, 24.1 ]
+        application: [ native/table, native/query, database_sql, gorm, xorm ]
     services:
       ydb:
-        image: cr.yandex/crpsjg1coh47p81vh2lc/yandex-docker-local-ydb:${{ matrix.ydb-version }}
+        image: ydbplatform/local-ydb:${{ matrix.ydb-version }}
         ports:
           - 2135:2135
           - 2136:2136
@@ -29,11 +29,13 @@ jobs:
         env:
           YDB_LOCAL_SURVIVE_RESTART: true
           YDB_USE_IN_MEMORY_PDISKS: true
+          YDB_TABLE_ENABLE_PREPARED_DDL: true
         options: '-h localhost'
     env:
       OS: ubuntu-latest
       YDB_CONNECTION_STRING: grpc://localhost:2136/local
       YDB_ANONYMOUS_CREDENTIALS: 1
+      YDB_VERSION: ${{ matrix.ydb-version }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v3

.github/workflows/lint.yml

@@ -69,7 +69,7 @@ jobs:
       - name: Install utilities
         run: |
           go install mvdan.cc/[email protected]
-          go install github.com/rinchsan/gosimports/cmd/gosimports@v0.1.5
+          go install github.com/daixiang0/gci@v0.12.1
       - name: format all files with auto-formatter
         run: bash ./.github/scripts/format-all-go-code.sh "$PWD"
       - name: Check repository diff

.github/workflows/publish.yml

@@ -70,7 +70,7 @@ jobs:
             TAG="v$MAJOR.$MINOR.$PATCH";
           fi;
           git tag $TAG
-          git push --tags && git push
+          git push && git push --tags
           CHANGELOG="$CHANGELOG
           
           Full Changelog: [$LAST_TAG...$TAG](https://github.com/ydb-platform/ydb-go-sdk/compare/$LAST_TAG...$TAG)"

.github/workflows/slo.yml

@@ -43,29 +43,35 @@ jobs:
           timeBetweenPhases: 30
           shutdownTime: 30
 
-          language_id0: 'native'
+          language_id0: 'native-table'
           workload_path0: 'tests/slo'
-          language0: 'Go SDK native'
+          language0: 'Native ydb-go-sdk/v3 over table-service'
           workload_build_context0: ../..
-          workload_build_options0: -f Dockerfile --build-arg SRC_PATH=native --build-arg JOB_NAME=workload-native
+          workload_build_options0: -f Dockerfile --build-arg SRC_PATH=native/table --build-arg JOB_NAME=workload-native-table
 
-          language_id1: 'databasesql'
+          language_id1: 'native-query'
           workload_path1: 'tests/slo'
-          language1: 'Go SDK database/sql'
+          language1: 'Native ydb-go-sdk/v3 over query-service'
           workload_build_context1: ../..
-          workload_build_options1: -f Dockerfile --build-arg SRC_PATH=database/sql --build-arg JOB_NAME=workload-databasesql
+          workload_build_options1: -f Dockerfile --build-arg SRC_PATH=native/query --build-arg JOB_NAME=workload-native-query
 
-          language_id2: 'gorm'
+          language_id2: 'database-sql'
           workload_path2: 'tests/slo'
-          language2: 'Go SDK gorm'
+          language2: 'Go SDK database/sql'
           workload_build_context2: ../..
-          workload_build_options2: -f Dockerfile --build-arg SRC_PATH=gorm --build-arg JOB_NAME=workload-gorm
+          workload_build_options2: -f Dockerfile --build-arg SRC_PATH=database/sql --build-arg JOB_NAME=workload-database-sql
 
-          language_id3: 'xorm'
+          language_id3: 'gorm'
           workload_path3: 'tests/slo'
-          language3: 'Go SDK xorm'
+          language3: 'Go SDK gorm'
           workload_build_context3: ../..
-          workload_build_options3: -f Dockerfile --build-arg SRC_PATH=xorm --build-arg JOB_NAME=workload-xorm
+          workload_build_options3: -f Dockerfile --build-arg SRC_PATH=gorm --build-arg JOB_NAME=workload-gorm
+
+          language_id4: 'xorm'
+          workload_path4: 'tests/slo'
+          language4: 'Go SDK xorm'
+          workload_build_context4: ../..
+          workload_build_options4: -f Dockerfile --build-arg SRC_PATH=xorm --build-arg JOB_NAME=workload-xorm
 
       - uses: actions/upload-artifact@v3
         if: always()

.github/workflows/tests.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go-version: [1.20.x, 1.21.x]
+        go-version: [1.21.x, 1.22.x]
         os: [ubuntu, windows, macOS]
     env:
       OS: ${{ matrix.os }}-latest
@@ -44,11 +44,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go-version: [1.20.x, 1.21.x]
-        ydb-version: [22.5, 23.1, 23.2, 23.3]
+        go-version: [1.21.x, 1.22.x]
+        ydb-version: [23.3, 24.1]
     services:
       ydb:
-        image: cr.yandex/yc/yandex-docker-local-ydb:${{ matrix.ydb-version }}
+        image: ydbplatform/local-ydb:${{ matrix.ydb-version }}
         ports:
           - 2135:2135
           - 2136:2136
@@ -58,6 +58,7 @@ jobs:
         env:
           YDB_LOCAL_SURVIVE_RESTART: true
           YDB_USE_IN_MEMORY_PDISKS: true
+          YDB_TABLE_ENABLE_PREPARED_DDL: true
         options: '-h localhost'
     env:
       OS: ubuntu-latest

.golangci.yml

@@ -64,6 +64,12 @@ linters-settings:
     # put imports beginning with prefix after 3rd-party packages;
     # it's a comma-separated list of prefixes
     local-prefixes: github.com/ydb-platform/ydb-go-sdk/v3
+  gci:
+    sections:
+      - standard # Standard section: captures all standard packages.
+      - default # Default section: contains all imports that could not be matched to another section type.
+      - prefix(github.com/ydb-platform/ydb-go-sdk/v3) # Custom section: groups all imports with the specified Prefix.
+    skip-generated: true
   goconst:
     # minimal length of string constant, 3 by default
     min-len: 2
@@ -106,6 +112,10 @@ linters-settings:
     # if it's called for subdir of a project it can't find external interfaces. All text editor integrations
     # with golangci-lint call it on a directory with the changed file.
     check-exported: false
+  gomoddirectives:
+    replace-local: true
+    replace-allow-list:
+      - xorm.io/xorm
   gocritic:
     disabled-checks:
       - whyNoLint # https://github.com/go-critic/go-critic/issues/1063
@@ -213,32 +223,24 @@ linters:
     - cyclop
     - depguard
     - dupl
-    - errname
     - exhaustive
     - exhaustivestruct
     - exhaustruct
     - forbidigo
     - forcetypeassert
     - funlen
-    - gci
     - gochecknoglobals
     - gocognit
     - godot
     - goerr113
     - golint
     - gomnd
-    - gomoddirectives
     - ifshort
     - interfacebloat
-    - interfacer
     - ireturn
     - maintidx
-    - maligned
-    - nilerr
-    - nlreturn
     - nonamedreturns
     - paralleltest
-    - protogetter
     - scopelint
     - structcheck
     - testableexamples

CHANGELOG.md

@@ -1,3 +1,98 @@
+## v3.59.3
+* Fixed `gstack` logic for parsing `ast.BlockStmt`
+
+## v3.59.2
+* Added internal `gstack` codegen tool for filling `stack.FunctionID` with value from call stack
+
+## v3.59.1
+* Fixed updating last usage timestamp for smart parking of the conns
+
+## v3.59.0
+* Added `Struct` support for `ydb.ParamsBuilder()`
+* Added support of `TzDate`,`TzDateTime`,`TzTimestamp` types in `ydb.ParamsBuilder()`
+* Added `trace.Query.OnTransactionExecute` event
+* Added query pool metrics
+* Fixed logic of query session pool
+* Changed initialization of internal driver clients to lazy
+* Removed `ydb.WithSessionPoolSizeLimit()` option
+* Added async put session into pool if external context is done
+* Dropped intermediate callbacks from `trace.{Table,Retry,Query}` events
+* Wrapped errors from `internal/pool.Pool.getItem` as retryable
+* Disabled the logic of background grpc-connection parking
+* Improved stringification for postgres types
+
+## v3.58.2
+* Added `trace.Query.OnSessionBegin` event
+* Added `trace.Query.OnResult{New,NextPart,NextResultSet,Close}` events
+* Added `trace.Query.OnRow{Scan,ScanNamed,ScanStruct}` events
+
+## v3.58.1
+* Dropped all deprecated callbacks and events from traces
+* Added `trace.Driver.OnConnStream{SendMsg,RecvMsg,CloseSend}` events
+* Added `trace.Query.OnSessionExecute` event
+
+## v3.58.0
+* Changed `List` constructor from `ydb.ParamsBuilder().List().Build().Build()` to `ydb.ParamsBuilder().BeginList().EndList().Build()`
+* Changed `Set` constructor from `ydb.ParamsBuilder().Set().Build().Build()` to `ydb.ParamsBuilder().BeginSet().EndSet().Build()`
+* Changed `Dict` constructor from `ydb.ParamsBuilder().Dict().Build().Build()` to `ydb.ParamsBuilder().BeginDict().EndDict().Build()`
+* Changed `Optional` constructor from `ydb.ParamsBuilder().Set().Build().Build()` to `ydb.ParamsBuilder().BeginOptional().EndOptional().Build()`
+* Added events into `trace.Query` trace
+* Rewrote `internal/pool` to buffered channel
+* Added `internal/xcontext.WithDone()`
+* Added `internal/xsync.{OnceFunc,OnceValue}`
+* Updated `google.golang.org/protobuf` from `v1.31.0` to `v.33.0`
+* Added `ydb.ParamsBuilder().Pg().{Value,Int4,Int8,Unknown}` for postgres arguments
+* Added `Tuple` support for `ydb.ParamsBuilder()`
+
+## v3.57.4
+* Added client pid to each gRPC requests to YDB over header `x-ydb-client-pid`
+* Added `ydb.WithApplicationName` option
+* Added `Dict` support for `ydb.ParamsBuilder()`
+
+## v3.57.3
+* Added metrics over query service internals
+* Added session create and delete events into `trace.Query`
+* Moved public type `query.SessionStatus` into `internal/query` package
+
+## v3.57.2
+* Fixed cases when some option is nil
+
+## v3.57.1
+* Added logs over query service internals
+* Changed `trace.Query` events
+* Changed visibility of `query.{Do,DoTx}Options` from public to private
+
+## v3.57.0
+* Added experimental implementation of query service client
+* Fixed sometime panic on topic writer closing
+* Added experimental query parameters builder `ydb.ParamsBuilder()`
+* Changed types of `table/table.{QueryParameters,ParameterOption}` to aliases on `internal/params.{Parameters,NamedValue}`
+* Fixed bug with optional decimal serialization 
+
+## v3.56.2
+* Fixed return private error for commit to stopped partition in topic reader.
+* Stopped wrapping err error as transport error at topic streams (internals)
+
+## v3.56.1
+* Fixed fixenv usage (related to tests only)
+
+## v3.56.0
+* Fixed handle of operational errors in topic streams
+* The minimum version of Go in `ydb-go-sdk` has been raised to `go1.21`
+* Fixed topic writer infinite reconnections in some cases
+* Refactored nil on err `internal/grpcwrapper/rawydb/issues.go`, when golangci-lint nilerr enabled
+* Refactored nil on err `internal/grpcwrapper/rawtopic/describe_topic.go`, when golangci-lint nilerr enabled
+
+## v3.55.3
+* Fixed handle of operational errors in topic streams (backported fix only)
+
+## v3.55.2
+* Fixed init info in topic writer, when autoseq num turned off.
+
+## v3.55.1
+* Supported column name prefix `__discard_column_` for discard columns in result sets
+* Made `StatusIds_SESSION_EXPIRED` retriable for idempotent operations
+
 ## v3.55.0
 * Refactored `internal/value/intervalValue.Yql()`
 * The minimum version of Go in `ydb-go-sdk` has been raised to `go1.20`
@@ -12,14 +107,14 @@
 
 ## v3.54.2
 * Added context to some internal methods for better tracing
-* Added `trace.FunctionID` helper and `FunctionID` field to trace start info's 
+* Added `trace.FunctionID` helper and `FunctionID` field to trace start info's
 * Replaced lazy initialization of ydb clients (table, topic, etc.) to explicit initialization on `ydb.Open` step
 
 ## v3.54.1
-* Fixed inconsistent labels in `metrics` 
+* Fixed inconsistent labels in `metrics`
 
 ## v3.54.0
-* Allowed `sql.LevelSerializable` isolation level in read-write mode in `database/sql` transactions 
+* Allowed `sql.LevelSerializable` isolation level in read-write mode in `database/sql` transactions
 * Refactored traces and metrics
 * Added `{retry,table}.WithLabel` options for mark retriers calls
 * Added `ydb.WithTraceRetry` option
@@ -43,7 +138,7 @@
 * Fixed stringification of credentials object
 
 ## v3.53.2
-* Fixed panic when try to unwrap values with more than 127 columns with custom ydb unmarshaler 
+* Fixed panic when try to unwrap values with more than 127 columns with custom ydb unmarshaler
 
 ## v3.53.1
 * Bumps `github.com/ydb-platform/ydb-go-genproto` for support `query` service
@@ -192,7 +287,7 @@
 * Added `table/options.WithCallOptions` options for append custom grpc call options into `session.{BulkUpsert,Execute,StreamExecuteScanQuery}`
 * Supported fake transactions in `database/sql` driver over connector option `ydb.WithFakeTx(queryMode)` and connection string param `go_fake_tx`
 * Removed `testutil/timeutil` package (all usages replaced with `clockwork` package)
-* Changed behaviour of retryer on transport errors `cancelled` and `deadline exceeded` - will retry idempotent operation if context is not done  
+* Changed behaviour of retryer on transport errors `cancelled` and `deadline exceeded` - will retry idempotent operation if context is not done
 * Added address of node to operation error description as optional
 * Fixed bug with put session from unknown node
 * Fixed bug with parsing of `TzTimestamp` without microseconds

CONTRIBUTING.md

@@ -56,7 +56,7 @@ go test -race -tags fast ./...
 ##### All tests
 
 ```sh
-docker run -itd --name ydb -dp 2135:2135 -dp 2136:2136 -dp 8765:8765 -v `pwd`/ydb_certs:/ydb_certs -e YDB_LOCAL_SURVIVE_RESTART=true -e YDB_USE_IN_MEMORY_PDISKS=true -h localhost cr.yandex/yc/yandex-docker-local-ydb:latest
+docker run -itd --name ydb -dp 2135:2135 -dp 2136:2136 -dp 8765:8765 -v `pwd`/ydb_certs:/ydb_certs -e YDB_LOCAL_SURVIVE_RESTART=true -e YDB_USE_IN_MEMORY_PDISKS=true -h localhost ydbplatform/local-ydb:latest
 export YDB_CONNECTION_STRING="grpcs://localhost:2135/local"
 export YDB_SSL_ROOT_CERTIFICATES_FILE="`pwd`/ydb_certs/ca.pem"
 export YDB_SESSIONS_SHUTDOWN_URLS="http://localhost:8765/actors/kqp_proxy?force_shutdown=all"