Merge #110930 #111510 #111701 #111704 #111776

110930: gcjob_test: add more logging to TestGCJobRetry r=rafiss a=andyyang890

This patch adds more logging to `TestGCJobRetry` to help debug
occasional flakes.

Informs: #110447

Release note: None

111510: api: increase timeout to request execution details r=maryliag a=adityamaru

In large clusters requesting execution details can definitely take longer than 5 seconds. This is because it involves collecting cluster wide traces, goroutines and contacting the coordinator node of the job to dump its execution details. Since this is a debug only tool we bump the timeout to 300s to give it all the time it needs.

Epic: none
Release note: None

111701: sql: add tests for privileges for statements in udfs r=rharding6373 a=rharding6373

This PR adds test coverage for privileges in UDFs, e.g., SELECT and INSERT privileges.

Epic: CRDB-25388
Informs: #87289

Release note: None

111704: Roachtest azure nightly r=healthy-pod,herkolategan a=smg260

These are a series of commits to enable roachtests to run on Azure in TeamCity. 

1. Add the relevant teamcity invoke script
2. Update authentication to look in CLI or environment for dev and TC respectively
3. Look for a default subscription in the environment, with fallback to existing "pick first" implementation
4. Add a security rule to allow roachtest host machine to connect to a vm via kafka admin
5. Update azure default location to one with more quota and `apt-get update` before installing go for a cdc test (failed on azure)

A follow up PR will enable an initial set of roachtests to run.

Epic: CC-25185
Release note: none



111776: go.mod: bump Pebble to b013ca78e9dc r=RaduBerinde a=RaduBerinde

b013ca78 db: keep track of virtual sstable size sum
62251e69 db: make checkpoint test even more deterministic
c7c47d6b db: turn testingAlwaysWaitForCleanup into an option
a05b0192 db: keep track of virtual sstable count in metrics
3c778710 db: add test for virtual sstable checkpointing
cb4dab66 db: add metrics for num backing sstables and size
8317cf38 db: incrementally keep tracking of backing table size
0f80e184 Update index.html
aa077af6 go.mod: specify Go 1.20
ccb9a7dc manifest: add invariant check for duplicate file backings
699fc0e8 db: only create one CreatedBackingTables entry per sstable
b2da10c6 db: remove trailing whitespace from compacting log line
1d696c79 db: cleanup btree obsoletion logic

Fixes: #111674
Release note: none

Co-authored-by: Andy Yang <[email protected]>
Co-authored-by: adityamaru <[email protected]>
Co-authored-by: rharding6373 <[email protected]>
Co-authored-by: Miral Gadani <[email protected]>
Co-authored-by: Radu Berinde <[email protected]>

DEPS.bzl

@@ -1599,10 +1599,10 @@ def go_deps():
         patches = [
             "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
         ],
-        sha256 = "9418eb96febfa50da2f02f87501d0459a8205fea8f91b539300a03c7f62a6d61",
-        strip_prefix = "github.com/cockroachdb/[email protected]20230927205513-725ebe297867",
+        sha256 = "da88d66f323769d609581b310f42a9bcc245cb3c321f0fae2aea7d2ed7c9fe13",
+        strip_prefix = "github.com/cockroachdb/[email protected]20231003235536-b013ca78e9dc",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20230927205513-725ebe297867.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20231003235536-b013ca78e9dc.zip",
         ],
     )
     go_repository(

build/bazelutil/distdir_files.bzl

@@ -321,7 +321,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/go-test-teamcity/com_github_cockroachdb_go_test_teamcity-v0.0.0-20191211140407-cff980ad0a55.zip": "bac30148e525b79d004da84d16453ddd2d5cd20528e9187f1d7dac708335674b",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20230118201751-21c54148d20b.zip": "ca7776f47e5fecb4c495490a679036bfc29d95bd7625290cfdb9abb0baf97476",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20230927205513-725ebe297867.zip": "9418eb96febfa50da2f02f87501d0459a8205fea8f91b539300a03c7f62a6d61",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20231003235536-b013ca78e9dc.zip": "da88d66f323769d609581b310f42a9bcc245cb3c321f0fae2aea7d2ed7c9fe13",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.5.zip": "11b30528eb0dafc8bc1a5ba39d81277c257cbe6946a7564402f588357c164560",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/sentry-go/com_github_cockroachdb_sentry_go-v0.6.1-cockroachdb.2.zip": "fbb2207d02aecfdd411b1357efe1192dbb827959e36b7cab7491731ac55935c9",

build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+
+source "$dir/teamcity-support.sh"  # For $root
+source "$dir/teamcity-bazel-support.sh"  # For run_bazel
+
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY" \
+			       run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

build/teamcity/util/roachtest_util.sh

@@ -70,13 +70,19 @@ case "${CLOUD}" in
       # - "aws" to ensure we select tests that now no longer have "default" because they have the "aws" tag
       # Ideally, refactor the tags themselves to be explicit about what cloud they are for and when they can run.
       # https://github.com/cockroachdb/cockroach/issues/100605
-      FILTER="tag:aws tag:default"
+      FILTER="tag:aws tag:default tag:azure"
     ;;
   aws)
     if [ -z "${FILTER}" ]; then
       FILTER="tag:aws"
     fi
     ;;
+  azure)
+    if [ -z "${FILTER}" ]; then
+      # Soon to go away with Radu's tag changes.
+      FILTER="tag:azure"
+    fi
+    ;;
   *)
     echo "unknown cloud ${CLOUD}"
     exit 1

go.mod

@@ -113,7 +113,7 @@ require (
 	github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
 	github.com/cockroachdb/gostdlib v1.19.0
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b
-	github.com/cockroachdb/pebble v0.0.0-20230927205513-725ebe297867
+	github.com/cockroachdb/pebble v0.0.0-20231003235536-b013ca78e9dc
 	github.com/cockroachdb/redact v1.1.5
 	github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
 	github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b

go.sum

@@ -489,8 +489,8 @@ github.com/cockroachdb/gostdlib v1.19.0/go.mod h1:+dqqpARXbE/gRDEhCak6dm0l14AaTy
 github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20230927205513-725ebe297867 h1:O/fBxpVLLedWnVw+kRDe2rcybSGFLiLXnnHgyrT/Pr0=
-github.com/cockroachdb/pebble v0.0.0-20230927205513-725ebe297867/go.mod h1:nindLFinxeDPjP4qI9LtVHAwDef57/0s5KMfWgdktQc=
+github.com/cockroachdb/pebble v0.0.0-20231003235536-b013ca78e9dc h1:eyGg4MSAQSEBY4fYPu7KNsCCFNIXwhkbEaSciMRfDWs=
+github.com/cockroachdb/pebble v0.0.0-20231003235536-b013ca78e9dc/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo=
 github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=

pkg/cli/interactive_tests/test_encryption.tcl

@@ -52,7 +52,7 @@ end_test
 
 start_test "Run pebble debug tool."
 send "$argv debug pebble db lsm $storedir\r"
-eexpect "level | tables  size val-bl | score |   in  | tables  size | tables  size | tables  size |  read |   r   w\r"
+eexpect "level | tables  size val-bl vtables | score |   in  | tables  size | tables  size | tables  size |  read |   r   w\r"
 end_test
 
 start_test "Restart with plaintext."
@@ -106,7 +106,7 @@ end_test
 
 start_test "Run pebble debug tool with AES-256."
 send "$argv debug pebble db lsm $storedir --enterprise-encryption=path=$storedir,key=$keydir/aes-256.key,old-key=$keydir/aes-256.key\r"
-eexpect "level | tables  size val-bl | score |   in  | tables  size | tables  size | tables  size |  read |   r   w\r"
+eexpect "level | tables  size val-bl vtables | score |   in  | tables  size | tables  size | tables  size |  read |   r   w\r"
 # Try running without the encryption flag.
 send "$argv debug pebble db lsm $storedir\r"
 eexpect "If this is an encrypted store, make sure the correct encryption key is set."

pkg/roachprod/install/install.go

@@ -102,7 +102,9 @@ sudo apt-get update;
 sudo apt-get install -y gcc;
 `,
 
-	"go": `sudo apt --yes install golang-go;`,
+	"go": `
+sudo apt-get update;
+sudo apt-get install -y golang-go;`,
 
 	"haproxy": `
 sudo apt-get update;

pkg/roachprod/roachprod.go

@@ -1379,7 +1379,7 @@ func Create(
 			if err := cleanupFailedCreate(l, clusterName); err != nil {
 				l.Errorf("Error while cleaning up partially-created cluster: %s\n", err)
 			} else {
-				l.Errorf("Cleaning up OK\n")
+				l.Printf("Cleaning up OK\n")
 			}
 		}()
 	} else {

pkg/roachprod/vm/azure/auth.go

@@ -12,12 +12,18 @@ package azure
 
 import (
 	"net/http"
+	"os"
 
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
 	"github.com/cockroachdb/errors"
 )
 
+// We look to the environment for sdk authentication first.
+var hasEnvAuth = os.Getenv("AZURE_TENANT_ID") != "" &&
+	os.Getenv("AZURE_CLIENT_ID") != "" &&
+	os.Getenv("AZURE_CLIENT_SECRET") != ""
+
 // getAuthorizer returns an Authorizer, which uses the Azure CLI
 // to log into the portal.
 //
@@ -33,29 +39,34 @@ func (p *Provider) getAuthorizer() (ret autorest.Authorizer, err error) {
 		return
 	}
 
-	// Use the azure CLI to bootstrap our authentication.
+	// Use the environment or azure CLI to bootstrap our authentication.
 	// https://docs.microsoft.com/en-us/go/azure/azure-sdk-go-authorization
-	ret, err = auth.NewAuthorizerFromCLI()
+	if hasEnvAuth {
+		ret, err = auth.NewAuthorizerFromEnvironment()
+	} else {
+		ret, err = auth.NewAuthorizerFromCLI()
+	}
+
 	if err == nil {
 		p.mu.Lock()
 		p.mu.authorizer = ret
 		p.mu.Unlock()
 	} else {
-		err = errors.Wrap(err, "could got get Azure auth token")
+		err = errors.Wrap(err, "could not get Azure auth token")
 	}
 	return
 }
 
 // getAuthToken extracts the JWT token from the active Authorizer.
 func (p *Provider) getAuthToken() (string, error) {
-	auth, err := p.getAuthorizer()
+	authorizer, err := p.getAuthorizer()
 	if err != nil {
 		return "", err
 	}
 
 	// We'll steal the auth Bearer token by creating a fake HTTP request.
 	fake := &http.Request{}
-	if _, err := auth.WithAuthorization()(&stealAuth{}).Prepare(fake); err != nil {
+	if _, err := authorizer.WithAuthorization()(&stealAuth{}).Prepare(fake); err != nil {
 		return "", err
 	}
 	return fake.Header.Get("Authorization")[7:], nil