server: create api to trigger table metadata update job

adds a new API v2 endpoint,
`POST /api/v2/table_metadata/updatejob`.

This new API will trigger the table metadata update job
to run and refresh the system.table_metadata table.

This will only trigger the job if it isn't already
running. If the `?onlyIfStale` query param is
provided, the job won't be triggered unless deemed
stale, determined by the following setting:
 `obs.tablemetadata.data_valid_duration`

note that `onlyIfStale` with any value other then
`false` will be treated as true.

Resolves: #128897
Epic: CRDB-37558
Release note: None

pkg/BUILD.bazel

@@ -2288,6 +2288,7 @@ GO_TARGETS = [
     "//pkg/sql/syntheticprivilege:syntheticprivilege",
     "//pkg/sql/syntheticprivilege:syntheticprivilege_test",
     "//pkg/sql/syntheticprivilegecache:syntheticprivilegecache",
+    "//pkg/sql/tablemetadatacache/util:util",
     "//pkg/sql/tablemetadatacache:tablemetadatacache",
     "//pkg/sql/tablemetadatacache:tablemetadatacache_test",
     "//pkg/sql/tests:tests",

pkg/base/testing_knobs.go

@@ -60,4 +60,5 @@ type TestingKnobs struct {
 	TenantCapabilitiesTestingKnobs ModuleTestingKnobs
 	TableStatsKnobs                ModuleTestingKnobs
 	Insights                       ModuleTestingKnobs
+	TableMetadata                  ModuleTestingKnobs
 }

pkg/server/BUILD.bazel

@@ -279,6 +279,8 @@ go_library(
         "//pkg/sql/stats",
         "//pkg/sql/stmtdiagnostics",
         "//pkg/sql/syntheticprivilegecache",
+        "//pkg/sql/tablemetadatacache",
+        "//pkg/sql/tablemetadatacache/util",
         "//pkg/sql/ttl/ttljob",
         "//pkg/sql/ttl/ttlschedule",
         "//pkg/storage",
@@ -530,6 +532,8 @@ go_test(
         "//pkg/sql/sessiondata",
         "//pkg/sql/sqlstats",
         "//pkg/sql/sqlstats/insights",
+        "//pkg/sql/tablemetadatacache",
+        "//pkg/sql/tablemetadatacache/util",
         "//pkg/storage",
         "//pkg/storage/disk",
         "//pkg/storage/fs",

pkg/server/api_v2_databases_metadata.go

@@ -21,11 +21,16 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/security/username"
 	"github.com/cockroachdb/cockroach/pkg/server/apiutil"
 	"github.com/cockroachdb/cockroach/pkg/server/authserver"
+	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
 	"github.com/cockroachdb/cockroach/pkg/server/srverrors"
 	"github.com/cockroachdb/cockroach/pkg/sql/isql"
 	"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
+	"github.com/cockroachdb/cockroach/pkg/sql/tablemetadatacache"
 	"github.com/cockroachdb/cockroach/pkg/util/safesql"
+	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/errors"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 const (
@@ -36,10 +41,19 @@ const (
 	pageNumKey      = "pageNum"
 	pageSizeKey     = "pageSize"
 	storeIdKey      = "storeId"
+	onlyIfStaleKey  = "onlyIfStale"
 	defaultPageSize = 10
 	defaultPageNum  = 1
 )
 
+type JobStatusMessage string
+
+const (
+	MetadataNotStale JobStatusMessage = "Not enough time has elapsed since last job run"
+	JobRunning       JobStatusMessage = "Job is already running"
+	JobTriggered     JobStatusMessage = "Job triggered successfully"
+)
+
 // GetTableMetadata returns a paginated response of table metadata and statistics. This is not a live view of
 // the table data but instead is cached data that had been precomputed at an earlier time.
 //
@@ -670,14 +684,13 @@ func (a *apiV2Server) getDBMetadata(
 // responses:
 //
 //	"200":
-//	  description: A tmUpdateJobStatusResponse
+//	  description: A tmUpdateJobStatusResponse for GET requests and tmJobTriggeredResponse for POST requests.
 //	"404":
 //	  description: Not found if the user doesn't have the correct authorizations
 func (a *apiV2Server) TableMetadataJob(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	ctx = a.sqlServer.AnnotateCtx(ctx)
 	sqlUser := authserver.UserFromHTTPAuthInfoContext(ctx)
-
 	authorized, err := a.updateTableMetadataJobAuthorized(ctx, sqlUser)
 	if err != nil {
 		srverrors.APIV2InternalError(ctx, err, w)
@@ -693,6 +706,13 @@ func (a *apiV2Server) TableMetadataJob(w http.ResponseWriter, r *http.Request) {
 	switch r.Method {
 	case http.MethodGet:
 		resp, err = a.getTableMetadataUpdateJobStatus(ctx)
+	case http.MethodPost:
+		// onlyIfStale will be true if the query param exists and has any value other than "false"
+		var onlyIfStale bool
+		if r.URL.Query().Has(onlyIfStaleKey) {
+			onlyIfStale = r.URL.Query().Get(onlyIfStaleKey) != "false"
+		}
+		resp, err = a.triggerTableMetadataUpdateJob(ctx, onlyIfStale)
 	default:
 		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
 		return
@@ -755,6 +775,34 @@ func (a *apiV2Server) getTableMetadataUpdateJobStatus(
 	return jobStatus, nil
 }
 
+// triggerTableMetadataUpdateJob will trigger the table metadata update job if it isn't currently running and if it
+// is stale, if onlyIfStale is true.
+func (a *apiV2Server) triggerTableMetadataUpdateJob(
+	ctx context.Context, onlyIfStale bool,
+) (tmJobTriggeredResponse, error) {
+	jobStatus, err := a.getTableMetadataUpdateJobStatus(ctx)
+	if err != nil {
+		return tmJobTriggeredResponse{}, err
+	}
+
+	stalenessDuration := tablemetadatacache.DataValidDurationSetting.Get(&a.sqlServer.execCfg.Settings.SV)
+	if onlyIfStale && jobStatus.LastCompletedTime != nil && timeutil.Since(*jobStatus.LastCompletedTime) < stalenessDuration {
+		return tmJobTriggeredResponse{JobTriggered: false, Message: MetadataNotStale}, nil
+	}
+
+	_, err = a.status.UpdateTableMetadataCache(ctx, &serverpb.UpdateTableMetadataCacheRequest{Local: false})
+	if err != nil {
+		st, ok := status.FromError(err)
+		if ok {
+			if st.Code() == codes.Aborted {
+				return tmJobTriggeredResponse{JobTriggered: false, Message: JobRunning}, nil
+			}
+		}
+		return tmJobTriggeredResponse{}, err
+	}
+	return tmJobTriggeredResponse{JobTriggered: true, Message: JobTriggered}, nil
+}
+
 func (a *apiV2Server) updateTableMetadataJobAuthorized(
 	ctx context.Context, sqlUser username.SQLUsername,
 ) (isAuthorized bool, retErr error) {
@@ -835,3 +883,8 @@ type tmUpdateJobStatusResponse struct {
 	LastCompletedTime *time.Time `json:"last_completed_time"`
 	LastUpdatedTime   *time.Time `json:"last_updated_time"`
 }
+
+type tmJobTriggeredResponse struct {
+	JobTriggered bool             `json:"job_triggered"`
+	Message      JobStatusMessage `json:"message"`
+}