Merge #82638 #130693 #130830

82638: docs/rfcs: split up jobs system tables r=dt a=dt

Release note: none.

Epic: none.

130693: cluster-ui: new db page components r=xinhaoz a=xinhaoz

Only the latest 2 commits should be reviewed.

-------------------------
cluster-ui: add layouts dir with some base layout components

This commit adds shared layout components to align spacing
within new db console pages.

Components to start off:
- PageLayout - page content wrapper
- PageSection - page section wrapper

Epic: [CRDB-37558](https://cockroachlabs.atlassian.net/browse/CRDB-37558)
Release note: None

-----------------------------

cluster-ui: new db page components

This commit adds the new db page, with components under
the `databasesv2` dir. The page currently uses mocked data
and does not support pagination or sorting.

The page is available for preview under the new route,
`/v2/databases`. We'll eventually replace the existing
db routes with the new components.

Epic: none
Fixes: #130674
Release note: None

130830: crosscluster/logical: use poller-collection for replicated time r=dt a=dt

    jobs/metricspoller: add optional 'highwater' metric from polled progress

    Many of our jobs maintain 'highwater' timestamps in their progress, reflecting the
    timestamp up to which they have processed. Many users of these jobs wish to monitor them
    via timeseries and exported metrics. However exporting these highwater timestamps via
    metrics has proven difficult to do directly from a running job for a few reasons, including
    the fact that many jobs may be running, so we need to reliably export the worst-case number
    and that each node exports its own number, so an aggregator will need to find the worst
    across those nodes. While this all is possible, an added complication is that if no node
    is executing a given job, its number is not moving and thus likely the most important to
    include in the worst-case, but when each job is responsible for pushing its number into the
    metric, this does not happen.

    This change instead extends our 'metrics poller' job that inspects the state of the jobs
    system to update certain metrics so that it can be asked to find the minimum timestamp
    across all jobs of a given type and update a metric for that type to the found number.

    Release note: none.
    Epic: none.

Co-authored-by: David Taylor <[email protected]>
Co-authored-by: Xin Hao Zhang <[email protected]>

pkg/ccl/crosscluster/logical/logical_replication_job.go

@@ -540,8 +540,6 @@ func (rh *rowHandler) handleRow(ctx context.Context, row tree.Datums) error {
 	case <-ctx.Done():
 		return ctx.Err()
 	}
-
-	rh.metrics.ReplicatedTimeSeconds.Update(replicatedTime.GoTime().Unix())
 	return nil
 }
 
@@ -601,8 +599,6 @@ func (r *logicalReplicationResumer) OnFailOrCancel(
 	ctx context.Context, execCtx interface{}, _ error,
 ) error {
 	execCfg := execCtx.(sql.JobExecContext).ExecCfg()
-	metrics := execCfg.JobRegistry.MetricsStruct().JobSpecificMetrics[jobspb.TypeLogicalReplication].(*Metrics)
-	metrics.ReplicatedTimeSeconds.Update(0)
 
 	// Remove the LDR job ID from the destination table descriptors.
 	details := r.job.Details().(jobspb.LogicalReplicationDetails)
@@ -689,14 +685,16 @@ func getRetryPolicy(knobs *sql.StreamingTestingKnobs) retry.Options {
 }
 
 func init() {
+	m := MakeMetrics(base.DefaultHistogramWindowInterval())
 	jobs.RegisterConstructor(
 		jobspb.TypeLogicalReplication,
 		func(job *jobs.Job, _ *cluster.Settings) jobs.Resumer {
 			return &logicalReplicationResumer{
 				job: job,
 			}
 		},
-		jobs.WithJobMetrics(MakeMetrics(base.DefaultHistogramWindowInterval())),
+		jobs.WithJobMetrics(m),
+		jobs.WithResolvedMetric(m.(*Metrics).ReplicatedTimeSeconds),
 		jobs.UsesTenantCostControl,
 	)
 }

pkg/jobs/metrics.go

@@ -36,6 +36,9 @@ type Metrics struct {
 	// implementation of job specific metrics.
 	JobSpecificMetrics [jobspb.NumJobTypes]metric.Struct
 
+	// ResolvedMetrics are the per job type metrics for resolved timestamps.
+	ResolvedMetrics [jobspb.NumJobTypes]*metric.Gauge
+
 	// RunningNonIdleJobs is the total number of running jobs that are not idle.
 	RunningNonIdleJobs *metric.Gauge
 
@@ -288,8 +291,13 @@ func (m *Metrics) init(histogramWindowInterval time.Duration, lookup *cidr.Looku
 			ExpiredPTS:             metric.NewCounter(makeMetaExpiredPTS(typeStr)),
 			ProtectedAge:           metric.NewGauge(makeMetaProtectedAge(typeStr)),
 		}
-		if opts, ok := getRegisterOptions(jt); ok && opts.metrics != nil {
-			m.JobSpecificMetrics[jt] = opts.metrics
+		if opts, ok := getRegisterOptions(jt); ok {
+			if opts.metrics != nil {
+				m.JobSpecificMetrics[jt] = opts.metrics
+			}
+			if opts.resolvedMetric != nil {
+				m.ResolvedMetrics[jt] = opts.resolvedMetric
+			}
 		}
 	}
 }

pkg/jobs/metricspoller/job_statistics.go

@@ -81,6 +81,48 @@ func updatePausedMetrics(ctx context.Context, execCtx sql.JobExecContext) error
 	return nil
 }
 
+// lowTSForTypeQuery finds the lowest non-null ts for all jobs of a given type.
+// min() ignores nulls; jobs of the type that do not have a ts are not included
+// in the result, e.g. a new changefeed that is still in initial scan would not
+// change the result of the query, but if after its initial scan it was several
+// minutes behind and was the most lagged changefeed, that would be reflected.
+const lowTSForTypeQuery = `SELECT min(high_water_timestamp) FROM crdb_internal.jobs WHERE job_type = $1 AND status IN ` + jobs.NonTerminalStatusTupleString
+
+// updateTSMetrics updates the metrics for jobs that have registered for ts
+// tracking.
+func updateTSMetrics(ctx context.Context, execCtx sql.JobExecContext) error {
+	for _, typ := range jobspb.Type_value {
+		m := execCtx.ExecCfg().JobRegistry.MetricsStruct().ResolvedMetrics[typ]
+		// If this job type does not register a resolved TS metric, skip it.
+		if m == nil {
+			continue
+		}
+
+		var ts hlc.Timestamp
+		if err := execCtx.ExecCfg().InternalDB.Txn(ctx, func(ctx context.Context, txn isql.Txn) error {
+			if err := txn.KV().SetUserPriority(roachpb.MinUserPriority); err != nil {
+				return err
+			}
+			row, err := txn.QueryRowEx(
+				ctx, "poll-jobs-metrics-ts", txn.KV(), sessiondata.NodeUserSessionDataOverride,
+				lowTSForTypeQuery, jobspb.Type(typ).String(),
+			)
+			// Feeding zero non-null rows to min() returns null; return and record
+			// a zero ts (i.e. no data.) in this case or an error case.
+			if err != nil || row == nil || row[0] == tree.DNull {
+				return err
+			}
+			d := *row[0].(*tree.DDecimal)
+			ts, err = hlc.DecimalToHLC(&d.Decimal)
+			return err
+		}); err != nil {
+			return errors.Wrap(err, "could not query jobs table")
+		}
+		m.Update(ts.GoTime().Unix())
+	}
+	return nil
+}
+
 type ptsStat struct {
 	numRecords int64
 	expired    int64

pkg/jobs/metricspoller/poller.go

@@ -94,6 +94,7 @@ type pollerMetrics struct {
 var metricPollerTasks = map[string]func(ctx context.Context, execCtx sql.JobExecContext) error{
 	"paused-jobs": updatePausedMetrics,
 	"manage-pts":  manageProtectedTimestamps,
+	"resolved-ts": updateTSMetrics,
 }
 
 func (m pollerMetrics) MetricStruct() {}

pkg/jobs/registry.go

@@ -1413,6 +1413,14 @@ func WithJobMetrics(m metric.Struct) RegisterOption {
 	}
 }
 
+// WithResolvedMetric registers a gauge metric that the poller will update to
+// reflect the minimum resolved timestamp of all the jobs of this type.
+func WithResolvedMetric(m *metric.Gauge) RegisterOption {
+	return func(opts *registerOptions) {
+		opts.resolvedMetric = m
+	}
+}
+
 // registerOptions are passed to RegisterConstructor and control how a job
 // resumer is created and configured.
 type registerOptions struct {
@@ -1427,6 +1435,9 @@ type registerOptions struct {
 
 	// metrics allow jobs to register job specific metrics.
 	metrics metric.Struct
+
+	// resolvedMetric, if set, is the metric to update using the min resolved ts.
+	resolvedMetric *metric.Gauge
 }
 
 // JobResultsReporter is an interface for reporting the results of the job execution.

pkg/ui/workspaces/cluster-ui/src/databasesV2/databaseTypes.ts

@@ -0,0 +1,20 @@
+// Copyright 2024 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+export type DatabaseRow = {
+  name: string;
+  id: number;
+  approximateDiskSizeMiB: number;
+  tableCount: number;
+  rangeCount: number;
+  nodesByRegion: Record<string, number[]>;
+  schemaInsightsCount: number;
+  key: string;
+};

pkg/ui/workspaces/cluster-ui/src/databasesV2/index.tsx

@@ -0,0 +1,168 @@
+// Copyright 2024 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+import { Space } from "antd";
+import React, { useState } from "react";
+import { Link } from "react-router-dom";
+import Select, { OptionsType } from "react-select";
+
+import { PageLayout, PageSection } from "src/layouts";
+import { PageConfig, PageConfigItem } from "src/pageConfig";
+import PageCount from "src/sharedFromCloud/pageCount";
+import { PageHeader } from "src/sharedFromCloud/pageHeader";
+import { Search } from "src/sharedFromCloud/search";
+import { ReactSelectOption } from "src/types/selectTypes";
+
+import { Table, TableColumnProps } from "../sharedFromCloud/table";
+import useTable from "../sharedFromCloud/useTable";
+import { Bytes, EncodeDatabaseUri } from "../util";
+
+import { DatabaseRow } from "./databaseTypes";
+
+const mockRegionOptions = [
+  { label: "US East (N. Virginia)", value: "us-east-1" },
+  { label: "US East (Ohio)", value: "us-east-2" },
+];
+
+const mockData: DatabaseRow[] = new Array(20).fill(1).map((_, i) => ({
+  name: `myDB-${i}`,
+  id: i,
+  approximateDiskSizeMiB: i * 100,
+  tableCount: i,
+  rangeCount: i,
+  nodesByRegion: {
+    [mockRegionOptions[0].value]: [1, 2],
+    [mockRegionOptions[1].value]: [3],
+  },
+  schemaInsightsCount: i,
+  key: i.toString(),
+}));
+
+const filters = {};
+
+const initialParams = {
+  filters,
+  pagination: {
+    page: 1,
+    pageSize: 10,
+  },
+  search: "",
+  sort: {
+    field: "name",
+    order: "asc" as const,
+  },
+};
+
+const columns: TableColumnProps<DatabaseRow>[] = [
+  {
+    title: "Name",
+    sorter: true,
+    render: (db: DatabaseRow) => {
+      const encodedDBPath = EncodeDatabaseUri(db.name);
+      // TODO (xzhang: For CC we have to use `${location.pathname}/${db.name}`
+      return <Link to={encodedDBPath}>{db.name}</Link>;
+    },
+  },
+  {
+    title: "Size",
+    sorter: true,
+    render: (db: DatabaseRow) => {
+      return Bytes(db.approximateDiskSizeMiB);
+    },
+  },
+  {
+    title: "Tables",
+    sorter: true,
+    render: (db: DatabaseRow) => {
+      return db.tableCount;
+    },
+  },
+  {
+    title: "Ranges",
+    sorter: true,
+    render: (db: DatabaseRow) => {
+      return db.rangeCount;
+    },
+  },
+  {
+    title: "Regions / Nodes",
+    render: (db: DatabaseRow) => (
+      <Space direction="vertical">
+        {db.nodesByRegion &&
+          Object.keys(db.nodesByRegion).map(
+            region => `${region}: ${db.nodesByRegion[region].length}`,
+          )}
+      </Space>
+    ),
+  },
+  {
+    title: "Schema insights",
+    render: (db: DatabaseRow) => {
+      return db.schemaInsightsCount;
+    },
+  },
+];
+
+export const DatabasesPageV2 = () => {
+  const { setSearch } = useTable({
+    initial: initialParams,
+  });
+  const data = mockData;
+
+  const [nodeRegions, setNodeRegions] = useState<ReactSelectOption[]>([]);
+  const onNodeRegionsChange = (selected: OptionsType<ReactSelectOption>) => {
+    setNodeRegions((selected ?? []).map(v => v));
+  };
+
+  return (
+    <PageLayout>
+      <PageHeader title="Databases" />
+      <PageSection>
+        <PageConfig>
+          <PageConfigItem>
+            <Search placeholder="Search databases" onSubmit={setSearch} />
+          </PageConfigItem>
+          <PageConfigItem minWidth={"200px"}>
+            <Select
+              placeholder={"Regions"}
+              name="nodeRegions"
+              options={mockRegionOptions}
+              clearable={true}
+              isMulti
+              value={nodeRegions}
+              onChange={onNodeRegionsChange}
+            />
+          </PageConfigItem>
+        </PageConfig>
+      </PageSection>
+      <PageSection>
+        <PageCount
+          page={1}
+          pageSize={10}
+          total={data.length}
+          entity="databases"
+        />
+        <Table
+          columns={columns}
+          dataSource={data}
+          pagination={{
+            size: "small",
+            current: 1,
+            pageSize: 10,
+            showSizeChanger: false,
+            position: ["bottomCenter"],
+            total: data.length,
+          }}
+          onChange={(_pagination, _sorter) => {}}
+        />
+      </PageSection>
+    </PageLayout>
+  );
+};

pkg/ui/workspaces/cluster-ui/src/index.ts

@@ -57,6 +57,7 @@ export * from "./contexts";
 export * from "./timestamp";
 export * from "./databases";
 export * from "./antdTheme";
+export * from "./databasesV2";
 // Reexport ConfigProvider instance from cluster-ui as exact instance
 // required in Db Console to apply Antd theme in Db Console.
 // TODO (koorosh): is it possible to define antd pacakge as peerDependency

pkg/ui/workspaces/cluster-ui/src/layouts/index.ts

@@ -0,0 +1,12 @@
+// Copyright 2024 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+export * from "./pageSection";
+export * from "./pageLayout";

pkg/ui/workspaces/cluster-ui/src/layouts/pageLayout.module.scss

@@ -0,0 +1,7 @@
+@import "src/core/index.module";
+
+.page-layout {
+  flex-grow: 0;
+  width: 100%;
+  padding-right: 16px;
+}

pkg/ui/workspaces/cluster-ui/src/layouts/pageLayout.tsx

@@ -0,0 +1,20 @@
+// Copyright 2024 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+import classNames from "classnames/bind";
+import React from "react";
+
+import styles from "./pageLayout.module.scss";
+
+const cx = classNames.bind(styles);
+
+export const PageLayout: React.FC = ({ children }) => {
+  return <div className={cx("page-layout")}>{children}</div>;
+};

pkg/ui/workspaces/cluster-ui/src/layouts/pageSection.module.scss

@@ -0,0 +1,6 @@
+@import "src/core/index.module";
+
+.page-section {
+  margin-bottom: 16px;
+}
+