Skip to content

Commit

Permalink
Merge #131307
Browse files Browse the repository at this point in the history 
131307: server: add API to fetch database metadata for database id r=kyle-a-wong a=kyle-a-wong

Only the last 2 commits in the PR need to be reviewed

---
### server: refactor databases_metadata api
In preparation for an upcoming API, this commit refactors the query and mapping of rows to dbMetadata structs to separate methods to be reused.

Release note: None
Epic: [CRDB-37558](https://cockroachlabs.atlassian.net/browse/CRDB-37558)

### server: add API to fetch database metadata for database id
Adds a new API v2 endpoint, `GET /api/v2/database_metadata/<id>`, where id is a specific database id.

This API will return database metadata

Resolves: #131304
Epic: [CRDB-37558](https://cockroachlabs.atlassian.net/browse/CRDB-37558)
Release note: None

Co-authored-by: Kyle Wong <[email protected]>
  • Loading branch information
@kyle-a-wong
craig[bot] and kyle-a-wong committed Sep 27, 2024
2 parents 727fc55 + 3790e56 commit 7641988
Show file tree
Hide file tree
Showing 3 changed files with 247 additions and 78 deletions.
3 changes: 2 additions & 1 deletion pkg/server/api_v2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,8 @@ func registerRoutes(
{"rules/", a.listRules, false, authserver.RegularRole, true},

{"sql/", a.execSQL, true, authserver.RegularRole, true},
{"database_metadata/", a.GetDBMetadata, true, authserver.RegularRole, true},
{"database_metadata/", a.GetDbMetadata, true, authserver.RegularRole, true},
{"database_metadata/{database_id:[0-9]+}/", a.GetDbMetadataForId, true, authserver.RegularRole, true},
{"table_metadata/", a.GetTableMetadata, true, authserver.RegularRole, true},
{"table_metadata/{table_id:[0-9]+}/", a.GetTableMetadataWithDetails, true, authserver.RegularRole, true},
{"table_metadata/updatejob/", a.TableMetadataJob, true, authserver.RegularRole, true},
Expand Down
204 changes: 150 additions & 54 deletions pkg/server/api_v2_databases_metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ const (
const (
TableNotFound string = "table not found"
InvalidTableId string = "invalid table ID"

DatabaseNotFound string = "database not found"
InvalidDatabaseId string = "invalid database ID"
)

// GetTableMetadata returns a paginated response of table metadata and statistics. This is not a live view of
Expand Down Expand Up @@ -520,7 +523,7 @@ func rowToTableMetadata(scanner resultScanner, row tree.Datums) (tmd tableMetada
return tmd, nil
}

// GetDBMetadata returns a paginated response of database metadata and statistics. This is not a live view of
// GetDbMetadata returns a paginated response of database metadata and statistics. This is not a live view of
// the database data but instead is cached data that had been precomputed at an earlier time.
//
// The user making the request will receive database metadata based on the CONNECT database grant and admin privilege.
Expand Down Expand Up @@ -578,7 +581,7 @@ func rowToTableMetadata(scanner resultScanner, row tree.Datums) (tmd tableMetada
// description: A paginated response of dbMetadata results.
// "400":
// description: Bad request. If the provided query parameters are invalid.
func (a *apiV2Server) GetDBMetadata(w http.ResponseWriter, r *http.Request) {
func (a *apiV2Server) GetDbMetadata(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ctx = a.sqlServer.AnnotateCtx(ctx)
sqlUser := authserver.UserFromHTTPAuthInfoContext(ctx)
Expand Down Expand Up @@ -648,7 +651,7 @@ func (a *apiV2Server) GetDBMetadata(w http.ResponseWriter, r *http.Request) {
dbNameFilter = fmt.Sprintf("%%%s%%", dbName)
}

dbm, totalRowCount, err := a.getDBMetadata(ctx, sqlUser, dbNameFilter, storeIds, sortBy, sortOrder, pageSize, offset)
dbm, totalRowCount, err := a.getDbMetadata(ctx, sqlUser, dbNameFilter, storeIds, sortBy, sortOrder, pageSize, offset)

if err != nil {
srverrors.APIV2InternalError(ctx, err, w)
Expand All @@ -664,10 +667,63 @@ func (a *apiV2Server) GetDBMetadata(w http.ResponseWriter, r *http.Request) {
},
}
apiutil.WriteJSONResponse(ctx, w, 200, resp)
}

// GetDbMetadataForId fetches database metadata for a specific database id.
//
// The user making the request must have the CONNECT database grant for the database, or the admin privilege.
//
// ---
// parameters:
//
// - name: database_id
// type: integer
// description: The id of the database to fetch database metadata.
// in: path
// required: false
//
// produces:
// - application/json
//
// responses:
//
// "200":
// description: A dbMetadataWithDetailsResponse containing the database metadata.
// "404":
// description: If the database for the provided id doesn't exist or the user doesn't have necessary permissions
// to access the database
func (a *apiV2Server) GetDbMetadataForId(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
return
}

ctx := a.sqlServer.AnnotateCtx(r.Context())
sqlUser := authserver.UserFromHTTPAuthInfoContext(ctx)
pathVars := mux.Vars(r)
databaseId, err := strconv.Atoi(pathVars["database_id"])
if err != nil {
http.Error(w, InvalidDatabaseId, http.StatusBadRequest)
return
}
dbm, err := a.getDbMetadataForId(ctx, sqlUser, databaseId)
if err != nil {
srverrors.APIV2InternalError(ctx, err, w)
return
}

// No db id means table couldn't be found or user doesn't have access to the table
if dbm.DbId == 0 {
http.Error(w, DatabaseNotFound, http.StatusNotFound)
return
}
resp := dbMetadataWithDetailsResponse{
Metadata: dbm,
}
apiutil.WriteJSONResponse(ctx, w, 200, resp)
}

func (a *apiV2Server) getDBMetadata(
func (a *apiV2Server) getDbMetadata(
ctx context.Context,
sqlUser username.SQLUsername,
dbName string,
Expand All @@ -677,39 +733,8 @@ func (a *apiV2Server) getDBMetadata(
limit int,
offset int,
) (dbms []dbMetadata, totalRowCount int64, retErr error) {
sqlUserStr := sqlUser.Normalized()
dbms = make([]dbMetadata, 0)
query := safesql.NewQuery()

// Base query aggregates table metadata by db_id. It joins on a subquery which flattens
// and deduplicates all store ids for tables in a database into a single array. This query
// will only return databases that the provided sql user has CONNECT privileges to. If they
// are an admin, they have access to all databases.
query.Append(`SELECT
n.id as db_id,
n.name as db_name,
COALESCE(sum(tbm.replication_size_bytes)::INT, 0) as size_bytes,
count(CASE WHEN tbm.table_type = 'TABLE' THEN 1 ELSE NULL END) as table_count,
max(tbm.last_updated) as last_updated,
COALESCE(s.store_ids, ARRAY[]) as store_ids,
count(*) OVER() as total_row_count
FROM system.namespace n
LEFT JOIN system.table_metadata tbm ON n.id = tbm.db_id
LEFT JOIN system.role_members rm ON rm.role = 'admin' AND member = $
LEFT JOIN (
SELECT db_id, array_agg(DISTINCT unnested_ids) as store_ids
FROM system.table_metadata, unnest(store_ids) as unnested_ids
GROUP BY db_id
) s ON s.db_id = tbm.db_id
WHERE (rm.role = 'admin' OR n.name in (
SELECT cdp.database_name
FROM "".crdb_internal.cluster_database_privileges cdp
WHERE grantee = $
AND privilege_type = 'CONNECT'
))
AND n."parentID" = 0
AND n."parentSchemaID" = 0
`, sqlUserStr, sqlUserStr)
query := getDatabaseMetadataBaseQuery(sqlUser.Normalized())

if dbName != "" {
query.Append("AND n.name ILIKE $ ", dbName)
Expand Down Expand Up @@ -738,7 +763,7 @@ func (a *apiV2Server) getDBMetadata(
query.Append("LIMIT $ ", limit)
query.Append("OFFSET $ ", offset)

it, err := a.admin.internalExecutor.QueryIteratorEx(
it, err := a.sqlServer.internalExecutor.QueryIteratorEx(
ctx, "get-database-metadata", nil, /* txn */
sessiondata.NodeUserSessionDataOverride,
query.String(), query.QueryArguments()...,
Expand All @@ -762,30 +787,15 @@ func (a *apiV2Server) getDBMetadata(
// If ok == false, the query returned 0 rows.
scanner := makeResultScanner(it.Types())
for ; ok; ok, err = it.Next(ctx) {
var dbm dbMetadata
row := it.Cur()
if setTotalRowCount {
if err := scanner.Scan(row, "total_row_count", &totalRowCount); err != nil {
return nil, totalRowCount, err
}
setTotalRowCount = false
}
if err := scanner.Scan(row, "db_id", &dbm.DbId); err != nil {
return nil, 0, err
}
if err := scanner.Scan(row, "db_name", &dbm.DbName); err != nil {
return nil, 0, err
}
if err := scanner.Scan(row, "size_bytes", &dbm.SizeBytes); err != nil {
return nil, 0, err
}
if err := scanner.Scan(row, "table_count", &dbm.TableCount); err != nil {
return nil, 0, err
}
if err := scanner.Scan(row, "store_ids", &dbm.StoreIds); err != nil {
return nil, totalRowCount, err
}
if err := scanner.Scan(row, "last_updated", &dbm.LastUpdated); err != nil {
dbm, err := rowToDatabaseMetadata(scanner, row)
if err != nil {
return nil, 0, err
}
dbms = append(dbms, dbm)
Expand All @@ -798,6 +808,88 @@ func (a *apiV2Server) getDBMetadata(
return dbms, totalRowCount, nil
}

func (a *apiV2Server) getDbMetadataForId(
ctx context.Context, sqlUser username.SQLUsername, dbId int,
) (dbMetadata, error) {
query := getDatabaseMetadataBaseQuery(sqlUser.Normalized())
query.Append("AND n.id = $ ", dbId)
query.Append("GROUP BY n.id, n.name, s.store_ids ")

row, types, err := a.sqlServer.internalExecutor.QueryRowExWithCols(ctx, "get-db-metadata-for-id", nil,
sessiondata.NodeUserSessionDataOverride, query.String(), query.QueryArguments()...)

if err != nil {
return dbMetadata{}, err
}

if row == nil {
return dbMetadata{}, nil
}

scanner := makeResultScanner(types)
return rowToDatabaseMetadata(scanner, row)
}

func getDatabaseMetadataBaseQuery(userName string) *safesql.Query {
query := safesql.NewQuery()

// Base query aggregates table metadata by db_id. It joins on a subquery which flattens
// and deduplicates all store ids for tables in a database into a single array. This query
// will only return databases that the provided sql user has CONNECT privileges to. If they
// are an admin, they have access to all databases.
query.Append(`SELECT
n.id as db_id,
n.name as db_name,
COALESCE(sum(tbm.replication_size_bytes)::INT, 0) as size_bytes,
count(CASE WHEN tbm.table_type = 'TABLE' THEN 1 ELSE NULL END) as table_count,
max(tbm.last_updated) as last_updated,
COALESCE(s.store_ids, ARRAY[]) as store_ids,
count(*) OVER() as total_row_count
FROM system.namespace n
LEFT JOIN system.table_metadata tbm ON n.id = tbm.db_id
LEFT JOIN system.role_members rm ON rm.role = 'admin' AND member = $
LEFT JOIN (
SELECT db_id, array_agg(DISTINCT unnested_ids) as store_ids
FROM system.table_metadata, unnest(store_ids) as unnested_ids
GROUP BY db_id
) s ON s.db_id = tbm.db_id
WHERE (rm.role = 'admin' OR n.name in (
SELECT cdp.database_name
FROM "".crdb_internal.cluster_database_privileges cdp
WHERE grantee = $
AND privilege_type = 'CONNECT'
))
AND n."parentID" = 0
AND n."parentSchemaID" = 0
`, userName, userName)

return query
}

func rowToDatabaseMetadata(scanner resultScanner, row tree.Datums) (dbm dbMetadata, err error) {
var emptyMetadata dbMetadata
if err = scanner.Scan(row, "db_id", &dbm.DbId); err != nil {
return emptyMetadata, err
}
if err = scanner.Scan(row, "db_name", &dbm.DbName); err != nil {
return emptyMetadata, err
}
if err = scanner.Scan(row, "size_bytes", &dbm.SizeBytes); err != nil {
return emptyMetadata, err
}
if err = scanner.Scan(row, "table_count", &dbm.TableCount); err != nil {
return emptyMetadata, err
}
if err = scanner.Scan(row, "store_ids", &dbm.StoreIds); err != nil {
return emptyMetadata, err
}
if err = scanner.Scan(row, "last_updated", &dbm.LastUpdated); err != nil {
return emptyMetadata, err
}

return dbm, nil
}

// TableMetadataJob routes to the necessary receiver based on the http method of the request. Requires
// The user making the request must have the CONNECT database grant on at least one database or admin privilege.
// ---
Expand Down Expand Up @@ -1023,3 +1115,7 @@ type tableMetadataWithDetailsResponse struct {
Metadata tableMetadata `json:"metadata"`
CreateStatement string `json:"create_statement"`
}

type dbMetadataWithDetailsResponse struct {
Metadata dbMetadata `json:"metadata"`
}
Loading

0 comments on commit 7641988

Please sign in to comment.