Skip to content

Commit

Permalink
[scd/crdb] Add columns ovn and past_ovns to scd_operations table; Bum…
Browse files Browse the repository at this point in the history
…p schema_version to v3.2.0
  • Loading branch information
mickmis committed Sep 2, 2024
1 parent 02f8ac9 commit dfa1062
Show file tree
Hide file tree
Showing 10 changed files with 74 additions and 11 deletions.
2 changes: 1 addition & 1 deletion build/db_schemas/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,6 @@ places:
schema_versions.schema_version
* [DSS main.jsonnet](../deploy/examples/minimum/main.jsonnet)
* [Schema manager main.jsonnet](../deploy/examples/schema_manager/main.jsonnet)
* scd_ or rid_ bootstrapper.sh in [dev/startup](../dev/startup)
* /pkg/{rid|scd}/store/cockroach/store.go
* /deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf
* /deploy/services/helm-charts/dss/templates/schema-manager.yaml
2 changes: 2 additions & 0 deletions build/db_schemas/scd.libsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
"upto-v2.0.0-support_api_1_0_0.sql": importstr "scd/upto-v2.0.0-support_api_1_0_0.sql",
"upto-v3.0.0-add_inverted_indices.sql": importstr "scd/upto-v3.0.0-add_inverted_indices.sql",
"upto-v3.1.0-create_uss_availability.sql": importstr "scd/upto-v3.1.0-create_uss_availability.sql",
"upto-v3.2.0-add_ovn_columns.sql": importstr "scd/upto-v3.2.0-add_ovn_columns.sql",
"downfrom-v3.2.0-remove_ovn_columns.sql": importstr "scd/downfrom-v3.2.0-remove_ovn_columns.sql",
"downfrom-v3.1.0-remove_uss_availability.sql": importstr "scd/downfrom-v3.1.0-remove_uss_availability.sql",
"downfrom-v3.0.0-remove_inverted_indices.sql": importstr "scd/downfrom-v3.0.0-remove_inverted_indices.sql",
"downfrom-v2.0.0-remove_api_1_0_0_support.sql": importstr "scd/downfrom-v2.0.0-remove_api_1_0_0_support.sql",
Expand Down
7 changes: 7 additions & 0 deletions build/db_schemas/scd/downfrom-v3.2.0-remove_ovn_columns.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
ALTER TABLE scd_operations
DROP IF EXISTS ovn,
DROP IF EXISTS past_ovns;

UPDATE schema_versions
SET schema_version = 'v3.1.0'
WHERE onerow_enforcer = TRUE;
14 changes: 14 additions & 0 deletions build/db_schemas/scd/upto-v3.2.0-add_ovn_columns.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
ALTER TABLE scd_operations
ADD COLUMN IF NOT EXISTS ovn STRING
CHECK (ovn != ''), -- ovn must be NULL if unspecified, not an empty string
ADD COLUMN IF NOT EXISTS past_ovns STRING[] NOT NULL
DEFAULT ARRAY []::STRING[]
CHECK (
array_position(past_ovns, NULL) IS NULL AND
array_position(past_ovns, '') IS NULL AND
array_position(past_ovns, ovn) IS NULL
); -- past_ovns must not contain NULL elements, empty strings or current ovn

UPDATE schema_versions
SET schema_version = 'v3.2.0'
WHERE onerow_enforcer = TRUE;
2 changes: 1 addition & 1 deletion build/deploy/examples/minimum/main.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ local metadata = metadataBase {
schema_manager+: {
image: 'VAR_DOCKER_IMAGE_NAME',
desired_rid_db_version: '4.0.0',
desired_scd_db_version: '3.1.0',
desired_scd_db_version: '3.2.0',
},
prometheus+: {
storageClass: 'VAR_STORAGE_CLASS',
Expand Down
2 changes: 1 addition & 1 deletion build/deploy/examples/schema_manager/main.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ local metadata = metadataBase {
schema_manager+: {
image: 'VAR_DOCKER_IMAGE_NAME',
desired_rid_db_version: '4.0.0',
desired_scd_db_version: '3.1.0',
desired_scd_db_version: '3.2.0',
},
};

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
locals {
rid_db_schema = var.desired_rid_db_version == "latest" ? "4.0.0" : var.desired_rid_db_version
scd_db_schema = var.desired_scd_db_version == "latest" ? "3.1.0" : var.desired_scd_db_version
scd_db_schema = var.desired_scd_db_version == "latest" ? "3.2.0" : var.desired_scd_db_version
}
4 changes: 2 additions & 2 deletions deploy/services/helm-charts/dss/templates/schema-manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
{{- $jobVersion := .Release.Revision -}} {{/* Jobs template definition is immutable, using the revision in the name forces the job to be recreated at each helm upgrade. */}}
{{- $waitForCockroachDB := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $cockroachHost)) -}}

{{- range $service, $schemaVersion := dict "rid" "4.0.0" "scd" "3.1.0" }}
{{- range $service, $schemaVersion := dict "rid" "4.0.0" "scd" "3.2.0" }}
---
apiVersion: batch/v1
kind: Job
Expand Down Expand Up @@ -50,4 +50,4 @@ spec:
volumes:
{{- include "ca-certs:volume" . | nindent 8 }}
{{- include "client-certs:volume" . | nindent 8 }}
{{- end -}}
{{- end -}}
1 change: 1 addition & 0 deletions pkg/scd/models/operational_intents.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ type OperationalIntent struct {
Version VersionNumber
State OperationalIntentState
OVN OVN
PastOVNs []OVN
StartTime *time.Time
EndTime *time.Time
USSBaseURL string
Expand Down
49 changes: 44 additions & 5 deletions pkg/scd/store/cockroach/operational_intents.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,14 @@ import (
scdmodels "github.com/interuss/dss/pkg/scd/models"
dsssql "github.com/interuss/dss/pkg/sql"
"github.com/interuss/stacktrace"
"github.com/jackc/pgx/v5/pgtype"

"github.com/jackc/pgx/v5"
"github.com/pkg/errors"
)

var (
operationFieldsWithIndices [12]string
operationFieldsWithIndices [14]string
operationFieldsWithPrefix string
operationFieldsWithoutPrefix string
)
Expand All @@ -36,6 +37,8 @@ func init() {
operationFieldsWithIndices[9] = "updated_at"
operationFieldsWithIndices[10] = "state"
operationFieldsWithIndices[11] = "cells"
operationFieldsWithIndices[12] = "ovn"
operationFieldsWithIndices[13] = "past_ovns"

operationFieldsWithoutPrefix = strings.Join(
operationFieldsWithIndices[:], ",",
Expand All @@ -58,8 +61,12 @@ func (s *repo) fetchOperationalIntents(ctx context.Context, q dsssql.Queryable,
}
defer rows.Close()

var payload []*scdmodels.OperationalIntent
var cids []int64
var (
payload []*scdmodels.OperationalIntent
cids []int64
ovn pgtype.Text
pastOVNs []string
)
ussAvailabilities := map[dssmodels.Manager]scdmodels.UssAvailabilityState{}
for rows.Next() {
var (
Expand All @@ -79,11 +86,27 @@ func (s *repo) fetchOperationalIntents(ctx context.Context, q dsssql.Queryable,
&updatedAt,
&o.State,
&cids,
&ovn,
&pastOVNs,
)
if err != nil {
return nil, stacktrace.Propagate(err, "Error scanning Operation row")
}
o.OVN = scdmodels.NewOVNFromTime(updatedAt, o.ID.String())

// If the managing USS has requested a specific OVN on this operational intent, it will be persisted in DB.
// If not, a default DSS-generated OVN based on the last update time is used.
// See https://github.com/interuss/dss/issues/1078 for more details.
if ovn.Valid {
o.OVN = scdmodels.OVN(ovn.String)
} else {
o.OVN = scdmodels.NewOVNFromTime(updatedAt, o.ID.String())
}

o.PastOVNs = make([]scdmodels.OVN, 0, len(pastOVNs))
for _, pastOVN := range pastOVNs {
o.PastOVNs = append(o.PastOVNs, scdmodels.OVN(pastOVN))
}

o.SetCells(cids)
ussAvailabilities[o.Manager] = scdmodels.UssAvailabilityStateUnknown
payload = append(payload, o)
Expand Down Expand Up @@ -177,7 +200,7 @@ func (s *repo) UpsertOperationalIntent(ctx context.Context, operation *scdmodels
scd_operations
(%s)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8, $9, transaction_timestamp(), $10, $11)
($1, $2, $3, $4, $5, $6, $7, $8, $9, transaction_timestamp(), $10, $11, $12, $13)
RETURNING
%s`, operationFieldsWithoutPrefix, operationFieldsWithPrefix)
)
Expand All @@ -199,6 +222,20 @@ func (s *repo) UpsertOperationalIntent(ctx context.Context, operation *scdmodels
if err != nil {
return nil, stacktrace.Propagate(err, "Failed to convert id to PgUUID")
}

var ovn pgtype.Text // if the OVN is to be generated by the DSS, it must be NULL in DB, not just an empty string
if operation.OVN != "" {
ovn = pgtype.Text{
String: operation.OVN.String(),
Valid: true,
}
}

pastOVNs := make([]string, 0, len(operation.PastOVNs))
for _, pastOVN := range operation.PastOVNs {
pastOVNs = append(pastOVNs, pastOVN.String())
}

operation, err = s.fetchOperationalIntent(ctx, s.q, upsertOperationsQuery,
opid,
operation.Manager,
Expand All @@ -211,6 +248,8 @@ func (s *repo) UpsertOperationalIntent(ctx context.Context, operation *scdmodels
subid,
operation.State,
cids,
ovn,
pastOVNs,
)
if err != nil {
return nil, stacktrace.Propagate(err, "Error fetching Operation")
Expand Down

0 comments on commit dfa1062

Please sign in to comment.