Update Database schema

Updates the database schema to match the columns provided in the historical city ratings. Adjust the seeder to reflect the changes. Signed-off-by: Rémy Greinhofer <[email protected]>
PeopleForBikes · Mar 10, 2024 · 902e68b · 902e68b
1 parent 29083a6
commit 902e68b
Show file tree

Hide file tree

Showing 27 changed files with 1,182 additions and 880 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -45,6 +45,7 @@ serde_with = { workspace = true }
 tokio = { workspace = true }
 
 [dev-dependencies]
-csv = "1.2.1"
-itertools = "0.12.0"
-bnacore = { git = "https://github.com/PeopleForBikes/brokenspoke", rev = "81c1c7f" }
+bnacore = { git = "https://github.com/PeopleForBikes/brokenspoke", rev = "b1f76eb" }
+# bnacore = { path = "../brokenspoke/bnacore" }
+csv = "1.3.0"
+itertools = "0.12.1"
diff --git a/docs/database.dbml b/docs/database.dbml
@@ -1,3 +1,8 @@
+Project brokenspoke_analyzer {
+  database_type: 'PostgreSQL'
+  Note: 'Brokenspoke-analyzer schema'
+}
+
 TableGroup analysis {
   bna.summary
   bna.features
@@ -10,8 +15,8 @@ TableGroup analysis {
 TableGroup city {
   bna.city
   bna.census
-  bna.ranking
   bna.speed_limit
+  bna.state_speed_limit
 }
 
 enum size {
@@ -29,7 +34,7 @@ enum approval_status {
 Table bna.summary [headercolor: #82F0D3] {
   bna_uuid uuid [pk]
   city_id uuid
-  created_at date
+  created_at date [not null]
   score float
   version varchar(10)
 
@@ -102,13 +107,17 @@ Table bna.infrastructure [headercolor: #82F0D3] {
 
 Table bna.city [headercolor: #6B41A3] {
   city_id uuid [pk]
-  country varchar(50)
-  latitude float
-  longitude float
-  name varchar(50)
-  region varchar(50)
-  state varchar(50)
-  state_abbrev char(2)
+  country varchar(50) [not null]
+  latitude float [not null]
+  longitude float [not null]
+  name varchar(50) [not null]
+  region varchar(50) [not null]
+  state varchar(50) [not null]
+  state_abbrev varchar(4)
+  size int [not null]
+  speed_limit int
+  created_at date [not null]
+  updated_at date
 
   indexes {
     city_id
@@ -119,7 +128,7 @@ Table bna.city [headercolor: #6B41A3] {
 Table bna.census [headercolor: #6B41A3] {
   census_id int [pk, increment]
   city_id uuid
-  created_at date
+  created_at date [not null]
   fips_code char(7)
   pop_size size
   population int
@@ -131,31 +140,28 @@ Table bna.census [headercolor: #6B41A3] {
   }
 }
 
-Table bna.ranking [headercolor: #6B41A3] {
-  ranking_id int [pk, increment]
+Table bna.speed_limit [headercolor: #6B41A3] {
+  speed_limit_id int [pk, increment]
   city_id uuid
-  country int
-  country_size int
-  created_at date
-  global int
-  size int
-  state int
+  created_at date [not null]
+  residential int
 
   indexes {
+    speed_limit_id
     city_id
-    ranking_id
   }
 }
 
-Table bna.speed_limit [headercolor: #6B41A3] {
-  speed_limit_id int [pk, increment]
-  city_id uuid
-  created_at date
-  residential int
+Table bna.state_speed_limit [headercolor: #6B41A3] {
+  state_abbrev char(2) [pk]
+  fips_code_state char(2) [not null, unique]
+  speed int [not null]
+  created_at date [not null]
+  updated_at date
 
   indexes {
-    city_id
-    speed_limit_id
+    state_abbrev
+    fips_code_state
   }
 }
 
@@ -172,6 +178,7 @@ Table bna.submission {
   fipscode varchar(50)
   consent varchar(50)
   status approval_status
+  created_at date [not null]
 
   indexes {
     submission_id
@@ -180,7 +187,6 @@ Table bna.submission {
 
 Ref: bna.city.city_id < bna.summary.city_id
 Ref: bna.city.city_id < bna.census.city_id
-Ref: bna.city.city_id < bna.ranking.city_id
 Ref: bna.city.city_id < bna.speed_limit.city_id
 Ref: bna.summary.bna_uuid - bna.core_services.bna_uuid
 Ref: bna.summary.bna_uuid - bna.opportunity.bna_uuid
@@ -208,10 +214,15 @@ Table bna.brokenspoke_pipeline {
   state brokenspoke_state
   sqs_message json
   neon_branch_id varchar(50)
-  fargate_task_id uuid
+  fargate_task_arn varchar(50) [unique]
   s3_bucket varchar(50)
+  start_time date [not null]
+  end_time date
+  torndown bool
 
   indexes {
     state_machine_id
   }
 }
+
+
diff --git a/docs/database.sql b/docs/database.sql
@@ -1,6 +1,6 @@
 -- SQL dump generated using DBML (dbml-lang.org)
 -- Database: PostgreSQL
--- Generated at: 2024-02-11T19:57:43.906Z
+-- Generated at: 2024-03-10T18:11:26.133Z
 
 CREATE SCHEMA "bna";
 
@@ -33,7 +33,7 @@ CREATE TYPE "brokenspoke_state" AS ENUM (
 CREATE TABLE "bna"."summary" (
   "bna_uuid" uuid PRIMARY KEY,
   "city_id" uuid,
-  "created_at" date,
+  "created_at" date NOT NULL,
   "score" float,
   "version" varchar(10)
 );
@@ -81,42 +81,43 @@ CREATE TABLE "bna"."infrastructure" (
 
 CREATE TABLE "bna"."city" (
   "city_id" uuid PRIMARY KEY,
-  "country" varchar(50),
-  "latitude" float,
-  "longitude" float,
-  "name" varchar(50),
-  "region" varchar(50),
-  "state" varchar(50),
-  "state_abbrev" char(2)
+  "country" varchar(50) NOT NULL,
+  "latitude" float NOT NULL,
+  "longitude" float NOT NULL,
+  "name" varchar(50) NOT NULL,
+  "region" varchar(50) NOT NULL,
+  "state" varchar(50) NOT NULL,
+  "state_abbrev" varchar(4),
+  "size" int NOT NULL,
+  "speed_limit" int,
+  "created_at" date NOT NULL,
+  "updated_at" date
 );
 
 CREATE TABLE "bna"."census" (
   "census_id" INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
   "city_id" uuid,
-  "created_at" date,
+  "created_at" date NOT NULL,
   "fips_code" char(7),
   "pop_size" size,
   "population" int
 );
 
-CREATE TABLE "bna"."ranking" (
-  "ranking_id" INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
-  "city_id" uuid,
-  "country" int,
-  "country_size" int,
-  "created_at" date,
-  "global" int,
-  "size" int,
-  "state" int
-);
-
 CREATE TABLE "bna"."speed_limit" (
   "speed_limit_id" INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
   "city_id" uuid,
-  "created_at" date,
+  "created_at" date NOT NULL,
   "residential" int
 );
 
+CREATE TABLE "bna"."state_speed_limit" (
+  "state_abbrev" char(2) PRIMARY KEY,
+  "fips_code_state" char(2) UNIQUE NOT NULL,
+  "speed" int NOT NULL,
+  "created_at" date NOT NULL,
+  "updated_at" date
+);
+
 CREATE TABLE "bna"."submission" (
   "submission_id" INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
   "firstname" varchar(50),
@@ -129,7 +130,8 @@ CREATE TABLE "bna"."submission" (
   "region" varchar(50),
   "fipscode" varchar(50),
   "consent" varchar(50),
-  "status" approval_status
+  "status" approval_status,
+  "created_at" date NOT NULL
 );
 
 CREATE TABLE "bna"."brokenspoke_pipeline" (
@@ -138,8 +140,11 @@ CREATE TABLE "bna"."brokenspoke_pipeline" (
   "state" brokenspoke_state,
   "sqs_message" json,
   "neon_branch_id" varchar(50),
-  "fargate_task_id" uuid,
-  "s3_bucket" varchar(50)
+  "fargate_task_arn" varchar(50) UNIQUE,
+  "s3_bucket" varchar(50),
+  "start_time" date NOT NULL,
+  "end_time" date,
+  "torndown" bool
 );
 
 CREATE INDEX ON "bna"."summary" ("bna_uuid");
@@ -166,13 +171,13 @@ CREATE INDEX ON "bna"."census" ("city_id");
 
 CREATE INDEX ON "bna"."census" ("fips_code");
 
-CREATE INDEX ON "bna"."ranking" ("city_id");
-
-CREATE INDEX ON "bna"."ranking" ("ranking_id");
+CREATE INDEX ON "bna"."speed_limit" ("speed_limit_id");
 
 CREATE INDEX ON "bna"."speed_limit" ("city_id");
 
-CREATE INDEX ON "bna"."speed_limit" ("speed_limit_id");
+CREATE INDEX ON "bna"."state_speed_limit" ("state_abbrev");
+
+CREATE INDEX ON "bna"."state_speed_limit" ("fips_code_state");
 
 CREATE INDEX ON "bna"."submission" ("submission_id");
 
@@ -182,8 +187,6 @@ ALTER TABLE "bna"."summary" ADD FOREIGN KEY ("city_id") REFERENCES "bna"."city"
 
 ALTER TABLE "bna"."census" ADD FOREIGN KEY ("city_id") REFERENCES "bna"."city" ("city_id");
 
-ALTER TABLE "bna"."ranking" ADD FOREIGN KEY ("city_id") REFERENCES "bna"."city" ("city_id");
-
 ALTER TABLE "bna"."speed_limit" ADD FOREIGN KEY ("city_id") REFERENCES "bna"."city" ("city_id");
 
 ALTER TABLE "bna"."core_services" ADD FOREIGN KEY ("bna_uuid") REFERENCES "bna"."summary" ("bna_uuid");