Main

.vscode/settings.json

@@ -17,7 +17,6 @@
   "files.autoSave": "afterDelay",
   "screencastMode.onlyKeyboardShortcuts": true,
   "terminal.integrated.fontSize": 18,
-  "workbench.activityBar.visible": true,
   "workbench.colorTheme": "Visual Studio Dark",
   "workbench.fontAliasing": "antialiased",
   "workbench.statusBar.visible": true

nyc_parking_violations/.gitignore

@@ -1,4 +1,4 @@
-.user.yml
+
 target/
 dbt_packages/
 logs/

nyc_parking_violations/.user.yml

@@ -0,0 +1 @@
+id: c977a4b5-58c9-46f5-8b95-8b94e46a4940

nyc_parking_violations/dbt_project.yml

@@ -34,7 +34,7 @@ models:
   nyc_parking_violations:
     # Config indicated by + and applies to all files under models/example/
     example:
-      +materialized: ephemeral
+      +materialized: view
     bronze:
       +materialized: view
     silver:

nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql

@@ -1,7 +1,7 @@
 SELECT
-    code AS violation_code,
-    definition,
-    manhattan_96th_st_below,
-    all_other_areas
+  code AS violation_code,
+  definition,
+  manhattan_96th_st_below,
+  all_other_areas
 FROM
-    parking_violation_codes
+  parking_violation_codes

nyc_parking_violations/models/docs/docs_blocks.md → nyc_parking_violations/models/docs/docs_block.md

@@ -104,4 +104,4 @@ The total number of tickets issued for a specific violation code.
 
 {% docs total_revenue_usd %}
 The total revenue accumulated from tickets, based on the violation code. This sum is represented in USD.
-{% enddocs %}
+{% enddocs %}

nyc_parking_violations/models/example/ref_model.sql

@@ -1,4 +1,3 @@
-SELECT
-  COUNT(*)
-FROM
-  {{ref('first_model')}}
+SELECT COUNT(*)
+FROM 
+  {{ref('first_model')}}

nyc_parking_violations/profiles.yml

@@ -1,12 +1,9 @@
 nyc_parking_violations:
   outputs:
-   dev:
-     type: duckdb
-     path: '../data/nyc_parking_violations.db'
-   prod:
-     type: duckdb
-     # note that path is slightly different as GitHub actions
-     # start in the root directory and not in the
-     # nyc_parking_violations directory
-     path: './data/prod_nyc_parking_violations.db'  
-  target: dev
+    dev:
+      type: duckdb
+      path: '../data/nyc_parking_violations.db'
+    prod:
+      type: duckdb
+      path: './data/prod_nyc_parking_violations.db'
+  target: dev

nyc_parking_violations/tests/violation_codes_revenue.sql

@@ -8,4 +8,4 @@ FROM
 GROUP BY
     violation_code
 HAVING
-    NOT(total_revenue_usd >= 1)
+    NOT(total_revenue_usd >= 1)

run_sql_queries_here.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -12,17 +12,216 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>violation_code</th>\n",
+       "      <th>total_revenue_usd</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>41</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   violation_code  total_revenue_usd\n",
+       "0              41                0.0"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sql_query = '''\n",
+    "  select * from \"nyc_parking_violations\".\"main_dbt_test__audit\".\"violation_codes_revenue\"\n",
+    "'''\n",
+    "\n",
+    "with duckdb.connect('data/nyc_parking_violations.db') as con:\n",
+    "    display(con.sql(sql_query).df())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
+   "source": [
+    "sql_query_import_1 = '''\n",
+    "CREATE OR REPLACE TABLE parking_violation_codes AS \n",
+    "SELECT * \n",
+    "FROM read_csv_auto(\n",
+    "  'data/dof_parking_violation_codes.csv', normalize_names=True\n",
+    "  )\n",
+    "'''\n",
+    "\n",
+    "sql_query_import_2 = '''\n",
+    "CREATE OR REPLACE TABLE parking_violations_2023 AS\n",
+    "SELECT *\n",
+    "FROM read_csv_auto(\n",
+    "  'data/parking_violations_issued_fiscal_year_2023_sample.csv', normalize_names=True\n",
+    ")\n",
+    "'''\n",
+    "\n",
+    "with duckdb.connect('data/nyc_parking_violations.db') as con:\n",
+    "  con.sql(sql_query_import_1)\n",
+    "  con.sql(sql_query_import_2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>name</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>bronze_parking_violation_codes</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>bronze_parking_violations</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>first_model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>gold_ticket_metrics</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>gold_vehicles_metrics</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>parking_violation_codes</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>parking_violations_2023</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>ref_model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>silver_violation_tickets</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>silver_violation_vehicles</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                             name\n",
+       "0  bronze_parking_violation_codes\n",
+       "1       bronze_parking_violations\n",
+       "2                     first_model\n",
+       "3             gold_ticket_metrics\n",
+       "4           gold_vehicles_metrics\n",
+       "5         parking_violation_codes\n",
+       "6         parking_violations_2023\n",
+       "7                       ref_model\n",
+       "8        silver_violation_tickets\n",
+       "9       silver_violation_vehicles"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "sql_query = '''\n",
     "show tables\n",
     "'''\n",
     "\n",
-    "with duckdb.connect('data/nyc_parking_violations.db') as con:\n",
+    "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n",
     "    display(con.sql(sql_query).df())"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sql_query_import_1 = '''\n",
+    "CREATE OR REPLACE TABLE parking_violation_codes AS \n",
+    "SELECT * \n",
+    "FROM read_csv_auto(\n",
+    "  'data/dof_parking_violation_codes.csv', normalize_names=True\n",
+    "  )\n",
+    "'''\n",
+    "\n",
+    "sql_query_import_2 = '''\n",
+    "CREATE OR REPLACE TABLE parking_violations_2023 AS\n",
+    "SELECT *\n",
+    "FROM read_csv_auto(\n",
+    "  'data/parking_violations_issued_fiscal_year_2023_sample.csv', normalize_names=True\n",
+    ")\n",
+    "'''\n",
+    "\n",
+    "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n",
+    "  con.sql(sql_query_import_1)\n",
+    "  con.sql(sql_query_import_2)"
+   ]
   }
  ],
  "metadata": {