add flag for shn to dim_stops_latest

warehouse/models/mart/gtfs_schedule_latest/dim_stops_latest.sql

@@ -4,6 +4,47 @@ dim_stops_latest AS (
     table_name = ref('dim_stops'),
     clean_table_name = 'dim_stops'
     ) }}
+),
+
+stg_state_geoportal__state_highway_network_stops AS (
+SELECT * FROM {{ ref('stg_state_geoportal__state_highway_network_stops') }}
+),
+
+
+buffer_geometry_table AS (
+    SELECT
+        ST_BUFFER(wkt_coordinates,
+            30.48) AS buffer_geometry
+    FROM stg_state_geoportal__state_highway_network_stops
+),
+
+current_stops AS (
+    SELECT
+        pt_geom,
+        key
+    FROM dim_stops_latest
+),
+
+
+stops_on_shn AS (
+    SELECT
+        current_stops.*
+    FROM buffer_geometry_table, current_stops
+    WHERE ST_DWITHIN(
+            buffer_geometry_table.buffer_geometry,current_stops.pt_geom, 0)
+),
+
+dim_stops_latest_with_shn_boolean AS (
+
+SELECT
+    dim_stops_latest.*,
+    IF(stops_on_shn.key IS NOT NULL, TRUE, FALSE) AS exists_in_dim_stops_latest
+FROM
+    dim_stops_latest
+LEFT JOIN
+    stops_on_shn
+ON
+    dim_stops_latest.key = stops_on_shn.key
 )
 
-SELECT * FROM dim_stops_latest
+SELECT * FROM dim_stops_latest_with_shn_boolean

warehouse/models/staging/state_geoportal/stg_state_geoportal__state_highway_network_stops.sql

@@ -1,43 +1,16 @@
 WITH external_state_geoportal__state_highway_network AS (
     SELECT *
     FROM
-    {{ source('external_state_geoportal', 'state_highway_network') }}
+    `cal-itp-data-infra-staging.external_state_geoportal.state_highway_network`
+    --{{ source('external_state_geoportal', 'state_highway_network') }}
 ),
 
-get_latest_extract AS(
+stg_state_geoportal__state_highway_network_stops AS(
 
     SELECT *
     FROM external_state_geoportal__state_highway_network
     -- we pull the whole table every month in the pipeline, so this gets only the latest extract
     QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
-),
-
-buffer_geometry_table AS (
-    SELECT
-        ST_BUFFER(wkt_coordinates,
-            30.48) AS buffer_geometry
-    FROM get_latest_extract
-),
-
-current_stops AS (
-    SELECT
-        pt_geom,
-        key
-    FROM {{ ref('dim_stops_latest') }}
-),
-
-
-stops_on_shn AS (
-    SELECT
-        current_stops.*
-    FROM buffer_geometry_table, current_stops
-    WHERE ST_DWITHIN(
-            buffer_geometry_table.buffer_geometry,current_stops.pt_geom, 0)
-),
-
-stg_state_geoportal__state_highway_network_stops AS (
-    SELECT *
-    FROM stops_on_shn
 )
 
 SELECT * FROM stg_state_geoportal__state_highway_network_stops