From 4a914c064be922c06b36ea024828228e331ac701 Mon Sep 17 00:00:00 2001 From: Charles Costanzo Date: Tue, 12 Nov 2024 12:04:16 -0500 Subject: [PATCH] readd templating for buckets and remove hardcode schema --- airflow/plugins/operators/scrape_state_geoportal.py | 2 -- .../stg_state_geoportal__state_highway_network_stops.sql | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/airflow/plugins/operators/scrape_state_geoportal.py b/airflow/plugins/operators/scrape_state_geoportal.py index c4afc0c37a..ebd86f1866 100644 --- a/airflow/plugins/operators/scrape_state_geoportal.py +++ b/airflow/plugins/operators/scrape_state_geoportal.py @@ -13,8 +13,6 @@ # from pydantic import HttpUrl, parse_obj_as - -# API_BUCKET = "gs://calitp-state-highway-network-stops" API_BUCKET = "gs://calitp-state-geoportal-scrape" # API_BUCKET = os.environ["CALITP_BUCKET__STATE_GEOPORTAL_DATA_PRODUCTS"] diff --git a/warehouse/models/staging/state_geoportal/stg_state_geoportal__state_highway_network_stops.sql b/warehouse/models/staging/state_geoportal/stg_state_geoportal__state_highway_network_stops.sql index ff67f185df..6fd2950a37 100644 --- a/warehouse/models/staging/state_geoportal/stg_state_geoportal__state_highway_network_stops.sql +++ b/warehouse/models/staging/state_geoportal/stg_state_geoportal__state_highway_network_stops.sql @@ -1,8 +1,8 @@ WITH external_state_geoportal__state_highway_network AS ( SELECT * FROM - `cal-itp-data-infra-staging.external_state_geoportal.state_highway_network` - --{{ source('external_state_geoportal', 'state_highway_network') }} + --`cal-itp-data-infra-staging.external_state_geoportal.state_highway_network` + {{ source('external_state_geoportal', 'state_highway_network') }} ), stg_state_geoportal__state_highway_network_stops AS(