Skip to content

Commit 1101210

Browse files
charlie-costanzocharlie-costanzo
authored
NTD: create external tables for new endpoints – asset_inventory_time_series and 2022 contractual relationships (#3857)
* feat: add external table for 2022 contractual relationships ntd table * fix: typo in 2 create external tables dags
1 parent 3298a03 commit 1101210

6 files changed

+96
-0
lines changed

‎airflow/dags/create_external_tables/ntd_data_products/2022__annual_database_contractual_relationships.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
operator: operators.ExternalTable
2+
bucket: gs://calitp-ntd-xlsx-products-clean
3+
prefix_bucket: false
4+
post_hook: |
5+
SELECT *
6+
FROM `{{ get_project_id() }}`.external_ntd__annual_reporting.2022__annual_database_contractual_relationships
7+
LIMIT 1;
8+
source_objects:
9+
- "annual_database_contractual_relationship/2022/_2022_contractual_relationships/*.jsonl.gz"
10+
destination_project_dataset_table: "external_ntd__annual_reporting.2022__annual_database_contractual_relationships"
11+
source_format: NEWLINE_DELIMITED_JSON
12+
use_bq_client: true
13+
hive_options:
14+
mode: AUTO
15+
require_partition_filter: false
16+
source_uri_prefix: "annual_database_contractual_relationship/2022/_2022_contractual_relationships/"

‎airflow/dags/create_external_tables/ntd_data_products/historical__asset_inventory_time_series__active_fleet.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
operator: operators.ExternalTable
2+
bucket: gs://calitp-ntd-xlsx-products-clean
3+
prefix_bucket: false
4+
post_hook: |
5+
SELECT *
6+
FROM `{{ get_project_id() }}`.external_ntd__assets.historical__asset_inventory_time_series__active_fleet
7+
LIMIT 1;
8+
source_objects:
9+
- "asset_inventory_time_series/historical/active_fleet/*.jsonl.gz"
10+
destination_project_dataset_table: "external_ntd__assets.historical__asset_inventory_time_series__active_fleet"
11+
source_format: NEWLINE_DELIMITED_JSON
12+
use_bq_client: true
13+
hive_options:
14+
mode: AUTO
15+
require_partition_filter: false
16+
source_uri_prefix: "asset_inventory_time_series/historical/active_fleet/"

‎airflow/dags/create_external_tables/ntd_data_products/historical__asset_inventory_time_series__ada_fleet.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
operator: operators.ExternalTable
2+
bucket: gs://calitp-ntd-xlsx-products-clean
3+
prefix_bucket: false
4+
post_hook: |
5+
SELECT *
6+
FROM `{{ get_project_id() }}`.external_ntd__assets.historical__asset_inventory_time_series__ada_fleet
7+
LIMIT 1;
8+
source_objects:
9+
- "asset_inventory_time_series/historical/ada_fleet/*.jsonl.gz"
10+
destination_project_dataset_table: "external_ntd__assets.historical__asset_inventory_time_series__ada_fleet"
11+
source_format: NEWLINE_DELIMITED_JSON
12+
use_bq_client: true
13+
hive_options:
14+
mode: AUTO
15+
require_partition_filter: false
16+
source_uri_prefix: "asset_inventory_time_series/historical/ada_fleet/"

‎airflow/dags/create_external_tables/ntd_data_products/historical__asset_inventory_time_series__avg_fleet_age.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
operator: operators.ExternalTable
2+
bucket: gs://calitp-ntd-xlsx-products-clean
3+
prefix_bucket: false
4+
post_hook: |
5+
SELECT *
6+
FROM `{{ get_project_id() }}`.external_ntd__assets.historical__asset_inventory_time_series__avg_fleet_age
7+
LIMIT 1;
8+
source_objects:
9+
- "asset_inventory_time_series/historical/avg_fleet_age/*.jsonl.gz"
10+
destination_project_dataset_table: "external_ntd__assets.historical__asset_inventory_time_series__avg_fleet_age"
11+
source_format: NEWLINE_DELIMITED_JSON
12+
use_bq_client: true
13+
hive_options:
14+
mode: AUTO
15+
require_partition_filter: false
16+
source_uri_prefix: "asset_inventory_time_series/historical/avg_fleet_age/"

‎airflow/dags/create_external_tables/ntd_data_products/historical__asset_inventory_time_series__avg_seating_capacity.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
operator: operators.ExternalTable
2+
bucket: gs://calitp-ntd-xlsx-products-clean
3+
prefix_bucket: false
4+
post_hook: |
5+
SELECT *
6+
FROM `{{ get_project_id() }}`.external_ntd__assets.historical__asset_inventory_time_series__avg_seating_capacity
7+
LIMIT 1;
8+
source_objects:
9+
- "asset_inventory_time_series/historical/avg_seating_capacity/*.jsonl.gz"
10+
destination_project_dataset_table: "external_ntd__assets.historical__asset_inventory_time_series__avg_seating_capacity"
11+
source_format: NEWLINE_DELIMITED_JSON
12+
use_bq_client: true
13+
hive_options:
14+
mode: AUTO
15+
require_partition_filter: false
16+
source_uri_prefix: "asset_inventory_time_series/historical/avg_seating_capacity/"

‎airflow/dags/create_external_tables/ntd_data_products/historical__asset_inventory_time_series__avg_standing_capacity.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
operator: operators.ExternalTable
2+
bucket: gs://calitp-ntd-xlsx-products-clean
3+
prefix_bucket: false
4+
post_hook: |
5+
SELECT *
6+
FROM `{{ get_project_id() }}`.external_ntd__assets.historical__asset_inventory_time_series__avg_standing_capacity
7+
LIMIT 1;
8+
source_objects:
9+
- "asset_inventory_time_series/historical/avg_standing_capacity/*.jsonl.gz"
10+
destination_project_dataset_table: "external_ntd__assets.historical__asset_inventory_time_series__avg_standing_capacity"
11+
source_format: NEWLINE_DELIMITED_JSON
12+
use_bq_client: true
13+
hive_options:
14+
mode: AUTO
15+
require_partition_filter: false
16+
source_uri_prefix: "asset_inventory_time_series/historical/avg_standing_capacity/"

0 commit comments

Comments
 (0)