Merge pull request #2495 from TEAMSchools/2479-warehouse-the-u_stored…

…grades_de-on-u_storedgrades_extension-from-powerschool

overriding failed dbt run: kippcamden source won't be materialized until merged

src/dbt/kippmiami/dbt_project.yml

@@ -53,6 +53,8 @@ models:
         +enabled: false
       stg_powerschool__u_def_ext_students:
         +enabled: false
+      stg_powerschool__u_storedgrades_de:
+        +enabled: false
       stg_powerschool__studentrace:
         +enabled: false
       stg_powerschool__gpnode:
@@ -87,6 +89,8 @@ sources:
         +enabled: false
       src_powerschool__u_def_ext_students:
         +enabled: false
+      src_powerschool__u_storedgrades_de:
+        +enabled: false
       src_powerschool__studentrace:
         +enabled: false
       src_powerschool__gpnode:

src/dbt/kipptaf/models/powerschool/sources-kippcamden.yml

@@ -814,3 +814,11 @@ sources:
               - kippcamden
               - powerschool
               - stg_powerschool__userscorefields
+      - name: stg_powerschool__u_storedgrades_de
+        meta:
+          dagster:
+            group: powerschool
+            asset_key:
+              - kippcamden
+              - powerschool
+              - stg_powerschool__u_storedgrades_de

src/dbt/kipptaf/models/powerschool/sources-kippnewark.yml

@@ -814,3 +814,11 @@ sources:
               - kippnewark
               - powerschool
               - stg_powerschool__userscorefields
+      - name: stg_powerschool__u_storedgrades_de
+        meta:
+          dagster:
+            group: powerschool
+            asset_key:
+              - kippnewark
+              - powerschool
+              - stg_powerschool__u_storedgrades_de

src/dbt/kipptaf/models/powerschool/staging/stg_powerschool__u_storedgrades_de.sql

@@ -0,0 +1,8 @@
+{{
+    dbt_utils.union_relations(
+        relations=[
+            source("kippnewark_powerschool", "stg_powerschool__u_storedgrades_de"),
+            source("kippcamden_powerschool", "stg_powerschool__u_storedgrades_de"),
+        ]
+    )
+}}

src/dbt/powerschool/models/sources.yml

@@ -1249,3 +1249,20 @@ sources:
             max_staleness: INTERVAL 1 DAY
             format: AVRO
             enable_logical_types: true
+      - name: src_powerschool__u_storedgrades_de
+        meta:
+          dagster:
+            asset_key:
+              - "{{ project_name }}"
+              - powerschool
+              - u_storedgrades_de
+        external:
+          location:
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+            }}/powerschool/u_storedgrades_de/*"
+          options:
+            connection_name: "{{ var('bigquery_external_connection_name') }}"
+            metadata_cache_mode: AUTOMATIC
+            max_staleness: INTERVAL 1 DAY
+            format: AVRO
+            enable_logical_types: true

src/dbt/powerschool/models/staging/stg_powerschool__u_storedgrades_de.sql

@@ -0,0 +1,14 @@
+select
+    de_course_name,
+    de_semester,
+    whocreated,
+    whencreated,
+    whomodified,
+    whenmodified,
+    de_score,
+    de_institution,
+
+    id.int_value as id,
+    storedgradesdcid.int_value as storedgradesdcid,
+    de_pass_yn.int_value as de_pass_yn,
+from {{ source("powerschool", "src_powerschool__u_storedgrades_de") }}

src/teamster/code_locations/kippcamden/powerschool/config/assets-full.yaml

@@ -18,6 +18,8 @@ assets:
     partition_column: whenmodified
   - asset_name: emailaddress
     partition_column: whenmodified
+  - asset_name: gpstudentwaiver
+    partition_column: whenmodified
   - asset_name: gradescaleitem
     partition_column: whenmodified
   - asset_name: originalcontactmap
@@ -62,7 +64,7 @@ assets:
     partition_column: whenmodified
   - asset_name: u_clg_et_stu_alt
     partition_column: whenmodified
-  - asset_name: gpstudentwaiver
+  - asset_name: u_storedgrades_de
     partition_column: whenmodified
   - asset_name: userscorefields
     partition_column: whenmodified

src/teamster/code_locations/kippnewark/powerschool/config/assets-full.yaml

@@ -58,6 +58,8 @@ assets:
     partition_column: whenmodified
   - asset_name: userscorefields
     partition_column: whenmodified
+  - asset_name: u_storedgrades_de
+    partition_column: whenmodified
   - asset_name: u_studentsuserfields
     partition_column: whenmodified
     select_columns:

src/teamster/libraries/powerschool/sis/utils.py

@@ -7,6 +7,7 @@
 
 
 def open_ssh_tunnel(ssh_resource: SSHResource):
+    # trunk-ignore(bandit/B603)
     ssh_tunnel = subprocess.Popen(
         args=[
             "sshpass",
@@ -57,15 +58,18 @@ def get_query_text(
 ):
     # TODO: paramterize sqlalchemy query to resolve bandit/B608
     if column is None:
+        # trunk-ignore(bandit/B608)
         query = f"SELECT COUNT(*) FROM {table}"
     elif end_value is None:
         query = (
+            # trunk-ignore(bandit/B608)
             f"SELECT COUNT(*) FROM {table} "
             f"WHERE {column} >= "
             f"TO_TIMESTAMP('{start_value}', 'YYYY-MM-DD\"T\"HH24:MI:SS.FF6')"
         )
     else:
         query = (
+            # trunk-ignore(bandit/B608)
             f"SELECT COUNT(*) FROM {table} "
             f"WHERE {column} BETWEEN "
             f"TO_TIMESTAMP('{start_value}', 'YYYY-MM-DD\"T\"HH24:MI:SS.FF6') AND "

tests/assets/test_assets_powerschool_sis.py

@@ -9,7 +9,7 @@
 from tests.utils import get_db_powerschool_resource, get_ssh_powerschool_resource
 
 
-def _test_asset(assets, asset_name, ssh_powerschool, db_powerschool):
+def _test_asset(assets, asset_name: str, code_location: str):
     asset = [a for a in assets if a.key.path[-1] == asset_name][0]
 
     if asset.partitions_def is not None:
@@ -24,8 +24,8 @@ def _test_asset(assets, asset_name, ssh_powerschool, db_powerschool):
         partition_key=partition_key,
         resources={
             "io_manager_gcs_file": get_io_manager_gcs_file("test"),
-            "ssh_powerschool": ssh_powerschool,
-            "db_powerschool": db_powerschool,
+            "ssh_powerschool": get_ssh_powerschool_resource(code_location),
+            "db_powerschool": get_db_powerschool_resource(code_location),
         },
     )
 
@@ -49,10 +49,7 @@ def test_schools_kippnewark():
     from teamster.code_locations.kippnewark.powerschool import assets
 
     _test_asset(
-        assets=assets,
-        asset_name="schools",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
+        assets=assets, asset_name="schools", code_location=CODE_LOCATION.upper()
     )
 
 
@@ -61,23 +58,15 @@ def test_reenrollments_kippnewark():
     from teamster.code_locations.kippnewark.powerschool import assets
 
     _test_asset(
-        assets=assets,
-        asset_name="reenrollments",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
+        assets=assets, asset_name="reenrollments", code_location=CODE_LOCATION.upper()
     )
 
 
 def test_cc_kippnewark():
     from teamster.code_locations.kippnewark import CODE_LOCATION
     from teamster.code_locations.kippnewark.powerschool import assets
 
-    _test_asset(
-        assets=assets,
-        asset_name="cc",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
-    )
+    _test_asset(assets=assets, asset_name="cc", code_location=CODE_LOCATION.upper())
 
 
 def test_u_studentsuserfields_kippnewark():
@@ -87,8 +76,7 @@ def test_u_studentsuserfields_kippnewark():
     _test_asset(
         assets=assets,
         asset_name="u_studentsuserfields",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
+        code_location=CODE_LOCATION.upper(),
     )
 
 
@@ -99,8 +87,7 @@ def test_u_studentsuserfields_kippcamden():
     _test_asset(
         assets=assets,
         asset_name="u_studentsuserfields",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
+        code_location=CODE_LOCATION.upper(),
     )
 
 
@@ -111,57 +98,36 @@ def test_u_studentsuserfields_kippmiami():
     _test_asset(
         assets=assets,
         asset_name="u_studentsuserfields",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
+        code_location=CODE_LOCATION.upper(),
     )
 
 
 def test_log_kippmiami():
     from teamster.code_locations.kippmiami import CODE_LOCATION
     from teamster.code_locations.kippmiami.powerschool import assets
 
-    _test_asset(
-        assets=assets,
-        asset_name="log",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
-    )
+    _test_asset(assets=assets, asset_name="log", code_location=CODE_LOCATION.upper())
 
 
 def test_log_kippcamden():
     from teamster.code_locations.kippcamden import CODE_LOCATION
     from teamster.code_locations.kippcamden.powerschool import assets
 
-    _test_asset(
-        assets=assets,
-        asset_name="log",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
-    )
+    _test_asset(assets=assets, asset_name="log", code_location=CODE_LOCATION.upper())
 
 
 def test_log_kippnewark():
     from teamster.code_locations.kippnewark import CODE_LOCATION
     from teamster.code_locations.kippnewark.powerschool import assets
 
-    _test_asset(
-        assets=assets,
-        asset_name="log",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
-    )
+    _test_asset(assets=assets, asset_name="log", code_location=CODE_LOCATION.upper())
 
 
 def test_gpnode_kippnewark():
     from teamster.code_locations.kippnewark import CODE_LOCATION
     from teamster.code_locations.kippnewark.powerschool import assets
 
-    _test_asset(
-        assets=assets,
-        asset_name="gpnode",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
-    )
+    _test_asset(assets=assets, asset_name="gpnode", code_location=CODE_LOCATION.upper())
 
 
 def test_gpprogresssubject_kippnewark():
@@ -171,8 +137,7 @@ def test_gpprogresssubject_kippnewark():
     _test_asset(
         assets=assets,
         asset_name="gpprogresssubject",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
+        code_location=CODE_LOCATION.upper(),
     )
 
 
@@ -183,8 +148,7 @@ def test_gpprogresssubjectwaived_kippnewark():
     _test_asset(
         assets=assets,
         asset_name="gpprogresssubjectwaived",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
+        code_location=CODE_LOCATION.upper(),
     )
 
 
@@ -193,10 +157,7 @@ def test_gpstudentwaiver_kippnewark():  # whenmodified
     from teamster.code_locations.kippnewark.powerschool import assets
 
     _test_asset(
-        assets=assets,
-        asset_name="gpstudentwaiver",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
+        assets=assets, asset_name="gpstudentwaiver", code_location=CODE_LOCATION.upper()
     )
 
 
@@ -205,10 +166,7 @@ def test_gpversion_kippnewark():
     from teamster.code_locations.kippnewark.powerschool import assets
 
     _test_asset(
-        assets=assets,
-        asset_name="gpversion",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
+        assets=assets, asset_name="gpversion", code_location=CODE_LOCATION.upper()
     )
 
 
@@ -217,20 +175,36 @@ def test_gradplan_kippnewark():
     from teamster.code_locations.kippnewark.powerschool import assets
 
     _test_asset(
-        assets=assets,
-        asset_name="gradplan",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
+        assets=assets, asset_name="gradplan", code_location=CODE_LOCATION.upper()
     )
 
 
 def test_userscorefields_kippnewark():
     from teamster.code_locations.kippnewark import CODE_LOCATION
     from teamster.code_locations.kippnewark.powerschool import assets
 
+    _test_asset(
+        assets=assets, asset_name="userscorefields", code_location=CODE_LOCATION.upper()
+    )
+
+
+def test_u_storedgrades_de_kippnewark():
+    from teamster.code_locations.kippnewark import CODE_LOCATION
+    from teamster.code_locations.kippnewark.powerschool import assets
+
+    _test_asset(
+        assets=assets,
+        asset_name="u_storedgrades_de",
+        code_location=CODE_LOCATION.upper(),
+    )
+
+
+def test_u_storedgrades_de_kippcamden():
+    from teamster.code_locations.kippcamden import CODE_LOCATION
+    from teamster.code_locations.kippcamden.powerschool import assets
+
     _test_asset(
         assets=assets,
-        asset_name="userscorefields",
-        ssh_powerschool=get_ssh_powerschool_resource(CODE_LOCATION.upper()),
-        db_powerschool=get_db_powerschool_resource(CODE_LOCATION.upper()),
+        asset_name="u_storedgrades_de",
+        code_location=CODE_LOCATION.upper(),
     )

tests/resources/test_resource_powerschool_sis.py

@@ -11,7 +11,7 @@ def test():
         resources={
             "ssh_powerschool": SSHResource(
                 remote_host=EnvVar("PS_SSH_HOST"),
-                remote_port=EnvVar("PS_SSH_PORT"),
+                remote_port=int(EnvVar("PS_SSH_PORT")),
                 username=EnvVar("PS_SSH_USERNAME"),
                 tunnel_remote_host=EnvVar("PS_SSH_REMOTE_BIND_HOST"),
             ),