Merge pull request #234 from SoryRawyer/rds/use-dict-dims

feat: use new dictionary-backed lookup tables to provide entity names (FC-0024)
openedx · Aug 1, 2023 · e85661f · e85661f
2 parents 518e21b + ec89692
commit e85661f
Show file tree

Hide file tree

Showing 10 changed files with 104 additions and 128 deletions.
diff --git a/tutoraspects/templates/openedx-assets/dim_courses.yaml b/tutoraspects/templates/openedx-assets/dim_courses.yaml
@@ -1,66 +1,54 @@
 - _file_name: dim_courses.yaml
   cache_timeout: null
   columns:
-  - advanced_data_type: null
-    column_name: course_key
-    description: null
-    expression: null
-    extra: null
-    filterable: true
-    groupby: true
-    is_active: true
-    is_dttm: false
-    python_date_format: null
-    type: String
-    verbose_name: null
-  - advanced_data_type: null
-    column_name: org
-    description: null
-    expression: null
-    extra: null
-    filterable: true
-    groupby: true
-    is_active: true
-    is_dttm: false
-    python_date_format: null
-    type: String
-    verbose_name: null
-  - advanced_data_type: null
-    column_name: course_name
-    description: null
-    expression: null
-    extra: null
-    filterable: true
-    groupby: true
-    is_active: true
-    is_dttm: false
-    python_date_format: null
-    type: String
-    verbose_name: null
-  - advanced_data_type: null
-    column_name: run_name
-    description: null
-    expression: null
-    extra: null
-    filterable: true
-    groupby: true
-    is_active: true
-    is_dttm: false
-    python_date_format: null
-    type: String
-    verbose_name: null
-  - advanced_data_type: null
-    column_name: run_id
-    description: null
-    expression: null
-    extra: null
-    filterable: true
-    groupby: true
-    is_active: true
-    is_dttm: false
-    python_date_format: null
-    type: String
-    verbose_name: null
+    - advanced_data_type: null
+      column_name: course_key
+      description: null
+      expression: null
+      extra: null
+      filterable: true
+      groupby: true
+      is_active: true
+      is_dttm: false
+      python_date_format: null
+      type: String
+      verbose_name: null
+    - advanced_data_type: null
+      column_name: org
+      description: null
+      expression: null
+      extra: null
+      filterable: true
+      groupby: true
+      is_active: true
+      is_dttm: false
+      python_date_format: null
+      type: String
+      verbose_name: null
+    - advanced_data_type: null
+      column_name: course_name
+      description: null
+      expression: null
+      extra: null
+      filterable: true
+      groupby: true
+      is_active: true
+      is_dttm: false
+      python_date_format: null
+      type: String
+      verbose_name: null
+    - advanced_data_type: null
+      column_name: run_name
+      description: null
+      expression: null
+      extra: null
+      filterable: true
+      groupby: true
+      is_active: true
+      is_dttm: false
+      python_date_format: null
+      type: String
+      verbose_name: null
   database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6
   default_endpoint: null
   description: null
@@ -69,14 +57,14 @@
   filter_select_enabled: false
   main_dttm_col: null
   metrics:
-  - d3format: null
-    description: null
-    expression: count(*)
-    extra: null
-    metric_name: count
-    metric_type: null
-    verbose_name: null
-    warning_text: null
+    - d3format: null
+      description: null
+      expression: count(*)
+      extra: null
+      metric_name: count
+      metric_type: null
+      verbose_name: null
+      warning_text: null
   offset: 0
   params: null
   schema: null

diff --git a/tutoraspects/templates/openedx-assets/queries/dim_course_problems.sql b/tutoraspects/templates/openedx-assets/queries/dim_course_problems.sql
@@ -9,7 +9,7 @@ with courses as (
     from
         {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_NODES_TABLE }}
     where
-        JSON_VALUE(xblock_data_json, '$.block_type') = 'problem'
+        location like '%problem+block%'
     {% raw -%}
         {% if filter_values('org') != [] %}
         and org in ({{ filter_values('org') | where_in }})

diff --git a/tutoraspects/templates/openedx-assets/queries/dim_course_videos.sql b/tutoraspects/templates/openedx-assets/queries/dim_course_videos.sql
@@ -10,7 +10,7 @@ with courses as (
         {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_NODES_TABLE }}
     where
     {% raw -%}
-        JSON_VALUE(xblock_data_json, '$.block_type') = 'video'
+        location like '%video+block%'
         {% if filter_values('org') != [] %}
         and org in ({{ filter_values('org') | where_in }})
         {% endif %}

diff --git a/tutoraspects/templates/openedx-assets/queries/dim_courses.sql b/tutoraspects/templates/openedx-assets/queries/dim_courses.sql
@@ -2,12 +2,11 @@ select distinct
     org,
     course_key,
     display_name as course_name,
-    location as run_id,
-    JSON_VALUE(xblock_data_json, '$.run') as run_name
+    splitByString('+', course_key)[-1] as run_name
 from
     {{ ASPECTS_EVENT_SINK_DATABASE }}.{{ ASPECTS_EVENT_SINK_NODES_TABLE }}
 where
-    JSON_VALUE(xblock_data_json, '$.block_type') = 'course'
+    location like '%course+block%'
 {% raw -%}
     {% if filter_values('org') != [] %}
     and org in {{ filter_values('org') | where_in }}

diff --git a/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql b/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql
@@ -1,6 +1,4 @@
-with courses as (
-    {% include 'openedx-assets/queries/dim_courses.sql' %}
-), enrollments as (
+with enrollments as (
     select
         emission_time,
         org,
@@ -22,12 +20,11 @@ select
     enrollments.emission_time,
     enrollments.org,
     courses.course_name,
-    courses.run_name,
+    splitByString('+', courses.course_key)[-1] as run_name,
     enrollments.actor_id,
     enrollments.enrollment_mode,
     enrollments.enrollment_status
 from
     enrollments
-    join courses
-        on (enrollments.org = courses.org
-            and enrollments.course_key = courses.course_key)
+    join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names courses
+        on enrollments.course_key = courses.course_key
diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql
@@ -21,17 +21,17 @@ with course_problems as (
 
 select
     summary.org as org,
-    course_problems.course_name as course_name,
-    course_problems.run_name as run_name,
-    course_problems.problem_name as problem_name,
+    courses.course_name as course_name,
+    splitByString('+', courses.course_key)[-1] as run_name,
+    blocks.block_name as problem_name,
     summary.actor_id as actor_id,
     summary.success as success,
     summary.attempts as attempts,
     summary.num_hints_displayed as num_hints_displayed,
     summary.num_answers_displayed as num_answers_displayed
 from
     summary
-    join course_problems
-        on (summary.org = course_problems.org
-            and summary.course_key = course_problems.course_key
-            and summary.problem_id = course_problems.problem_id)
+    join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names courses
+         on summary.course_key = courses.course_key
+    join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names blocks
+         on summary.problem_id = blocks.location
diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql
@@ -1,6 +1,4 @@
-with course_problems as (
-     {% include 'openedx-assets/queries/dim_course_problems.sql' %}
-), responses as (
+with responses as (
     select
         emission_time,
         org,
@@ -22,17 +20,17 @@ with course_problems as (
 
 select
     responses.emission_time as emission_time,
-    course_problems.org as org,
-    course_problems.course_name as course_name,
-    course_problems.run_name as run_name,
-    course_problems.problem_name as problem_name,
+    responses.org as org,
+    courses.course_name as course_name,
+    splitByString('+', courses.course_key)[-1] as run_name,
+    blocks.block_name as problem_name,
     responses.actor_id as actor_id,
     responses.responses as responses,
     responses.success as success,
     responses.attempts as attempts
 from
     responses
-    join course_problems
-        on (responses.org = course_problems.org
-            and responses.course_key = course_problems.course_key
-            and responses.problem_id = course_problems.problem_id)
+    join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names courses
+         on responses.course_key = courses.course_key
+    join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names blocks
+         on responses.problem_id = blocks.location
diff --git a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql
@@ -1,6 +1,4 @@
-with videos as (
-    {% include 'openedx-assets/queries/dim_course_videos.sql' %}
-), transcripts as (
+with transcripts as (
     select
         emission_time,
         org,
@@ -20,13 +18,13 @@ with videos as (
 select
     transcripts.emission_time as emission_time,
     transcripts.org as org,
-    videos.course_name as course_name,
-    videos.run_name as run_name,
-    videos.video_name as video_name,
+    courses.course_name as course_name,
+    splitByString('+', courses.course_key)[-1] as run_name,
+    blocks.block_name as video_name,
     transcripts.actor_id as actor_id
 from
     transcripts
-    join videos
-        on (transcripts.org = videos.org
-            and transcripts.course_key = videos.course_key
-            and transcripts.video_id = videos.video_id)
+    join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names courses
+         on transcripts.course_key = courses.course_key
+    join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names blocks
+         on transcripts.video_id = blocks.location
diff --git a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql
@@ -1,6 +1,4 @@
-with videos as (
-    {% include 'openedx-assets/queries/dim_course_videos.sql' %}
-), plays as (
+with plays as (
     select
         emission_time,
         org,
@@ -14,13 +12,13 @@ with videos as (
 select
     plays.emission_time as emission_time,
     plays.org as org,
-    videos.course_name as course_name,
-    videos.run_name as run_name,
-    videos.video_name as video_name,
+    courses.course_name as course_name,
+    splitByString('+', courses.course_key)[-1] as run_name,
+    blocks.block_name as video_name,
     plays.actor_id as actor_id
 from
     plays
-    join videos
-        on (plays.org = videos.org
-            and plays.course_key = videos.course_key
-            and plays.video_id = videos.video_id)
+    join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names courses
+         on plays.course_key = courses.course_key
+    join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names blocks
+         on plays.video_id = blocks.location
diff --git a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql
@@ -1,6 +1,4 @@
-with courses as (
-    {% include 'openedx-assets/queries/dim_course_videos.sql' %}
-), starts as (
+with starts as (
     select
         emission_time,
         org,
@@ -60,20 +58,20 @@ with courses as (
                 and starts.emission_time < ends.emission_time)
 ), enriched_segments as (
     select
-        courses.org as org,
+        segments.org as org,
         courses.course_name as course_name,
-        courses.run_name as run_name,
-        courses.video_name as video_name,
+        splitByString('+', courses.course_key)[-1] as run_name,
+        blocks.block_name as video_name,
         segments.actor_id as actor_id,
         segments.started_at as started_at,
         segments.start_position - (segments.start_position % 5) as start_position,
         segments.end_position - (segments.end_position % 5) as end_position
     from
         segments
-        join courses
-            on (segments.org = courses.org
-                and segments.course_key = courses.course_key
-                and segments.video_id = courses.video_id)
+        join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names courses
+            on segments.course_key = courses.course_key
+        join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names blocks
+            on segments.video_id = blocks.location
 )
 
 select