SQL templating fixes

The-Academic-Observatory · Dec 21, 2023 · 0f51163 · 0f51163
1 parent 547bd27
commit 0f51163
Show file tree

Hide file tree

Showing 12 changed files with 44 additions and 105 deletions.
diff --git a/oaebu_workflows/google_analytics3_telescope/sql/export_author_struct_google_analytics3.sql b/oaebu_workflows/google_analytics3_telescope/sql/export_author_struct_google_analytics3.sql
diff --git a/oaebu_workflows/irus_oapen_telescope/sql/export_city_irus_oapen.sql b/oaebu_workflows/irus_oapen_telescope/sql/export_city_irus_oapen.sql
@@ -0,0 +1,7 @@
+STRUCT(
+    SUM(city.title_requests) as title_requests,
+    SUM(city.total_item_investigations) as total_item_investigations,
+    SUM(city.total_item_requests) as total_item_requests,
+    SUM(city.unique_item_investigations) as unique_item_investigations,
+    SUM(city.unique_item_requests) as unique_item_requests
+) as irus_oapen
diff --git a/oaebu_workflows/jstor_telescope/sql/export_subject_metrics_jstor_country.sql b/oaebu_workflows/jstor_telescope/sql/export_subject_metrics_jstor_country.sql
@@ -0,0 +1,3 @@
+STRUCT(
+    group_counts(ARRAY_CONCAT_AGG(month.jstor_country)) as Total_Item_Requests
+) as jstor
diff --git a/oaebu_workflows/onix_workflow/sql/create_book_products.sql.jinja2 b/oaebu_workflows/onix_workflow/sql/create_book_products.sql.jinja2
@@ -16,8 +16,8 @@
 
 # Add data partner temp functions
 {% for dp in data_partners %}
-    {% if dp.bp_temp_functions %}
-        {% include dp.bp_temp_functions %}
+    {% if dp.book_product_functions %}
+        {% include dp.files.book_product_functions %}
     {% endif %}
 {% endfor %}
 
@@ -36,9 +36,7 @@ CREATE TEMP FUNCTION custom_split(str STRING, sep STRING) AS (
 
 # Add data partner main body blocks
 {% for dp in data_partners %}
-    {% if dp.bp_temp_functions %}
-        {% include dp.bp_body %}
-    {% endif %}
+    {% include dp.files.book_product_body %}
 {% endfor %}
 
 {#

diff --git a/oaebu_workflows/onix_workflow/sql/export_book_author_metrics.sql.jinja2 b/oaebu_workflows/onix_workflow/sql/export_book_author_metrics.sql.jinja2
@@ -30,18 +30,14 @@ SELECT
     MAX(author.ORCID) as orcid,
     count(ISBN13) as unique_books,
     month.month,
-    {% for dp in data_partners %}
-        {% if dp.export_author %}
-            {% include dp.files.month_metrics_sum %},
-        {% endif %}
+    {% for dp in data_partners | selectattr("export_author", "equalto", True)%}
+        {% include dp.files.month_metrics_sum %},
     {% endfor %}
     STRUCT(group_counts(ARRAY_CONCAT_AGG(month.crossref_events)) as count) as crossref_events
 FROM `{{ book_product_table_id }}`, UNNEST(onix.authors) as author, UNNEST(months) as month
 WHERE 
-    {% for dp in data_partners %}
-        {% if dp.export_author %}
-            {% include dp.files.month_mull %} OR
-        {% endif %}
+    {% for dp in data_partners | selectattr("export_author", "equalto", True) %}
+        {% include dp.files.month_mull %} OR
     {% endfor %}
 ARRAY_LENGTH(month.crossref_events) > 0
 GROUP BY author.PersonName, month

diff --git a/oaebu_workflows/onix_workflow/sql/export_book_metrics.sql.jinja2 b/oaebu_workflows/onix_workflow/sql/export_book_metrics.sql.jinja2
@@ -26,17 +26,13 @@ SELECT
     onix.authors,
     CAST(onix.published_year as INT64) as published_year,
     month.month,
-    {% for dp in data_partners %}
-        {% if dp.export_book_metrics %}
-            {% include dp.files.export_book_metrics %}
-        {% endif %}
+    {% for dp in data_partners | selectattr("export_book_metrics", "equalto", True) %}
+        {% include dp.files.export_book_metrics %}
     {% endfor %}    
     STRUCT((SELECT SUM(count) FROM UNNEST(month.crossref_events)) as count) as crossref_events
 FROM `{{ book_product_table_id }}`, UNNEST(months) as month
 WHERE 
-{% for dp in data_partners %}
-    {% if dp.month_null%}
-        {% include dp.files.month_null %} OR
-    {% endif %}
+{% for dp in data_partners | selectattr("export_book_metrics", "equalto", True) %}
+    {% include dp.files.month_null %} OR
 {% endfor %}
 ARRAY_LENGTH(month.crossref_events) > 0
diff --git a/oaebu_workflows/onix_workflow/sql/export_book_metrics_city.sql.jinja2 b/oaebu_workflows/onix_workflow/sql/export_book_metrics_city.sql.jinja2
@@ -27,22 +27,17 @@ SELECT
     month.month,
     city.city,
     CONCAT(CAST(MAX(city.latitude) as STRING), ", ", CAST(MAX(city.longitude) as STRING)) as coordinates,
-
     {# Currently only used by IRUS OAPEN #}
-    {%if irus_oapen in (dp.type_id for dp in data_partners) %} 
-        STRUCT(
-            SUM(city.title_requests) as title_requests,
-            SUM(city.total_item_investigations) as total_item_investigations,
-            SUM(city.total_item_requests) as total_item_requests,
-            SUM(city.unique_item_investigations) as unique_item_investigations,
-            SUM(city.unique_item_requests) as unique_item_requests
-        ) as irus_oapen
-    {% endif %}
+    STRUCT(
+        SUM(city.title_requests) as title_requests,
+        SUM(city.total_item_investigations) as total_item_investigations,
+        SUM(city.total_item_requests) as total_item_requests,
+        SUM(city.unique_item_investigations) as unique_item_investigations,
+        SUM(city.unique_item_requests) as unique_item_requests
+    ) as irus_oapen
 
 
 FROM `{{ book_product_table_id }}`, UNNEST(months) as month, 
-    {%if irus_oapen in (dp.type_id for dp in data_partners) %} 
         UNNEST(month.irus_oapen.locations) as city 
-    {% endif %}
 WHERE city.city IS NOT NULL
 GROUP BY ISBN13, month, city
diff --git a/oaebu_workflows/onix_workflow/sql/export_book_metrics_country.sql.jinja2 b/oaebu_workflows/onix_workflow/sql/export_book_metrics_country.sql.jinja2
@@ -42,10 +42,8 @@ month_country as (
     CROSS JOIN countries
 ),
 
-{% for dp in data_partners %}
-    {% if dp.export_country_body %}
-        {% include dp.files.export_country_body %}{% if not loop.last %}, {% endif %}
-    {% endif %}
+{% for dp in data_partners | selectattr("export_country", "equalto", True) %}
+    {% include dp.files.export_country_body %}{% if not loop.last %}, {% endif %}
 {% endfor %}
 
 
@@ -60,20 +58,14 @@ SELECT
     month_country.country_name,
     month_country.country_iso_name,
     month_country.country_wikipedia_name,
-    {% for dp in data_partners %}
-        {% if dp.export_country_struct %}
-            {% include dp.files.export_country_struct %}{% if not loop.last %}, {% endif %}
-        {% endif %}
+    {% for dp in data_partners | selectattr("export_country_struct", "equalto", True) %}
+        {% include dp.files.export_country_struct %}{% if not loop.last %}, {% endif %}
     {% endfor %}
 FROM month_country
-    {% for dp in data_partners %}
-        {% if dp.export_country_join %}
-            {% include dp.files.export_country_join %}
-        {% endif %}
+    {% for dp in data_partners | selectattr("export_country_join", "equalto", True) %}
+        {% include dp.files.export_country_join %}
     {% endfor %}
 WHERE 
-    {% for dp in data_partners %}
-        {% if dp.export_country_null %}
-            {% include dp.files.export_country_null %}{% if not loop.last %}OR {% endif %}
-        {% endif %}
+    {% for dp in data_partners | selectattr("export_country_null", "equalto", True) %}
+        {% include dp.files.export_country_null %}{% if not loop.last %}OR {% endif %}
     {% endfor %}
diff --git a/oaebu_workflows/onix_workflow/sql/export_book_metrics_institution.sql.jinja2 b/oaebu_workflows/onix_workflow/sql/export_book_metrics_institution.sql.jinja2
@@ -24,6 +24,7 @@ SELECT
     CAST(onix.published_year as INT64) as published_year,
     month.month,
     institution.institution,
+    {# Currently only used by jstor #}
     STRUCT(
         institution.Total_Item_Requests
     ) as jstor,

diff --git a/oaebu_workflows/onix_workflow/sql/export_book_subject_bic_metrics.sql.jinja2 b/oaebu_workflows/onix_workflow/sql/export_book_subject_bic_metrics.sql.jinja2
@@ -42,20 +42,17 @@ FROM(
         count(ISBN13) as unique_books,
         month.month,
 
-        {% for dp in data_partners %}
-          {% if dp.export_subject %}
+        {% for dp in data_partners | selectattr("export_subject", "equalto", True) %}
             {% include dp.files.month_metrics_sum %},
-          {% endif %}
         {% endfor %}
+
         STRUCT(group_counts(ARRAY_CONCAT_AGG(month.crossref_events)) as count) as crossref_events
 
     FROM `{{ book_product_table_id }}`, UNNEST(top_level_subjects(onix.bic_subjects)) as subject, UNNEST(months) as month
     WHERE 
 
-    {% for dp in data_partners %}
-      {% if dp.export_subject %}
+    {% for dp in data_partners | selectattr("export_subject", "equalto", True) %}
         {% include dp.files.month_null %} OR 
-      {% endif %}
     {% endfor %}
 
     ARRAY_LENGTH(month.crossref_events) > 0 

diff --git a/oaebu_workflows/onix_workflow/sql/export_book_subject_bisac_metrics.sql.jinja2 b/oaebu_workflows/onix_workflow/sql/export_book_subject_bisac_metrics.sql.jinja2
@@ -42,21 +42,17 @@ FROM(
         count(ISBN13) as unique_books,
         month.month,
 
-        {% for dp in data_partners %}
-          {% if dp.export_subject %}
+        {% for dp in data_partners | selectattr("export_subject", "equalto", True) %}
             {% include dp.month_metrics_sum %},
-          {% endif %}
         {% endfor %}
 
         STRUCT(group_counts(ARRAY_CONCAT_AGG(month.crossref_events)) as count) as crossref_events,
 
     FROM `{{ book_product_table_id }}`, UNNEST(top_level_subjects(onix.bisac_subjects)) as subject, UNNEST(months) as month
     WHERE 
 
-    {% for dp in data_partners %}
-      {% if dp.export_subject %}
+    {% for dp in data_partners | selectattr("export_subject", "equalto", True) %}
         {% include dp.month_null %} OR 
-      {% endif %}
     {% endfor %}
 
     ARRAY_LENGTH(month.crossref_events) > 0

diff --git a/oaebu_workflows/onix_workflow/sql/export_book_subject_thema_metrics.sql.jinja2 b/oaebu_workflows/onix_workflow/sql/export_book_subject_thema_metrics.sql.jinja2
@@ -42,21 +42,17 @@ FROM(
         count(ISBN13) as unique_books,
         month.month,
 
-        {% for dp in data_partners %}
-          {% if dp.export_subject %}
+        {% for dp in data_partners | selectattr("export_subject", "equalto", True) %}
             {% include dp.month_metrics_sum %},
-          {% endif %}
         {% endfor %}
 
         STRUCT(group_counts(ARRAY_CONCAT_AGG(month.crossref_events)) as count) as crossref_events
 
     FROM `{{ book_product_table_id }}`, UNNEST(top_level_subjects(onix.thema_subjects)) as subject, UNNEST(months) as month
     WHERE 
 
-    {% for dp in data_partners %}
-      {% if dp.export_subject %}
+    {% for dp in data_partners | selectattr("export_subject", "equalto", True) %}
         {% include dp.month_null %} OR 
-      {% endif %}
     {% endfor %}
 
     ARRAY_LENGTH(month.crossref_events) > 0