Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Tidy up execution timings logic #389

Merged
merged 4 commits into from
Oct 5, 2023
Merged

Tidy up execution timings logic #389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
de2ad66
Minor tidy up
Sep 29, 2023
97d9da8
Simplifying timing extract
Sep 29, 2023
0cabd84
Merge branch 'main' of github.com:brooklyn-data/dbt_artifacts into fi…
Sep 29, 2023
6c74274
Merge branch 'main' into fix/tidy-up-execution-timings
glsdown Oct 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions integration_test_project/example-env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ export DBT_ENV_SPARK_DRIVER_PATH= # /Library/simba/spark/lib/libsparkodbc_sbu.dy
export DBT_ENV_SPARK_ENDPOINT= # The endpoint ID from the Databricks HTTP path

# dbt environment variables, change these
export DBT_VERSION="1_5_0"
export DBT_CLOUD_PROJECT_ID=
export DBT_CLOUD_JOB_ID=
export DBT_CLOUD_RUN_ID=
Expand Down
72 changes: 12 additions & 60 deletions macros/upload_individual_datasets/upload_seed_executions.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,26 +38,10 @@
'{{ model.thread_id }}', {# thread_id #}
'{{ model.status }}', {# status #}

{% if model.timing != [] %}
{% for stage in model.timing if stage.name == "compile" %}
{% if loop.length == 0 %}
null, {# compile_started_at #}
{% else %}
'{{ stage.started_at }}', {# compile_started_at #}
{% endif %}
{% endfor %}

{% for stage in model.timing if stage.name == "execute" %}
{% if loop.length == 0 %}
null, {# query_completed_at #}
{% else %}
'{{ stage.completed_at }}', {# query_completed_at #}
{% endif %}
{% endfor %}
{% else %}
null, {# compile_started_at #}
null, {# query_completed_at #}
{% endif %}
{% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
{% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
{% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
{% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}

{{ model.execution_time }}, {# total_node_runtime #}
null, -- rows_affected not available {# Only available in Snowflake #}
Expand Down Expand Up @@ -95,26 +79,10 @@
'{{ model.thread_id }}', {# thread_id #}
'{{ model.status }}', {# status #}

{% if model.timing != [] %}
{% for stage in model.timing if stage.name == "compile" %}
{% if loop.length == 0 %}
null, {# compile_started_at #}
{% else %}
'{{ stage.started_at }}', {# compile_started_at #}
{% endif %}
{% endfor %}

{% for stage in model.timing if stage.name == "execute" %}
{% if loop.length == 0 %}
null, {# query_completed_at #}
{% else %}
'{{ stage.completed_at }}', {# query_completed_at #}
{% endif %}
{% endfor %}
{% else %}
null, {# compile_started_at #}
null, {# query_completed_at #}
{% endif %}
{% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
{% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
{% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
{% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}

{{ model.execution_time }}, {# total_node_runtime #}
null, -- rows_affected not available {# Databricks #}
Expand Down Expand Up @@ -170,26 +138,10 @@
'{{ model.thread_id }}', {# thread_id #}
'{{ model.status }}', {# status #}

{% if model.timing != [] %}
{% for stage in model.timing if stage.name == "compile" %}
{% if loop.length == 0 %}
null, {# compile_started_at #}
{% else %}
'{{ stage.started_at }}', {# compile_started_at #}
{% endif %}
{% endfor %}

{% for stage in model.timing if stage.name == "execute" %}
{% if loop.length == 0 %}
null, {# query_completed_at #}
{% else %}
'{{ stage.completed_at }}', {# query_completed_at #}
{% endif %}
{% endfor %}
{% else %}
null, {# compile_started_at #}
null, {# query_completed_at #}
{% endif %}
{% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
{% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
{% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
{% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}

{{ model.execution_time }}, {# total_node_runtime #}
try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #}
Expand Down
72 changes: 12 additions & 60 deletions macros/upload_individual_datasets/upload_snapshot_executions.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,26 +38,10 @@
'{{ model.thread_id }}', {# thread_id #}
'{{ model.status }}', {# status #}

{% if model.timing != [] %}
{% for stage in model.timing if stage.name == "compile" %}
{% if loop.length == 0 %}
null, {# compile_started_at #}
{% else %}
'{{ stage.started_at }}', {# compile_started_at #}
{% endif %}
{% endfor %}

{% for stage in model.timing if stage.name == "execute" %}
{% if loop.length == 0 %}
null, {# query_completed_at #}
{% else %}
'{{ stage.completed_at }}', {# query_completed_at #}
{% endif %}
{% endfor %}
{% else %}
null, {# compile_started_at #}
null, {# query_completed_at #}
{% endif %}
{% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
{% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
{% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
{% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}

{{ model.execution_time }}, {# total_node_runtime #}
null, -- rows_affected not available {# Only available in Snowflake #}
Expand Down Expand Up @@ -95,26 +79,10 @@
'{{ model.thread_id }}', {# thread_id #}
'{{ model.status }}', {# status #}

{% if model.timing != [] %}
{% for stage in model.timing if stage.name == "compile" %}
{% if loop.length == 0 %}
null, {# compile_started_at #}
{% else %}
'{{ stage.started_at }}', {# compile_started_at #}
{% endif %}
{% endfor %}

{% for stage in model.timing if stage.name == "execute" %}
{% if loop.length == 0 %}
null, {# query_completed_at #}
{% else %}
'{{ stage.completed_at }}', {# query_completed_at #}
{% endif %}
{% endfor %}
{% else %}
null, {# compile_started_at #}
null, {# query_completed_at #}
{% endif %}
{% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
{% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
{% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
{% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}

{{ model.execution_time }}, {# total_node_runtime #}
null, -- rows_affected not available {# Databricks #}
Expand Down Expand Up @@ -170,26 +138,10 @@
'{{ model.thread_id }}', {# thread_id #}
'{{ model.status }}', {# status #}

{% if model.timing != [] %}
{% for stage in model.timing if stage.name == "compile" %}
{% if loop.length == 0 %}
null, {# compile_started_at #}
{% else %}
'{{ stage.started_at }}', {# compile_started_at #}
{% endif %}
{% endfor %}

{% for stage in model.timing if stage.name == "execute" %}
{% if loop.length == 0 %}
null, {# query_completed_at #}
{% else %}
'{{ stage.completed_at }}', {# query_completed_at #}
{% endif %}
{% endfor %}
{% else %}
null, {# compile_started_at #}
null, {# query_completed_at #}
{% endif %}
{% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
{% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
{% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
{% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}

{{ model.execution_time }}, {# total_node_runtime #}
try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #}
Expand Down
48 changes: 8 additions & 40 deletions macros/upload_individual_datasets/upload_test_executions.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,26 +35,10 @@
'{{ test.thread_id }}', {# thread_id #}
'{{ test.status }}', {# status #}

{% if test.timing != [] %}
{% for stage in test.timing if stage.name == "compile" %}
{% if loop.length == 0 %}
null, {# compile_started_at #}
{% else %}
'{{ stage.started_at }}', {# compile_started_at #}
{% endif %}
{% endfor %}

{% for stage in test.timing if stage.name == "execute" %}
{% if loop.length == 0 %}
null, {# query_completed_at #}
{% else %}
'{{ stage.completed_at }}', {# query_completed_at #}
{% endif %}
{% endfor %}
{% else %}
null, {# compile_started_at #}
null, {# query_completed_at #}
{% endif %}
{% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
{% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
{% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
{% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}

{{ test.execution_time }}, {# total_node_runtime #}
null, {# rows_affected not available in Databricks #}
Expand Down Expand Up @@ -89,26 +73,10 @@
'{{ test.thread_id }}', {# thread_id #}
'{{ test.status }}', {# status #}

{% if test.timing != [] %}
{% for stage in test.timing if stage.name == "compile" %}
{% if loop.length == 0 %}
null, {# compile_started_at #}
{% else %}
'{{ stage.started_at }}', {# compile_started_at #}
{% endif %}
{% endfor %}

{% for stage in test.timing if stage.name == "execute" %}
{% if loop.length == 0 %}
null, {# query_completed_at #}
{% else %}
'{{ stage.completed_at }}', {# query_completed_at #}
{% endif %}
{% endfor %}
{% else %}
null, {# compile_started_at #}
null, {# query_completed_at #}
{% endif %}
{% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %}
{% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #}
{% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %}
{% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #}

{{ test.execution_time }}, {# total_node_runtime #}
null, {# rows_affected not available in Databricks #}
Expand Down
Loading