Skip to content

Commit

Permalink
feat: Adds idc_current dataset for IDC-TCIA (#239)
Browse files Browse the repository at this point in the history 
* adds idc_current dataset

* fix: conventional commits
  • Loading branch information
@adlersantos
adlersantos committed Dec 14, 2021
1 parent 25d615d commit f92a2f7
Show file tree
Hide file tree
Showing 22 changed files with 35 additions and 3 deletions.
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/analysis_results_metadata.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.analysis_results_metadata`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/auxiliary_metadata.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.auxiliary_metadata`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/dicom_all.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.dicom_all`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/dicom_metadata.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.dicom_metadata`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/dicom_metadata_curated.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.dicom_metadata_curated`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/measurement_groups.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.measurement_groups`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/nlst_canc.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.nlst_canc`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/nlst_ctab.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.nlst_ctab`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/nlst_ctabc.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.nlst_ctabc`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/nlst_prsn.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.nlst_prsn`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/nlst_screen.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.nlst_screen`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/original_collections_metadata.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.original_collections_metadata`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/qualitative_measurements.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.qualitative_measurements`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/quantitative_measurements.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.quantitative_measurements`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/segmentations.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.segmentations`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/tcga_biospecimen_rel9.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.tcga_biospecimen_rel9`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/tcga_clinical_rel9.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.tcga_clinical_rel9`
1 change: 1 addition & 0 deletions datasets/idc/_images/generate_bq_views/queries/current/version_metadata.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select * from `PROJECT.idc_v5.version_metadata`
10 changes: 10 additions & 0 deletions datasets/idc/_terraform/idc_dataset.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,3 +75,13 @@ resource "google_bigquery_dataset" "idc_v5" {
output "bigquery_dataset-idc_v5-dataset_id" {
value = google_bigquery_dataset.idc_v5.dataset_id
}

resource "google_bigquery_dataset" "idc_current" {
dataset_id = "idc_current"
project = var.project_id
description = "Imaging Data Commons (IDC) - The Cancer Imaging Archive (TCIA) current data"
}

output "bigquery_dataset-idc_current-dataset_id" {
value = google_bigquery_dataset.idc_current.dataset_id
}
2 changes: 1 addition & 1 deletion datasets/idc/copy_tcia_data/copy_tcia_data_dag.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@
"QUERIES_DIR": "/custom/queries",
"GCP_PROJECT": "{{ var.value.gcp_project }}",
"DATASET_NAME": "idc",
"DATASET_VERSIONS": '["v1", "v2", "v3", "v4", "v5"]',
"DATASET_VERSIONS": '["v1", "v2", "v3", "v4", "v5", "current"]',
},
resources={"limit_memory": "128M", "limit_cpu": "200m"},
)
Expand Down
4 changes: 2 additions & 2 deletions datasets/idc/copy_tcia_data/pipeline.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ dag:
default_args:
owner: "Google"
depends_on_past: False
start_date: '2021-11-17'
start_date: '2021-11-23'
max_active_runs: 1
schedule_interval: "@monthly"
catchup: False
Expand Down Expand Up @@ -92,7 +92,7 @@ dag:
GCP_PROJECT: "{{ var.value.gcp_project }}"
DATASET_NAME: "idc"
DATASET_VERSIONS: >-
["v1", "v2", "v3", "v4", "v5"]
["v1", "v2", "v3", "v4", "v5", "current"]
resources:
limit_memory: "128M"
limit_cpu: "200m"
Expand Down
4 changes: 4 additions & 0 deletions datasets/idc/dataset.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,7 @@ resources:
- type: bigquery_dataset
dataset_id: idc_v5
description: Imaging Data Commons (IDC) - The Cancer Imaging Archive (TCIA) v5 data

- type: bigquery_dataset
dataset_id: idc_current
description: Imaging Data Commons (IDC) - The Cancer Imaging Archive (TCIA) current data

0 comments on commit f92a2f7

Please sign in to comment.