Sprint task (#65)

* adding dug parsers for kids first and cancer commons

* dir for new dag tasks

* adding steps for kfdrc and crdc

* bump dug version

* adding indexes

* changing create index for v2.8.4

* adding sprint parsers

* fixing merge issue

* adding dug new code

* correct sprint path

* bump dug version

* bump dug version

Co-authored-by: Yaphetkg <[email protected]>

dags/annotate.py

@@ -6,7 +6,8 @@
 
 from dug_helpers.dug_utils import DugUtil, get_topmed_files, get_dbgap_files,\
     get_nida_files, get_sparc_files, get_anvil_files,\
-    get_cancer_data_commons_files, get_kids_first_files
+    get_cancer_data_commons_files, get_kids_first_files,\
+    get_sprint_files
 from roger.dag_util import default_args, create_python_task
 
 DAG_ID = 'annotate_dug'
@@ -64,6 +65,10 @@
             prepare_files = create_python_task(dag, "get_kids_first_files", get_kids_first_files)
             annotate_files = create_python_task(dag, "annotate_kids_first_files",
                                                 DugUtil.annotate_kids_first_files)
+        elif data_set == "sprint":
+            prepare_files = create_python_task(dag, "get_sprint_files", get_sprint_files)
+            annotate_files = create_python_task(dag, "annotate_sprint_files",
+                                                DugUtil.annotate_sprint_files)
         intro >> prepare_files
         prepare_files >> clear_annotation_items
         clear_annotation_items >> annotate_files

dags/dug_helpers/dug_utils.py

@@ -645,6 +645,17 @@ def annotate_sparc_files(config=None, to_string=False, files=None):
             output_log = dug.log_stream.getvalue() if to_string else ''
         return output_log
 
+    @staticmethod
+    def annotate_sprint_files(config=None, to_string=False, files=None):
+        with Dug(config, to_string=to_string) as dug:
+            if files is None:
+                files = Util.dug_sprint_objects()
+            parser_name = "SPRINT"
+            dug.annotate_files(parser_name=parser_name,
+                               parsable_files=files)
+            output_log = dug.log_stream.getvalue() if to_string else ''
+        return output_log
+
     @staticmethod
     def annotate_topmed_files(config=None, to_string=False, files=None):
         with Dug(config, to_string=to_string) as dug:
@@ -877,6 +888,9 @@ def get_cancer_data_commons_files(config: RogerConfig, to_string=False) -> List[
     return get_versioned_files(config, "crdc", "crdc", data_store=config.dug_inputs.data_source, unzip=True)
 
 
+def get_sprint_files(config: RogerConfig, to_string=False) -> List[str]:
+    return get_versioned_files(config, "sprint", "sprint", data_store=config.dug_inputs.data_source, unzip=True)
+
 
 def get_topmed_files(config: RogerConfig, to_string=False) -> List[str]:
     return get_versioned_files(config, "topmed", "topmed", data_store=config.dug_inputs.data_source, unzip=False)

dags/metadata.yaml

@@ -140,4 +140,10 @@ dug_inputs:
       files:
         s3:
           - "bdc/v1.0/CRDC.tar.gz"
-      format: crdc
+      format: crdc
+    - name: sprint
+      version: v1.0
+      files:
+        s3:
+          - "sprint/v1.0/StanfordSPRINT_DataDictionary_2020-12-16.tar.gz"
+      format: sprint

dags/roger/core.py

@@ -292,6 +292,11 @@ def dug_anvil_path():
         """Anvil source files"""
         return Util.dug_input_files_path('anvil')
 
+    @staticmethod
+    def dug_sprint_path():
+        """Anvil source files"""
+        return Util.dug_input_files_path('sprint')
+
     @staticmethod
     def dug_crdc_path():
         """Anvil source files"""
@@ -319,6 +324,13 @@ def dug_anvil_objects():
             lambda file_name: not file_name.startswith('GapExchange_') and file_name.endswith('.xml'), file_path)
         return sorted([str(f) for f in files])
 
+    @staticmethod
+    def dug_sprint_objects():
+        file_path = Util.dug_sprint_path()
+        files = Util.get_files_recursive(
+            lambda file_name: file_name.endswith('.xml'), file_path)
+        return sorted([str(f) for f in files])
+
     @staticmethod
     def dug_crdc_objects():
         file_path = Util.dug_crdc_path()

requirements.txt

@@ -11,7 +11,7 @@ redisgraph-bulk-loader==0.9.5
 requests<2.24.0
 pytest==6.2.2
 PyYAML==5.3.1
-git+https://github.com/helxplatform/[email protected].4
+git+https://github.com/helxplatform/[email protected].5
 elasticsearch==7.11.0
 biolinkml>=1.5.10
 orjson