Merge pull request #15 from SergioLangaritaBenitez/main

compress and decompress input/output

DT_flood/workflows/cwl/fa_wflow_workflow.cwl

@@ -11,7 +11,7 @@ inputs:
     user: string
     password: string
     service: string
-    filename: File
+    filename: Directory
     oscar_service: Directory
     output: Directory
 outputs:

DT_flood/workflows/cwl_OSCAR/oscar_services/sfincs-interlink.yaml

@@ -0,0 +1,21 @@
+functions:
+  oscar:
+  - cluster-id:
+      name: sfincs-interlink
+      cpu: '4.0'
+      memory: 6Gi
+      image: docker://deltares/sfincs-cpu:sfincs-v2.1.1-Dollerup-Release
+      script: sfincs_interlink_script.sh
+      log_level: DEBUG   # To avoid supervisor logs in response
+      interlink_node_name: vega-new-vk
+      annotations:
+        slurm-job.vk.io/singularity-options: --no-home --no-mount /exa5
+        slurm-job.vk.io/singularity-mounts:  --bind /ceph/hpc/home/ciangottinid/oscar_home:/tmp
+        job.vk.io/singularity-mounts: --bind /ceph/hpc/data/st2301-itwin-users/:/interlink/
+        slurm-job.vk.io/flags: --job-name=test-sfincs --time=02:30:00   --nodes=1
+      input:
+      - storage_provider: minio
+        path: sfincs/in
+      output:
+      - storage_provider: minio
+        path: sfincs/out

DT_flood/workflows/cwl_OSCAR/oscar_services/sfincs-interlink_script.sh

@@ -0,0 +1,6 @@
+OUTPUT_FILE="$TMP_OUTPUT_DIR/sfincs_output.tar"
+mkdir -p /tmp/data/
+tar -xvf  "$INPUT_FILE_PATH" -C /tmp/data/
+/bin/bash -c "cd /tmp/data/ && sfincs && tar -cf sfincs_output.tar sfincs_map.nc sfincs_his.nc"
+mv /tmp/data/sfincs_output.tar  $OUTPUT_FILE
+rm /tmp/data/ -r

DT_flood/workflows/cwl_OSCAR/oscar_services/wflow_script.sh

@@ -1,6 +1,7 @@
 FILE_NAME=`basename "$INPUT_FILE_PATH"`
-OUTPUT_FILE="$TMP_OUTPUT_DIR/$FILE_NAME.nc"
+OUTPUT_FILE="$TMP_OUTPUT_DIR/wflow_output.tar"
 mkdir -p /model
 tar -xvf  "$INPUT_FILE_PATH" -C /model/
 /app/build/create_binaries/wflow_bundle/bin/wflow_cli /model/wflow_warmup/wflow_sbm.toml
-mv /model/wflow_event/instate/instates.nc  $OUTPUT_FILE
+tar -cf wflow_output.tar /model/wflow_event/instate/instates.nc
+mv wflow_output.tar  $OUTPUT_FILE

DT_flood/workflows/cwl_OSCAR/oscar_workflow/oscar.cwl

@@ -18,7 +18,7 @@ inputs:
   service:
     type: string
   filename:
-    type: File
+    type: Directory
   oscar_service:
     type: Directory
   output:

DT_flood/workflows/cwl_OSCAR/oscar_workflow/oscar.py

@@ -3,6 +3,7 @@
 import os
 from minio import Minio
 from oscar_python.client import Client
+import tarfile
 
 parser = argparse.ArgumentParser()
 
@@ -83,17 +84,17 @@ def connect_minio(minio_info):
     return client
 
 
-def upload_file_minio(client, input_info):
+def upload_file_minio(client, input_info, input_file):
     #Upload the file into input bucket
     print("Uploading the file into input bucket")
     result = client.fput_object(
         input_info["path"].split("/")[0],
-        '/'.join(input_info["path"].split("/")[1:]) + "/" + filename.split("/")[-1],
-        filename,
+        '/'.join(input_info["path"].split("/")[1:]) + "/" + input_file.split("/")[-1],
+        input_file,
     )
 
 
-def wait_output(client, output_info):
+def wait_output_and_download(client, output_info):
     #Wait the output 
     print("Waiting the output")
     with client.listen_bucket_notification(
@@ -109,12 +110,30 @@ def wait_output(client, output_info):
     print("Downloading the file")
     client.fget_object(output_info["path"].split("/")[0], 
                     outputfile,
-                    output + "/" + filename.split("/")[-1] + ".nc"
+                    output + "/" + outputfile.split("/")[-1]
     )
+    return output + "/" + outputfile.split("/")[-1]
 
 
+def compress():
+    files = os.listdir(filename)
+    tar_file_ = tarfile.open(filename + ".tar", "w")
+    for x in files:
+        tar_file_.add( name=filename + "/" +x, arcname=x)
+    tar_file_.close()
+    return filename + ".tar"
+
+
+
+def decompress(output_file):
+    with tarfile.open(output_file, 'r') as tar:
+        for member in tar.getmembers():
+            tar.extract(member, path=output)
+
+input_file = compress()
 client = check_oscar_connection()
 minio_info, input_info, output_info = check_service(client, service, service_directory)
 minio_client = connect_minio(minio_info)
-upload_file_minio(minio_client, input_info)
-wait_output(minio_client, output_info)
+upload_file_minio(minio_client, input_info, input_file)
+output_file = wait_output_and_download(minio_client, output_info)
+decompress(output_file)

DT_flood/workflows/cwl_OSCAR/workflow-minio.cwl → DT_flood/workflows/cwl_OSCAR/workflow.cwl

@@ -15,10 +15,14 @@ inputs:
     type: string
   password:
     type: string
-  service:
+  service_wflow:
     type: string
-  filename:
-    type: File
+  service_sfincs:
+    type: string
+  filename_wflow:
+    type: Directory
+  filename_sfincs:
+    type: Directory
   oscar_service:
     type: Directory
   output:
@@ -31,15 +35,27 @@ outputs:
       items: [File, Directory, string]
 
 steps:
-  oscar:
+  wflow:
     run: oscar_workflow/oscar.cwl
     in:
       oscar_script: oscar_script
       endpoint: endpoint
       user: user
       password: password
-      service: service
-      filename: filename
+      service: service_wflow
+      filename: filename_wflow
       oscar_service: oscar_service
       output: output
     out: [fa_database_out]
+  sfincs:
+    run: oscar_workflow/oscar.cwl  
+    in:
+      oscar_script: oscar_script
+      endpoint: endpoint
+      user: user
+      password: password
+      service: service_sfincs
+      filename: filename_sfincs
+      oscar_service: oscar_service
+      output: output
+    out: [fa_database_out]

DT_flood/workflows/cwl_OSCAR/workflow.yaml → DT_flood/workflows/cwl_OSCAR/workflow_config.yaml

@@ -4,10 +4,14 @@ oscar_script:
 endpoint: 
 user: 
 password: 
-service: wflow
-filename:
-  class: File
-  path: wflow_input.tar
+service_wflow: wflow
+service_sfincs: sfincs
+filename_wflow:
+  class: Directory
+  path: wflow_input
+filename_sfincs:
+  class: Directory
+  path: sfincs_input
 oscar_service:
   class: Directory
   path: oscar_services

DT_flood/workflows/run_fa_scenario.cwl

@@ -19,7 +19,7 @@ inputs:
     user: string
     password: string
     service: string
-    filename: File
+    filename: Directory
     oscar_service: Directory
     output: Directory
 outputs:

DT_flood/workflows/run_fa_scenario_config.yml

@@ -33,8 +33,8 @@ user:
 password: 
 service: wflow
 filename:
-  class: File
-  path: cwl_OSCAR/wflow_input.tar
+  class: Directory
+  path: cwl_OSCAR/wflow_input
 oscar_service:
   class: Directory
   path: cwl_OSCAR/services