diff --git a/DT_flood/workflows/cwl/fa_wflow_workflow.cwl b/DT_flood/workflows/cwl/fa_wflow_workflow.cwl index bfb6767..7b793fa 100644 --- a/DT_flood/workflows/cwl/fa_wflow_workflow.cwl +++ b/DT_flood/workflows/cwl/fa_wflow_workflow.cwl @@ -11,7 +11,7 @@ inputs: user: string password: string service: string - filename: File + filename: Directory oscar_service: Directory output: Directory outputs: diff --git a/DT_flood/workflows/cwl_OSCAR/oscar_services/sfincs-interlink.yaml b/DT_flood/workflows/cwl_OSCAR/oscar_services/sfincs-interlink.yaml new file mode 100644 index 0000000..777f2ed --- /dev/null +++ b/DT_flood/workflows/cwl_OSCAR/oscar_services/sfincs-interlink.yaml @@ -0,0 +1,21 @@ +functions: + oscar: + - cluster-id: + name: sfincs-interlink + cpu: '4.0' + memory: 6Gi + image: docker://deltares/sfincs-cpu:sfincs-v2.1.1-Dollerup-Release + script: sfincs_interlink_script.sh + log_level: DEBUG # To avoid supervisor logs in response + interlink_node_name: vega-new-vk + annotations: + slurm-job.vk.io/singularity-options: --no-home --no-mount /exa5 + slurm-job.vk.io/singularity-mounts: --bind /ceph/hpc/home/ciangottinid/oscar_home:/tmp + job.vk.io/singularity-mounts: --bind /ceph/hpc/data/st2301-itwin-users/:/interlink/ + slurm-job.vk.io/flags: --job-name=test-sfincs --time=02:30:00 --nodes=1 + input: + - storage_provider: minio + path: sfincs/in + output: + - storage_provider: minio + path: sfincs/out \ No newline at end of file diff --git a/DT_flood/workflows/cwl_OSCAR/oscar_services/sfincs-interlink_script.sh b/DT_flood/workflows/cwl_OSCAR/oscar_services/sfincs-interlink_script.sh new file mode 100644 index 0000000..3f64878 --- /dev/null +++ b/DT_flood/workflows/cwl_OSCAR/oscar_services/sfincs-interlink_script.sh @@ -0,0 +1,6 @@ +OUTPUT_FILE="$TMP_OUTPUT_DIR/sfincs_output.tar" +mkdir -p /tmp/data/ +tar -xvf "$INPUT_FILE_PATH" -C /tmp/data/ +/bin/bash -c "cd /tmp/data/ && sfincs && tar -cf sfincs_output.tar sfincs_map.nc sfincs_his.nc" +mv /tmp/data/sfincs_output.tar $OUTPUT_FILE +rm /tmp/data/ -r \ No newline at end of file diff --git a/DT_flood/workflows/cwl_OSCAR/oscar_services/wflow_script.sh b/DT_flood/workflows/cwl_OSCAR/oscar_services/wflow_script.sh index 4d48252..68a0e23 100644 --- a/DT_flood/workflows/cwl_OSCAR/oscar_services/wflow_script.sh +++ b/DT_flood/workflows/cwl_OSCAR/oscar_services/wflow_script.sh @@ -1,6 +1,7 @@ FILE_NAME=`basename "$INPUT_FILE_PATH"` -OUTPUT_FILE="$TMP_OUTPUT_DIR/$FILE_NAME.nc" +OUTPUT_FILE="$TMP_OUTPUT_DIR/wflow_output.tar" mkdir -p /model tar -xvf "$INPUT_FILE_PATH" -C /model/ /app/build/create_binaries/wflow_bundle/bin/wflow_cli /model/wflow_warmup/wflow_sbm.toml -mv /model/wflow_event/instate/instates.nc $OUTPUT_FILE +tar -cf wflow_output.tar /model/wflow_event/instate/instates.nc +mv wflow_output.tar $OUTPUT_FILE diff --git a/DT_flood/workflows/cwl_OSCAR/oscar_workflow/oscar.cwl b/DT_flood/workflows/cwl_OSCAR/oscar_workflow/oscar.cwl index a57f6f4..e530cb2 100755 --- a/DT_flood/workflows/cwl_OSCAR/oscar_workflow/oscar.cwl +++ b/DT_flood/workflows/cwl_OSCAR/oscar_workflow/oscar.cwl @@ -18,7 +18,7 @@ inputs: service: type: string filename: - type: File + type: Directory oscar_service: type: Directory output: diff --git a/DT_flood/workflows/cwl_OSCAR/oscar_workflow/oscar.py b/DT_flood/workflows/cwl_OSCAR/oscar_workflow/oscar.py index 0bc2d46..2a3661f 100644 --- a/DT_flood/workflows/cwl_OSCAR/oscar_workflow/oscar.py +++ b/DT_flood/workflows/cwl_OSCAR/oscar_workflow/oscar.py @@ -3,6 +3,7 @@ import os from minio import Minio from oscar_python.client import Client +import tarfile parser = argparse.ArgumentParser() @@ -83,17 +84,17 @@ def connect_minio(minio_info): return client -def upload_file_minio(client, input_info): +def upload_file_minio(client, input_info, input_file): #Upload the file into input bucket print("Uploading the file into input bucket") result = client.fput_object( input_info["path"].split("/")[0], - '/'.join(input_info["path"].split("/")[1:]) + "/" + filename.split("/")[-1], - filename, + '/'.join(input_info["path"].split("/")[1:]) + "/" + input_file.split("/")[-1], + input_file, ) -def wait_output(client, output_info): +def wait_output_and_download(client, output_info): #Wait the output print("Waiting the output") with client.listen_bucket_notification( @@ -109,12 +110,30 @@ def wait_output(client, output_info): print("Downloading the file") client.fget_object(output_info["path"].split("/")[0], outputfile, - output + "/" + filename.split("/")[-1] + ".nc" + output + "/" + outputfile.split("/")[-1] ) + return output + "/" + outputfile.split("/")[-1] +def compress(): + files = os.listdir(filename) + tar_file_ = tarfile.open(filename + ".tar", "w") + for x in files: + tar_file_.add( name=filename + "/" +x, arcname=x) + tar_file_.close() + return filename + ".tar" + + + +def decompress(output_file): + with tarfile.open(output_file, 'r') as tar: + for member in tar.getmembers(): + tar.extract(member, path=output) + +input_file = compress() client = check_oscar_connection() minio_info, input_info, output_info = check_service(client, service, service_directory) minio_client = connect_minio(minio_info) -upload_file_minio(minio_client, input_info) -wait_output(minio_client, output_info) \ No newline at end of file +upload_file_minio(minio_client, input_info, input_file) +output_file = wait_output_and_download(minio_client, output_info) +decompress(output_file) diff --git a/DT_flood/workflows/cwl_OSCAR/output/wflow_input.tar.nc b/DT_flood/workflows/cwl_OSCAR/output/wflow_input.tar.nc deleted file mode 100644 index da11ba8..0000000 Binary files a/DT_flood/workflows/cwl_OSCAR/output/wflow_input.tar.nc and /dev/null differ diff --git a/DT_flood/workflows/cwl_OSCAR/wflow_input.tar b/DT_flood/workflows/cwl_OSCAR/wflow_input.tar deleted file mode 100644 index ef5b48d..0000000 Binary files a/DT_flood/workflows/cwl_OSCAR/wflow_input.tar and /dev/null differ diff --git a/DT_flood/workflows/cwl_OSCAR/workflow-minio.cwl b/DT_flood/workflows/cwl_OSCAR/workflow.cwl similarity index 56% rename from DT_flood/workflows/cwl_OSCAR/workflow-minio.cwl rename to DT_flood/workflows/cwl_OSCAR/workflow.cwl index e023ddf..8814bd4 100755 --- a/DT_flood/workflows/cwl_OSCAR/workflow-minio.cwl +++ b/DT_flood/workflows/cwl_OSCAR/workflow.cwl @@ -15,10 +15,14 @@ inputs: type: string password: type: string - service: + service_wflow: type: string - filename: - type: File + service_sfincs: + type: string + filename_wflow: + type: Directory + filename_sfincs: + type: Directory oscar_service: type: Directory output: @@ -31,15 +35,27 @@ outputs: items: [File, Directory, string] steps: - oscar: + wflow: run: oscar_workflow/oscar.cwl in: oscar_script: oscar_script endpoint: endpoint user: user password: password - service: service - filename: filename + service: service_wflow + filename: filename_wflow oscar_service: oscar_service output: output out: [fa_database_out] + sfincs: + run: oscar_workflow/oscar.cwl + in: + oscar_script: oscar_script + endpoint: endpoint + user: user + password: password + service: service_sfincs + filename: filename_sfincs + oscar_service: oscar_service + output: output + out: [fa_database_out] \ No newline at end of file diff --git a/DT_flood/workflows/cwl_OSCAR/workflow.yaml b/DT_flood/workflows/cwl_OSCAR/workflow_config.yaml similarity index 54% rename from DT_flood/workflows/cwl_OSCAR/workflow.yaml rename to DT_flood/workflows/cwl_OSCAR/workflow_config.yaml index 2a87ed4..e551521 100755 --- a/DT_flood/workflows/cwl_OSCAR/workflow.yaml +++ b/DT_flood/workflows/cwl_OSCAR/workflow_config.yaml @@ -4,10 +4,14 @@ oscar_script: endpoint: user: password: -service: wflow -filename: - class: File - path: wflow_input.tar +service_wflow: wflow +service_sfincs: sfincs +filename_wflow: + class: Directory + path: wflow_input +filename_sfincs: + class: Directory + path: sfincs_input oscar_service: class: Directory path: oscar_services diff --git a/DT_flood/workflows/run_fa_scenario.cwl b/DT_flood/workflows/run_fa_scenario.cwl index b6a7133..76a3579 100644 --- a/DT_flood/workflows/run_fa_scenario.cwl +++ b/DT_flood/workflows/run_fa_scenario.cwl @@ -19,7 +19,7 @@ inputs: user: string password: string service: string - filename: File + filename: Directory oscar_service: Directory output: Directory outputs: diff --git a/DT_flood/workflows/run_fa_scenario_config.yml b/DT_flood/workflows/run_fa_scenario_config.yml index e96074f..8c2c2bb 100644 --- a/DT_flood/workflows/run_fa_scenario_config.yml +++ b/DT_flood/workflows/run_fa_scenario_config.yml @@ -33,8 +33,8 @@ user: password: service: wflow filename: - class: File - path: cwl_OSCAR/wflow_input.tar + class: Directory + path: cwl_OSCAR/wflow_input oscar_service: class: Directory path: cwl_OSCAR/services