run_workflow.jx

{
  "define": {
    "STEPS": workflow,
    "BASE_DIR": configuration["working_space"],
    "WORKING_SPACE": configuration["working_space"],
    "RESULTS_FILE_NAMES": ["result.json", "cached_files_makeflow_list.json"],
    "IMAGE_MOUNT_SOURCE": "testing",
    "EXPERIMENT_METADATA_FILENAME": configuration["experiment_filename"],
    "SOURCE_DATA_FOLDER_NAME": configuration["source_data_folder_name"],
    "CACHE_FOLDER_NAME": configuration["cache_folder_name"],
    "WORKFLOW_STEP_CACHE_FILE_NAME": "cached_files_makeflow_list.json",
    "SCRIPT_FOLDER": configuration["script_folder"]
   },
  "rules": [
    {
      "command": "echo Creating starting folder for first step in workflow && mkdir -p ${STARTING_FOLDER}",
      "environment": {
        "STARTING_FOLDER": WORKING_SPACE + "1"
      },
      "inputs": [],
      "outputs": [
        WORKING_SPACE + "1"
      ]
    },
    {
      "workflow": "${WORKFLOW_NAME}",
      "environment": {
        "WORKFLOW_NAME": format("sub_workflow%d.jx", ONE_STEP["execution_order"])
      },
      "args": {
        "NAME": ONE_STEP["name"],
        "STEP_NUMBER": format("%d", ONE_STEP["execution_order"]),
        "WORKING_SPACE": WORKING_SPACE,
        "MAKEFLOW_FILE": ONE_STEP["makeflow_file"],
        "DOCKER_IMAGE": ONE_STEP["docker_image"],
        "RESULTS_FILE_NAMES": RESULTS_FILE_NAMES,
        "BASE_DIR": BASE_DIR,
        "IMAGE_MOUNT_SOURCE": IMAGE_MOUNT_SOURCE,
        "RELATIVE_WORKING_FOLDER": WORKING_SPACE + ONE_STEP["step_folder"],
        "CACHE_DIR": WORKING_SPACE + ONE_STEP["step_folder"] + CACHE_FOLDER_NAME + "/",
        "DATA_FOLDER_NAME": WORKING_SPACE + ONE_STEP["step_folder"] + SOURCE_DATA_FOLDER_NAME,
        "SOURCES_FILE_FOLDER": ONE_STEP["sources_folder"],
        "EXPERIMENT_METADATA_RELATIVE_PATH": WORKING_SPACE + ONE_STEP["step_folder"] + SOURCE_DATA_FOLDER_NAME + "/" + EXPERIMENT_METADATA_FILENAME,
        "EXPERIMENT_METADATA_FILENAME": EXPERIMENT_METADATA_FILENAME,
        "RESULTS_FILE_PATH": BASE_DIR + ONE_STEP["step_folder"],
        "PREVSTEP_CACHE_JSON": ONE_STEP["sources_folder"] + WORKFLOW_STEP_CACHE_FILE_NAME,
        "NEXTSTEP_FOLDER": WORKING_SPACE + format("%d", ONE_STEP["next_step"]),
        "CURRENT_STEP_CACHE_JSON": WORKING_SPACE + ONE_STEP["step_folder"] + WORKFLOW_STEP_CACHE_FILE_NAME,
        "SCRIPT_FOLDER": SCRIPT_FOLDER
      },
      "inputs": [
        WORKING_SPACE + format("%d", ONE_STEP["execution_order"])
      ],
      "outputs": [
        WORKING_SPACE + format("%d", ONE_STEP["next_step"])
      ]
    } for ONE_STEP in STEPS,
  ]
}