Add wrapper to github actions

.github/workflows/test_modules.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches: "**"
   pull_request:
-    types: [opened, reopened, synchronize, closed]
+    types: [opened, reopened, synchronize]
     branches: "**"
 
 jobs:

.github/workflows/test_sftp_handle.yml → .github/workflows/test_sftp_modules.yml

@@ -1,12 +1,12 @@
-name: test_sftp_handle
+name: test_sftp_modules
 
 on:
   pull_request_target:
     types: [opened, reopened, synchronize]
     branches: "**"
 
 concurrency:
-  group: ${{ github.repository }}-test_sftp_handle
+  group: ${{ github.repository }}-test_sftp_modules
   cancel-in-progress: false
 
 jobs:
@@ -26,11 +26,11 @@ jobs:
         echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
         echo "Job originally triggered by ${{ github.actor }}"
         exit 1
-
-  test_sftp_handle:
+
+  test_download:
+    runs-on: ubuntu-latest
     needs: security_check
     if: github.repository_owner == 'BU-ISCIII'
-    runs-on: ubuntu-latest
     strategy:
       max-parallel: 1
       matrix:
@@ -56,10 +56,45 @@ jobs:
 
     - name: Run sftp_handle tests
       run: |
-        python3 tests/test_sftp_handle.py --download_option ${{ matrix.download_options }} ${{ matrix.target_folders }}
+        python3 tests/test_sftp_modules.py --download_option ${{ matrix.download_options }} ${{ matrix.target_folders }}
+      env:
+        TEST_USER: ${{ secrets.TEST_USER }}
+        TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }}
+        TEST_PORT: ${{ secrets.TEST_PORT }}
+        OUTPUT_LOCATION: ${{ github.workspace }}/tests/
+
+  test_wrapper:
+    runs-on: ubuntu-latest
+    needs: test_download
+    if: github.repository_owner == 'BU-ISCIII'
+    strategy:
+      max-parallel: 1
+      matrix:
+        download_options: ["download_only", "download_clean"]
+        target_folders: ["-t COD-test-1"]
+
+    steps:
+    - name: Set up Python 3.9.16
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.9.16'
+
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+
+    - name: Install package and dependencies
+      run: |
+        pip install -r requirements.txt
+        pip install .
+
+    - name: Run Wrapper tests
+      run: |
+        python3 tests/test_wrapper_handle.py --download_option ${{ matrix.download_options }} ${{ matrix.target_folders }}
       env:
         TEST_USER: ${{ secrets.TEST_USER }}
         TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }}
         TEST_PORT: ${{ secrets.TEST_PORT }}
         OUTPUT_LOCATION: ${{ github.workspace }}/tests/
-

CHANGELOG.md

@@ -36,6 +36,7 @@ Code contributions to the release:
 - Implement non-interactive execution of build-schema module [#416](https://github.com/BU-ISCIII/relecov-tools/pull/416)
 - Add repeated samples to test data [#413](https://github.com/BU-ISCIII/relecov-tools/pull/413)
 - Add create_summary_tables.py to assets to process data from a given epidemiological week [#418](https://github.com/BU-ISCIII/relecov-tools/pull/418)
+- Add wrapper to github actions (test_sftp_modules) [#409](https://github.com/BU-ISCIII/relecov-tools/pull/409)
 
 #### Fixes
 

tests/test_wrapper_handle.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+import os
+import sys
+import yaml
+import argparse
+from relecov_tools.download_manager import DownloadManager
+from relecov_tools.dataprocess_wrapper import ProcessWrapper
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d",
+        "--download_option",
+        type=str,
+        help="Download option",
+    )
+    parser.add_argument("-t", "--target_folders", type=str, help="Target folders")
+    args = parser.parse_args()
+
+    val_dict = {
+        "user": os.environ["TEST_USER"],
+        "password": os.environ["TEST_PASSWORD"],
+        "download_option": args.download_option,
+        "output_location": os.environ["OUTPUT_LOCATION"],
+        "target_folders": args.target_folders,
+    }
+    prepare_remote_test(**val_dict)
+
+
+def generate_config_yaml(user, password, download_option, target_folders):
+    """Generate the wrapper_config.yaml file with the desired structure."""
+    config_data = {
+        "download": {
+            "user": user,
+            "passwd": password,
+            "download_option": download_option,
+            "target_folders": target_folders,
+        },
+        "read-lab-metadata": {
+            "metadata_file": "tests/data/read_lab_metadata/metadata_lab_test.xlsx",
+            "sample_list_file": "tests/data/read_lab_metadata/samples_data_test.json",
+        },
+        "validate": {
+            "json_schema_file": "relecov_tools/schema/relecov_schema.json",
+        },
+    }
+
+    with open("wrapper_config.yaml", "w") as file:
+        yaml.dump(config_data, file, default_flow_style=False)
+
+    return "wrapper_config.yaml"
+
+
+def prepare_remote_test(**kwargs):
+    # First clean the repository.
+    print("Initating sftp module")
+
+    download_manager = DownloadManager(
+        user=kwargs["user"],
+        passwd=kwargs["password"],
+        conf_file=None,
+        download_option=kwargs["download_option"],
+        output_location=kwargs["output_location"],
+        target_folders=kwargs["target_folders"],
+    )
+    print("Openning connection to sftp")
+    download_manager.relecov_sftp.sftp_port = os.environ["TEST_PORT"]
+    if not download_manager.relecov_sftp.open_connection():
+        print("Could not open connection to remote sftp")
+        sys.exit(1)
+    remote_folders = download_manager.relecov_sftp.list_remote_folders(
+        ".", recursive=True
+    )
+    print("Connection opened with sftp")
+    clean_folders = [folder.replace("./", "") for folder in remote_folders]
+    print("Cleaning folders")
+    for folder in clean_folders:
+        if len(folder.split("/")) < 2:
+            continue
+        filelist = download_manager.relecov_sftp.get_file_list(folder)
+        for file in filelist:
+            download_manager.relecov_sftp.remove_file(file)
+        print(f"Removing remote folder {folder}")
+        download_manager.relecov_sftp.remove_dir(folder)
+
+    # Upload the test dataset to the sftp.
+    data_loc = "tests/data/sftp_handle"
+    folder_files_dict = {folder: files for folder, _, files in os.walk(data_loc)}
+    print("Uploading files to sftp...")
+    for folder, files in folder_files_dict.items():
+        if "datatest" in folder:
+            remote_dir = "COD-test-1"
+        elif "empty_test" in folder:
+            remote_dir = "COD-test-2"
+        else:
+            continue
+        base_folder = folder.split("/")[-1]
+        download_manager.relecov_sftp.make_dir(os.path.join(remote_dir, base_folder))
+        print(f"Uploading files from {base_folder}")
+        for file in files:
+            remote_path = os.path.join(remote_dir, base_folder, file)
+            local_path = os.path.join(os.path.abspath(folder), file)
+            download_manager.relecov_sftp.upload_file(local_path, remote_path)
+
+    download_manager.relecov_sftp.close_connection()
+
+    print("Initiating wrapper configuration")
+    conf_file = generate_config_yaml(
+        kwargs["user"],
+        kwargs["password"],
+        kwargs["download_option"],
+        kwargs["target_folders"],
+    )
+
+    print("Initiating Wrapper")
+    wrapper_manager = ProcessWrapper(
+        config_file=conf_file,
+        output_folder=kwargs["output_location"],
+    )
+
+    def test_wrapper(wrapper_manager):
+        wrapper_manager.run_wrapper()
+
+    test_wrapper(wrapper_manager)
+
+
+if __name__ == "__main__":
+    main()