Merge pull request #26 from apriltuesday/EVA-3517

EVA-3517 - Issues from test run

README.md

@@ -20,13 +20,14 @@ eva-sub-cli.py --help
 
 ### From source using Docker
 
+Docker provides an easy way to run eva-sub-cli without installing dependencies separately.
 This method requires just Python 3.8+ and [Docker](https://docs.docker.com/engine/install/) to be installed.
 Then either clone the git repository, or download the newest tagged release from [here](https://github.com/EBIvariation/eva-sub-cli/tags):
 ```bash
 git clone [email protected]:EBIvariation/eva-sub-cli.git
-# OR
+# OR (replace "v0.2" with the newest version)
 wget -O eva-sub-cli.zip https://github.com/EBIvariation/eva-sub-cli/archive/refs/tags/v0.2.zip
-unzip eva-sub-cli.zip
+unzip eva-sub-cli.zip && mv eva-sub-cli-* eva-sub-cli
 ```
 
 Then install the library and its dependencies as follows (e.g. in a virtual environment):
@@ -36,6 +37,11 @@ pip install -r requirements.txt
 python setup.py install
 ```
 
+To check it is installed correctly, you can run:
+```bash
+eva-sub-cli.py -h
+```
+
 ### From source natively
 
 This method requires the following:

bin/eva-sub-cli.py

@@ -56,11 +56,11 @@ def validate_command_line_arguments(args, argparser):
                                help="Json file that describe the project, analysis, samples and files")
     metadata_group.add_argument("--metadata_xlsx",
                                help="Excel spreadsheet  that describe the project, analysis, samples and files")
-    argparser.add_argument('--tasks', nargs='*', choices=[VALIDATE, SUBMIT], default=[SUBMIT],
-                           help='Select a task to perform. Selecting VALIDATE will run the validation regardless of the outcome of '
-                                'previous runs. Selecting SUBMIT will run validate only if the validation was not performed '
-                                'successfully before and then run the submission.')
-    argparser.add_argument('--executor', choices=[DOCKER, NATIVE], default=NATIVE,
+    argparser.add_argument('--tasks', nargs='*', choices=[VALIDATE, SUBMIT], default=[SUBMIT], type=str.lower,
+                           help='Select a task to perform. Selecting VALIDATE will run the validation regardless of the'
+                                ' outcome of previous runs. Selecting SUBMIT will run validate only if the validation'
+                                ' was not performed successfully before and then run the submission.')
+    argparser.add_argument('--executor', choices=[DOCKER, NATIVE], default=NATIVE, type=str.lower,
                            help='Select an execution type for running validation (default native)')
     credential_group = argparser.add_argument_group('Credential', 'Specify the Webin credential you want to use to '
                                                                   'upload to the EVA')

eva_sub_cli/jinja_templates/file_validation.html

@@ -5,7 +5,7 @@
         {% if check_type == "assembly_check" %}
             {% set nb_match = result.get("match", 0) %}
             {% set nb_total = result.get("total", 0) %}
-            {% set match_percentage = nb_match / nb_total * 100 %}
+            {% set match_percentage = nb_match / nb_total * 100 if nb_total else 0 %}
             {% if result.get("nb_mismatch", 0) > 0 %}
                 {% set icon = "❌" %}
                 {% set row_class = "report-section fail collapsible" %}

eva_sub_cli/validators/docker_validator.py

@@ -74,7 +74,7 @@ def run_docker_validator(self):
             # copy validation result to user host
             self._run_quiet_command(
                 "Copy validation output from container to host",
-                f"{self.docker_path} cp {self.container_name}:{container_validation_dir}/{container_validation_output_dir} {self.output_dir}"
+                f"{self.docker_path} cp {self.container_name}:{container_validation_dir}/{container_validation_output_dir}/. {self.output_dir}"
             )
         except subprocess.CalledProcessError as ex:
             logger.error(ex)
@@ -200,7 +200,7 @@ def _copy(file_description, file_path):
                 _copy('vcf files', row['vcf'])
                 _copy('fasta files', row['fasta'])
                 # report is optional
-                if row['report']:
+                if row.get('report'):
                     _copy('assembly report files', row['report'])
 
 

eva_sub_cli/validators/validator.py

@@ -56,10 +56,12 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.sub_config.backup()
         self.sub_config.write()
+
     @staticmethod
     def _run_quiet_command(command_description, command, **kwargs):
         return run_command_with_output(command_description, command, stdout_log_level=logging.DEBUG,
                                        stderr_log_level=logging.DEBUG, **kwargs)
+
     def _find_vcf_and_fasta_files(self):
         vcf_files = []
         fasta_files = []
@@ -267,6 +269,7 @@ def _assembly_check_log(self, vcf_name):
         return resolve_single_file_path(
             os.path.join(self.output_dir, 'assembly_check', vcf_name + '.assembly_check.log')
         )
+
     @lru_cache
     def _assembly_check_valid_vcf(self, vcf_name):
         return resolve_single_file_path(
@@ -317,10 +320,15 @@ def _load_fasta_check_results(self):
             fasta_file_name = os.path.basename(fasta_file)
             fasta_check = resolve_single_file_path(os.path.join(self.output_dir, f'{fasta_file_name}_check.yml'))
             self.results['fasta_check'] = {}
+            if not fasta_check:
+                continue
             with open(fasta_check) as open_yaml:
                 self.results['fasta_check'][fasta_file_name] = yaml.safe_load(open_yaml)
 
     def _load_sample_check_results(self):
+        self.results['sample_check'] = {}
+        if not self._sample_check_yaml:
+            return
         with open(self._sample_check_yaml) as open_yaml:
             self.results['sample_check'] = yaml.safe_load(open_yaml)
         self.results['sample_check']['report_path'] = self._sample_check_yaml
@@ -337,6 +345,9 @@ def clean_read(ifile):
             if l:
                 return ansi_escape.sub('', l).strip()
 
+        self.results['metadata_check'] = {}
+        if not metadata_check_file:
+            return
         with open(metadata_check_file) as open_file:
             errors = []
             collect = False
@@ -375,7 +386,7 @@ def _convert_biovalidator_validation_to_spreadsheet(self):
             xls2json_conf = yaml.safe_load(open_file)
 
         self.results['metadata_check']['spreadsheet_errors'] = []
-        for error in self.results['metadata_check']['json_errors']:
+        for error in self.results['metadata_check'].get('json_errors', {}):
             sheet_json, row_json, attribute_json = self._parse_metadata_property(error['property'])
             sheet = self._convert_metadata_sheet(sheet_json, xls2json_conf)
             row = self._convert_metadata_row(sheet, row_json, xls2json_conf)
@@ -400,7 +411,8 @@ def _convert_biovalidator_validation_to_spreadsheet(self):
             })
 
     def _write_spreadsheet_validation_results(self):
-        if 'spreadsheet_errors' in self.results['metadata_check']:
+        if ('spreadsheet_errors' in self.results['metadata_check']
+                and 'json_report_path' in self.results['metadata_check']):
             spreadsheet_report_file = os.path.join(os.path.dirname(self.results['metadata_check']['json_report_path']),
                                                    'metadata_spreadsheet_validation.txt')
             with open(spreadsheet_report_file, 'w') as open_file:
@@ -438,4 +450,5 @@ def create_reports(self):
         file_path = os.path.join(self.output_dir, 'report.html')
         with open(file_path, "w") as f:
             f.write(report_html)
+        self.info(f'View the validation report: {file_path}')
         return file_path