Merge branch 'SciLifeLab:master' into master

.devcontainer/devcontainer.json

@@ -6,12 +6,13 @@
     // Sets the run context to one level up instead of the .devcontainer folder.
     "context": "..",
     // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
-    "dockerfile": "../Dockerfile"
+    "dockerfile": "../Dockerfile",
+    "target": "base"
   },
   "features": {},
   "customizations": {
     "vscode": {
-      "extensions": ["ms-python.python"]
+      "extensions": ["ms-python.python", "eamodio.gitlens"]
     }
   },
   // Features to add to the dev container. More info: https://containers.dev/features.

.github/workflows/check-log.yml

@@ -12,12 +12,8 @@ jobs:
 
       - name: Check for VERSIONLOG.MD changes
         id: versionlog_check
-        # 1) Find the common ancestor between the current HEAD and the base branch
-        # 2) Then see if the versionlog has been updated in the PR since it diverged
-        #    from the common ancestor
         run: |
-          PR_BASE_SHA=$(git merge-base HEAD ${{ github.event.pull_request.base.sha }})
-          FILE_CHANGED=$(git diff --name-only $PR_BASE_SHA HEAD | grep 'VERSIONLOG.md' || true)
+          FILE_CHANGED=$(git diff --name-only ${{ github.event.pull_request.base.sha }} HEAD | grep 'VERSIONLOG.md' || true)
           if [ -n "$FILE_CHANGED" ]; then
             echo "VERSIONLOG.MD has been changed."
           else

.github/workflows/lint-code.yml

@@ -76,7 +76,7 @@ jobs:
         run: pip install -r requirements.txt
 
       - name: Run pipreqs
-        run: pipreqs --savepath pipreqs.txt
+        run: pipreqs --savepath pipreqs.txt taca
 
       - name: Compare requirements
         run: |

.github/workflows/test-code.yml

@@ -7,6 +7,8 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -21,3 +23,10 @@ jobs:
       - name: pytest
         # Options are configured in pyproject.toml
         run: pytest
+      - name: CodeCov
+        run: |
+          # Replace `linux` below with the appropriate OS
+          # Options are `alpine`, `linux`, `macos`, `windows`
+          curl -Os https://uploader.codecov.io/latest/linux/codecov
+          chmod +x codecov
+          ./codecov -t ${{ secrets.CODECOV_TOKEN }}

Dockerfile

@@ -1,14 +1,13 @@
-FROM python:3.10
+FROM python:3.10 AS base
 
 # Update pip to latest version
 RUN python -m pip install --upgrade pip
 
-
-# Install Nextflow dependencies
+# Install dependencies
 RUN apt-get update \
     && apt-get upgrade -y \
-    && apt-get install -y git \
-    && apt-get install -y curl
+    && apt-get install -y curl \
+    && apt-get install -y rsync
 
 # Needed to install requirements,
 # in devcontainer a local mounted version of flowcell_parser is used
@@ -25,3 +24,9 @@ RUN python -m pip install -r requirements-dev.txt
 
 RUN mkdir /root/.taca/
 COPY tests/data/taca_test_cfg.yaml /root/.taca/taca.yaml
+
+FROM base AS testing
+COPY . /taca
+RUN python -m pip install -e /taca
+WORKDIR /taca/tests
+CMD ["python", "-m", "unittest"]

README.md

@@ -12,6 +12,15 @@
 
 This package contains several tools for projects and data management in the [National Genomics Infrastructure](https://ngisweden.scilifelab.se/) in Stockholm, Sweden.
 
+### Run tests in docker
+
+```shell
+git clone https://github.com/SciLifeLab/TACA.git
+cd TACA
+docker build -t taca_testing --target testing .
+docker run -it taca_testing
+```
+
 ## Installation
 
 Inside the repo, run `pip install .`

VERSIONLOG.md

@@ -1,5 +1,45 @@
 # TACA Version Log
 
+## 20240520.1
+
+Fix erroneous name of pod5 output dir for ONT runs.
+
+## 20240507.1
+
+Broaden ONT transfer script's categorization of QC runs to either experiment dir or sample dir starting with "QC\_".
+
+## 20240422.1
+
+Refine GHA VERSIONLOG.md check to compare to merge-base, not branch-base.
+
+## 20240410.1
+
+Expand test coverage by starting and checking demultiplexing for a NovaSeqXPlus run.
+
+## 20240321.1
+
+Include project IDs in the run folder tarball
+
+## 20240315.1
+
+Fix cases that MiSeq samplesheet misses index or index2
+
+## 20240304.1
+
+- Make sure TACA can handle runs that generate NO sequencing data at all
+- Refactor logic of control function to reduce complexity
+- Introduce custom Exception for quiet skipping (waiting on) runs
+- Improve documentation
+- Minor polishing of test to pass
+
+## 20240229.1
+
+Increase test coverage to 20%.
+
+## 20240209.1
+
+Implement CodeCoverage in CI.
+
 ## 20240208.2
 
 Implement CI testing and increase testing coverage.

pyproject.toml

@@ -2,6 +2,9 @@ title = "taca"
 
 # === LINTING ================================================================
 
+[tool.ruff]
+exclude = ["tests_old/**"]
+
 [tool.ruff.lint]
 select = [
     #         Ruff default rules
@@ -34,7 +37,7 @@ filterwarnings = [
     'ignore::DeprecationWarning:couchdb.*',
     'ignore::DeprecationWarning:pkg_resources.*',
 ]
-addopts = "--cov=./taca --cov-report term-missing -vv --cache-clear tests/pytest/"
+addopts = "--cov=./taca --cov-report term-missing -vv --cache-clear tests/"
 
 [tool.coverage.run]
 # The comment "# pragma: no cover" can be used to exclude a line from coverage

requirements.txt

@@ -6,4 +6,3 @@ pandas
 python_crontab
 python_dateutil
 setuptools
-sphinx_rtd_theme

taca/__init__.py

@@ -1,4 +1,3 @@
-""" Main TACA module
-"""
+"""Main TACA module"""
 
 __version__ = "1.0.0"

taca/analysis/analysis.py

@@ -1,4 +1,5 @@
 """Analysis methods for TACA."""
+
 import glob
 import logging
 import os
@@ -19,14 +20,11 @@
 logger = logging.getLogger(__name__)
 
 
-def get_runObj(run, software):
+def get_runObj(
+    run: os.PathLike, software: str
+) -> MiSeq_Run | NextSeq_Run | NovaSeq_Run | NovaSeqXPlus_Run | None:
     """Tries to read runParameters.xml to parse the type of sequencer
-        and then return the respective Run object (MiSeq, HiSeq..)
-
-    :param run: run name identifier
-    :type run: string
-    :returns: returns the sequencer type object,
-    None if the sequencer type is unknown of there was an error
+    and then return the respective Run object (MiSeq, HiSeq..)
     """
 
     if os.path.exists(os.path.join(run, "runParameters.xml")):
@@ -37,7 +35,7 @@ def get_runObj(run, software):
         logger.error(
             f"Cannot find RunParameters.xml or runParameters.xml in the run folder for run {run}"
         )
-        return
+        return None
 
     run_parameters_path = os.path.join(run, run_parameters_file)
     try:
@@ -116,8 +114,8 @@ def _upload_to_statusdb(run):
     parser = run.runParserObj
     # Check if I have NoIndex lanes
     for element in parser.obj["samplesheet_csv"]:
-        if (
-            "NoIndex" in element["index"] or not element["index"]
+        if "NoIndex" in element.get("index", "") or not element.get(
+            "index"
         ):  # NoIndex in the case of HiSeq, empty in the case of HiSeqX
             lane = element["Lane"]  # This is a lane with NoIndex
             # In this case PF Cluster is the number of undetermined reads
@@ -210,7 +208,7 @@ def transfer_runfolder(run_dir, pid, exclude_lane):
 
     # Create a tar archive of the runfolder
     dir_name = os.path.basename(run_dir)
-    archive = run_dir + ".tar.gz"
+    archive = run_dir + "_" + "_".join(pid_list) + ".tar.gz"
     run_dir_path = os.path.dirname(run_dir)
 
     # Prepare the options for excluding lanes
@@ -413,13 +411,13 @@ def _process(run):
                     )
                 else:
                     sbt = f"{run.id} Demultiplexing Completed!"
-                    msg = """The run {run} has been demultiplexed without any error or warning.
+                    msg = f"""The run {run.id} has been demultiplexed without any error or warning.
 
                     The Run will be transferred to the analysis cluster for further analysis.
 
-                    The run is available at : https://genomics-status.scilifelab.se/flowcells/{run}
+                    The run is available at : https://genomics-status.scilifelab.se/flowcells/{run.id}
 
-                    """.format(run=run.id)
+                    """
                 run.send_mail(sbt, msg, rcp=CONFIG["mail"]["recipients"])
 
             # Copy demultiplex stats file, InterOp meta data and run xml files to shared file system for LIMS purpose