feature: rust bind silo-input-transformer in docker

.github/workflows/docker-build-test.yml

@@ -42,8 +42,8 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Build and test Docker image
-        run: docker-compose --env-file docker-compose.env up --build --exit-code-from sr2silo
+      - name: Build Docker image
+        run: docker-compose --env-file docker-compose.env build
 
       - name: Push to DockerHub
         if: github.ref == 'refs/heads/main'

Dockerfile

@@ -4,6 +4,9 @@ FROM continuumio/miniconda3
 # Set the working directory in the container
 WORKDIR /app
 
+# Install build-essential package for gcc and other build tools - required for the Rust project
+RUN apt-get update && apt-get install -y build-essential
+
 # Copy the environment.yml file into the container at /app
 COPY environment.yml /app/environment.yml
 
@@ -16,6 +19,9 @@ SHELL ["conda", "run", "-n", "sr2silo", "/bin/bash", "-c"]
 # Copy the current directory contents into the container at /app
 COPY . /app
 
+# Build the Rust project as a Python extension
+RUN cd silo_input_transformer && maturin develop --release
+
 # Install the sr2silo package
 RUN pip install -e .
 

README.md

@@ -26,6 +26,7 @@ The V-Pipe Docker is designed to process a single `.bam` file and upload the res
 
 ## Project Organization
 
+- `silo-input-transformer`: Is a rust based utility to handle the `fasta` to `ndjson` transformation and is here imported as a git submodule.
 - `.github/workflows`: Contains GitHub Actions used for building, testing, and publishing.
 install, and whether or not to mount the project directory into the container.
 - `.vscode/settings.json`: Contains VSCode settings specific to the project, such as the Python interpreter to use and the maximum line length for auto-formatting.

docker-compose.env

@@ -1,7 +1,7 @@
-SAMPLE_DIR=./tests/data/samples/A1_05_2024_10_08/20241024_2411515907/alignments
+SAMPLE_DIR=./tests/data/samples_large/A1_05_2024_10_08/20241024_2411515907/alignments
 SAMPLE_ID=A1_05_2024_10_08
 BATCH_ID=20241024_2411515907
-TIMELINE_FILE=./tests/data/samples/timeline_A1_05_2024_10_08.tsv
-PRIMER_FILE=./tests/data/samples/primers.yaml
+TIMELINE_FILE=./tests/data/samples_large/timeline_A1_05_2024_10_08.tsv
+PRIMER_FILE=./tests/data/samples_large/primers.yaml
 NEXTCLADE_REFERENCE=sars-cov2
 RESULTS_DIR=./results

docker-compose.yml

@@ -6,7 +6,8 @@ services:
       - ${TIMELINE_FILE}:/app/timeline.tsv
       - ${PRIMER_FILE}:/app/primers.yaml
       - ${RESULTS_DIR}:/app/results
-      - ./scripts/vp_config.json:/app/scripts/vp_config.json
+      - ./scripts/database_config.yaml:/app/scripts/database_config.yaml
+      - ./scripts/reference_genomes.json:/app/scripts/- ./scripts/database_config.yaml:/app/scripts/reference_genomes.json
     environment:
       - PYTHONUNBUFFERED=1
       - SAMPLE_DIR=${SAMPLE_DIR}

environment.yml

@@ -9,3 +9,5 @@ dependencies:
   - poetry
   - pydantic
   - click
+  - rust
+  - maturin

scripts/database_config.yaml

@@ -0,0 +1,45 @@
+schema:
+  metadata:
+    - name: sample_id
+      type: string
+      generateIndex: false
+    - name: batch_id
+      type: string
+      generateIndex: false
+    - name: sequencing_well_position
+      type: string
+      generateIndex: false
+    - name: location_code
+      type: string
+      generateIndex: false
+    - name: sampling_date
+      type: date
+      generateIndex: false
+    - name: sequencing_date
+      type: string
+      generateIndex: false
+    - name: flow_cell_serial_number
+      type: string
+      generateIndex: false
+    - name: read_length
+      type: number
+      generateIndex: false
+    - name: primer_protocol
+      type: string
+      generateIndex: false
+    - name: location_name
+      type: string
+      generateIndex: false
+    - name: primer_protocol_name
+      type: string
+      generateIndex: false
+    - name: nextclade_reference
+      type: string
+      generateIndex: false
+    - name: read_id
+      type: string
+      generateIndex: false
+  opennessLevel: OPEN
+  instanceName: wise-sarsCoV2
+  features: []
+  primaryKey: read_id