![](\"https://lamin-site-assets.s3.amazonaws.com/.lamindb/FtEgTeQ9FXdbVWNnVTZ2.png\")
"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "ee3db779",
- "metadata": {},
- "source": [
- "## View the database content"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "e45c2584",
- "metadata": {},
- "outputs": [],
- "source": [
- "ln.view()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "5f3f95a8",
- "metadata": {
- "tags": [
- "hide-cell"
- ]
- },
- "outputs": [],
- "source": [
- "# clean up the test instance:\n",
- "!lamin delete --force nextflow-mcmicro"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3 (ipykernel)",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.1.-1"
- },
- "nbproject": {
- "id": "8124Vtle6ZrO",
- "parent": null,
- "pypackage": null,
- "time_init": "2023-07-24T13:06:04.795992+00:00",
- "user_handle": null,
- "user_id": null,
- "user_name": null,
- "version": "0"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
diff --git a/docs/mcmicro.py b/docs/mcmicro.py
deleted file mode 100644
index 63d3649..0000000
--- a/docs/mcmicro.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# mcmicro.py
-
-"""This script runs the MCMICRO pipeline and tracks input/output data in LaminDB.
-
-Usage: python mcmicro.py exemplar-001
-"""
-
-import subprocess
-import argparse
-import lamindb as ln
-import yaml
-
-# get args from command line
-parser = argparse.ArgumentParser()
-parser.add_argument("input", type=str, help="Input folder name.")
-args = parser.parse_args()
-
-transform = ln.Transform(
- name="MCMICRO",
- version="1.0.0",
- type="pipeline",
- reference="https://github.com/labsyspharm/mcmicro",
-)
-ln.context.track(transform=transform)
-run = ln.context.run
-
-# get the input data from LaminDB
-mcmicro_input = ln.Artifact.filter(description=args.input).one()
-input_dir = mcmicro_input.cache()
-
-# execute the nextflow pipeline to download example data
-report = f"{args.input}-mcmicro-execution_report.html"
-subprocess.run(
- [
- "nextflow",
- "run",
- "https://github.com/labsyspharm/mcmicro",
- "--in",
- input_dir,
- "--start-at",
- "illumination",
- "--stop-at",
- "registration",
- "-with-report",
- report,
- ]
-)
-
-# get the nextflow execution id from the log (first row)
-nextflow_id = subprocess.getoutput(
- "nextflow log | tail -n 1 | awk -F '\t' '{print $6}'"
-)
-# optionally, tag the transform
-ulabel = ln.ULabel(name="nextflow").save()
-run.transform.ulabels.add(ulabel)
-# track the execution report, set visibility to "hidden" to avoid cluttering the artifacts
-report_artifact = ln.Artifact(
- report, description=f"nextflow report of {nextflow_id}", visibility=0, run=False
-).save()
-run.report = report_artifact
-run.reference = nextflow_id
-run.reference_type = "nextflow_id"
-run.save()
-# optionally, track the pipeline parameters
-with open(f"{input_dir}/qc/params.yml") as params_file:
- qc_params = yaml.safe_load(params_file)
-ln.Param(name="qc_params", dtype="dict").save()
-run.params.add_values({"qc_params": qc_params})
-
-# register the output artifact
-output = ln.Artifact.from_dir(f"{input_dir}/registration")
-ln.save(output)
-
-ln.finish()
diff --git a/docs/mcmicro_exemplar.py b/docs/mcmicro_exemplar.py
deleted file mode 100644
index 4ffa238..0000000
--- a/docs/mcmicro_exemplar.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# mcmicro_exemplar.py
-
-"""This script downloads the MCMICRO exemplar data and registers it in LaminDB.
-
-Usage: python mcmicro_exemplar.py exemplar-001
-"""
-
-import lamindb as ln
-import subprocess
-import argparse
-
-
-# get args from command line
-parser = argparse.ArgumentParser()
-parser.add_argument("name", type=str, help="Name of the example data.")
-args = parser.parse_args()
-if args.name not in ["exemplar-001", "exemplar-002"]:
- raise ValueError("Invalid name. Use 'exemplar-001' or 'exemplar-002'.")
-
-# execute the nextflow pipeline to download example data
-report = f"{args.name}_mcmicro-exemplar_execution_report.html"
-subprocess.run(
- [
- "nextflow",
- "run",
- "labsyspharm/mcmicro/exemplar.nf",
- "--name",
- args.name,
- "-with-report",
- report,
- ]
-)
-
-# get the nextflow execution id from the log (last row, latest run is at the bottom)
-nextflow_id = subprocess.getoutput(
- "nextflow log | tail -n 1 | awk -F '\t' '{print $6}'"
-)
-
-# track the pipeline transform
-transform = ln.Transform(
- name="MCMICRO exemplar",
- version="1.0.0",
- type="pipeline",
- reference="https://github.com/labsyspharm/mcmicro",
-)
-ln.context.track(transform=transform)
-run = ln.context.run
-# optionally, tag the transform
-ulabel = ln.ULabel(name="nextflow").save()
-run.transform.ulabels.add(ulabel)
-# track the execution report, set visibility to "hidden" to avoid cluttering the artifacts
-report_artifact = ln.Artifact(
- report, description=f"nextflow report of {nextflow_id}", visibility=0, run=False
-).save()
-run.report = report_artifact
-run.reference = nextflow_id
-run.reference_type = "nextflow_id"
-run.save()
-# optionally, track the pipeline parameters
-ln.Param(name="name", dtype="str").save()
-run.params.add_values({"name": args.name})
-
-# register the downloaded folder
-exemplar_dir = ln.Artifact(args.name, description=args.name)
-exemplar_dir.save()
-
-ln.finish()
diff --git a/docs/nf_core_scrnaseq.ipynb b/docs/nf_core_scrnaseq.ipynb
new file mode 100644
index 0000000..21d3505
--- /dev/null
+++ b/docs/nf_core_scrnaseq.ipynb
@@ -0,0 +1,287 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "a1c40541-e05e-48a3-8b50-33baf3d6d0d4",
+ "metadata": {},
+ "source": [
+ "# Nextflow"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cd12ac1a-73e6-44e2-a854-6fa5e52cfd41",
+ "metadata": {},
+ "source": [
+ "[Nextflow](https://www.nextflow.io/) is the most widely used workflow manager in bioinformatics.\n",
+ "\n",
+ "This guide shows how to register a Nextflow run with inputs & outputs for the example of the [nf-core/scrnaseq](https://nf-co.re/scrnaseq/latest) pipeline by running a Python script.\n",
+ "\n",
+ "The approach could be automated by deploying the script via\n",
+ "\n",
+ "1. a serverless environment trigger (e.g., AWS Lambda)\n",
+ "2. a [post-run script](https://docs.seqera.io/platform/23.4.0/launch/advanced#pre-and-post-run-scripts) on the Seqera Platform"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f003e7d4",
+ "metadata": {},
+ "source": [
+ ":::{dropdown} What steps are executed by the nf-core/scrnaseq pipeline?\n",
+ "\n",
+ "\n",
+ "\n",
+ ":::"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "756e24d7-b2a0-4a10-bf6c-f532e0cc323b",
+ "metadata": {
+ "tags": [
+ "hide-output"
+ ]
+ },
+ "outputs": [],
+ "source": [
+ "!lamin init --storage ./test-nextflow --name test-nextflow"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ecb68cf2-1188-4f8b-a2ab-01c60d5779b8",
+ "metadata": {},
+ "source": [
+ "## Run the pipeline\n",
+ "\n",
+ "Let's download the input data from an S3 bucket."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "258cdd13",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import lamindb as ln\n",
+ "\n",
+ "input_path = ln.UPath(\"s3://lamindb-test/scrnaseq_input\")\n",
+ "input_path.download_to(\"scrnaseq_input\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5a3671cb",
+ "metadata": {},
+ "source": [
+ "And run the `nf-core/scrnaseq` pipeline."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2e98958f",
+ "metadata": {
+ "tags": [
+ "hide-output"
+ ]
+ },
+ "outputs": [],
+ "source": [
+ "# the test profile uses all downloaded input files as an input\n",
+ "!nextflow run nf-core/scrnaseq -r 2.7.1 -profile docker,test -resume --outdir scrnaseq_output"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d1a77f5a",
+ "metadata": {},
+ "source": [
+ ":::{dropdown} What is the full run command for the test profile?\n",
+ "\n",
+ "```\n",
+ "nextflow run nf-core/scrnaseq -r 2.7.1 \\\n",
+ " -profile docker \\\n",
+ " -resume \\\n",
+ " --outdir scrnaseq_output \\\n",
+ " --input 'scrnaseq_input/samplesheet-2-0.csv' \\\n",
+ " --skip_emptydrops \\\n",
+ " --fasta 'https://github.com/nf-core/test-datasets/raw/scrnaseq/reference/GRCm38.p6.genome.chr19.fa' \\\n",
+ " --gtf 'https://github.com/nf-core/test-datasets/raw/scrnaseq/reference/gencode.vM19.annotation.chr19.gtf' \\\n",
+ " --aligner 'star' \\\n",
+ " --protocol '10XV2' \\\n",
+ " --max_cpus 2 \\\n",
+ " --max_memory '6.GB' \\\n",
+ " --max_time '6.h'\n",
+ "```\n",
+ ":::"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4b1c950f",
+ "metadata": {},
+ "source": [
+ "## Run the registration script"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "45ec5b04",
+ "metadata": {},
+ "source": [
+ "After the pipeline has completed, a Python script registers inputs & outputs in LaminDB."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2d0a0e06",
+ "metadata": {},
+ "source": [
+ "```{eval-rst}\n",
+ ".. literalinclude:: register_scrnaseq_run.py\n",
+ " :language: python\n",
+ " :caption: nf-core/scrnaseq run registration\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "285ee7b8",
+ "metadata": {
+ "tags": [
+ "hide-output"
+ ]
+ },
+ "outputs": [],
+ "source": [
+ "!python register_scrnaseq_run.py --input scrnaseq_input --output scrnaseq_output"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9f607150",
+ "metadata": {},
+ "source": [
+ "## Data lineage"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "806408d0",
+ "metadata": {},
+ "source": [
+ "The output data could now be accessed (in a different notebook/script) for analysis with full lineage."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bc17c815",
+ "metadata": {
+ "tags": [
+ "hide-output"
+ ]
+ },
+ "outputs": [],
+ "source": [
+ "matrix_af = ln.Artifact.get(description__icontains=\"filtered count matrix\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "482d78b9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "matrix_af.view_lineage()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "038286a7",
+ "metadata": {},
+ "source": [
+ "## View transforms & runs on the hub\n",
+ "\n",
+ "[](https://lamin.ai/laminlabs/lamindata/transform/cpMwOcY2YJ5G0000/214dWyrIPWHX7pg24EDR)\n",
+ "\n",
+ "| \n", - " | uid | \n", - "storage_id | \n", - "key | \n", - "suffix | \n", - "accessor | \n", - "description | \n", - "version | \n", - "size | \n", - "hash | \n", - "hash_type | \n", - "transform_id | \n", - "run_id | \n", - "initial_version_id | \n", - "visibility | \n", - "key_is_virtual | \n", - "updated_at | \n", - "created_by_id | \n", - "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| id | \n", - "\n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " |
| 14 | \n", - "MMLLAnPZeLjEmJ68bJVB | \n", - "1 | \n", - "rna-seq-results/salmon/salmon.merged.gene_coun... | \n", - ".tsv | \n", - "None | \n", - "None | \n", - "None | \n", - "3838 | \n", - "af4b1GaLBBdvP3L8cdb7CQ | \n", - "md5 | \n", - "1.0 | \n", - "1.0 | \n", - "None | \n", - "0 | \n", - "False | \n", - "2023-12-06 11:23:06.463741+00:00 | \n", - "1 | \n", - "
| 13 | \n", - "qv2isaM9MxLGuRf5KSx8 | \n", - "1 | \n", - "rna-seq-results/multiqc/star_salmon/multiqc_re... | \n", - ".html | \n", - "None | \n", - "None | \n", - "None | \n", - "1573341 | \n", - "EiwB3o2YlPtHZL5vhbrsqg | \n", - "md5 | \n", - "1.0 | \n", - "1.0 | \n", - "None | \n", - "0 | \n", - "False | \n", - "2023-12-06 11:23:06.421060+00:00 | \n", - "1 | \n", - "
| 12 | \n", - "zGCk9P0zDnnZvALlJp9B | \n", - "1 | \n", - "test-datasets/testdata/GSE110004/SRR6357073_1.... | \n", - ".fastq.gz | \n", - "None | \n", - "None | \n", - "None | \n", - "2253497 | \n", - "B3tTegMt6wglWBeFvfHI7g | \n", - "md5 | \n", - "NaN | \n", - "NaN | \n", - "None | \n", - "0 | \n", - "False | \n", - "2023-12-06 11:09:03.772956+00:00 | \n", - "1 | \n", - "
| 11 | \n", - "HCHxiZmUi6jLgy28UyAp | \n", - "1 | \n", - "test-datasets/testdata/GSE110004/SRR6357070_2.... | \n", - ".fastq.gz | \n", - "None | \n", - "None | \n", - "None | \n", - "2232117 | \n", - "0SbMZu0Wn2mvLAfaDdc-Ag | \n", - "md5 | \n", - "NaN | \n", - "NaN | \n", - "None | \n", - "0 | \n", - "False | \n", - "2023-12-06 11:09:03.772636+00:00 | \n", - "1 | \n", - "
| 10 | \n", - "WRF9aNKguK8LxwGYjPZk | \n", - "1 | \n", - "test-datasets/testdata/GSE110004/SRR6357074_1.... | \n", - ".fastq.gz | \n", - "None | \n", - "None | \n", - "None | \n", - "2289662 | \n", - "n2UX-ubh4ChzsZ85nPsKZQ | \n", - "md5 | \n", - "NaN | \n", - "NaN | \n", - "None | \n", - "0 | \n", - "False | \n", - "2023-12-06 11:09:03.772316+00:00 | \n", - "1 | \n", - "
| 9 | \n", - "KvmrKmzMBSc7f6uuewlK | \n", - "1 | \n", - "test-datasets/testdata/GSE110004/SRR6357075_1.... | \n", - ".fastq.gz | \n", - "None | \n", - "None | \n", - "None | \n", - "2207238 | \n", - "ak4u6Oo56bYwTqkEtWcJYA | \n", - "md5 | \n", - "NaN | \n", - "NaN | \n", - "None | \n", - "0 | \n", - "False | \n", - "2023-12-06 11:09:03.771984+00:00 | \n", - "1 | \n", - "
| 8 | \n", - "G6nehRAbplwCHx5p6iLI | \n", - "1 | \n", - "test-datasets/testdata/GSE110004/SRR6357076_2.... | \n", - ".fastq.gz | \n", - "None | \n", - "None | \n", - "None | \n", - "2250281 | \n", - "ADd-doz9AF-4VW0X_S7iOQ | \n", - "md5 | \n", - "NaN | \n", - "NaN | \n", - "None | \n", - "0 | \n", - "False | \n", - "2023-12-06 11:09:03.771627+00:00 | \n", - "1 | \n", - "
| \n", - " | uid | \n", - "transform_id | \n", - "run_at | \n", - "created_by_id | \n", - "report_id | \n", - "is_consecutive | \n", - "reference | \n", - "reference_type | \n", - "
|---|---|---|---|---|---|---|---|---|
| id | \n", - "\n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " |
| 1 | \n", - "Bcs8m6VxzFM42qhCD4VF | \n", - "1 | \n", - "2023-12-06 11:09:03.865886+00:00 | \n", - "1 | \n", - "None | \n", - "None | \n", - "b187d9dc-7074-4da5-91f1-d5abe704614b\\n76fbcdf1... | \n", - "nextflow_id | \n", - "
| \n", - " | uid | \n", - "root | \n", - "type | \n", - "region | \n", - "updated_at | \n", - "created_by_id | \n", - "
|---|---|---|---|---|---|---|
| id | \n", - "\n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " |
| 1 | \n", - "pfjurGAu | \n", - "/home/zeth/PycharmProjects/nextflow-lamin-usec... | \n", - "local | \n", - "None | \n", - "2023-12-06 11:09:01.212047+00:00 | \n", - "1 | \n", - "
| \n", - " | uid | \n", - "name | \n", - "short_name | \n", - "version | \n", - "type | \n", - "latest_report_id | \n", - "source_file_id | \n", - "reference | \n", - "reference_type | \n", - "initial_version_id | \n", - "updated_at | \n", - "created_by_id | \n", - "
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| id | \n", - "\n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " |
| 1 | \n", - "A3xe64gBMC18zK | \n", - "nf-core rnaseq | \n", - "None | \n", - "3.11.2 | \n", - "pipeline | \n", - "None | \n", - "None | \n", - "https://github.com/laminlabs/nextflow-lamin-us... | \n", - "None | \n", - "None | \n", - "2023-12-06 11:09:03.810901+00:00 | \n", - "1 | \n", - "
| \n", - " | uid | \n", - "handle | \n", - "name | \n", - "updated_at | \n", - "
|---|---|---|---|---|
| id | \n", - "\n", - " | \n", - " | \n", - " | \n", - " |
| 1 | \n", - "I5DPZdiE | \n", - "zethson | \n", - "None | \n", - "2023-12-06 11:09:01.191310+00:00 | \n", - "