Merge branch 'main' into fc_cipy

.github/workflows/spatial_deconvolution-ci.yml

@@ -0,0 +1,102 @@
+name: Run tutorials (spatial deconvolution)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  debug: 'true'
+
+jobs:
+  spatial_deconvolution:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"] # , "macos-latest", "windows-latest"
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          auto-activate-base: true
+          auto-update-conda: true
+          channels: conda-forge
+          channel-priority: strict
+          activate-environment: pipeline_env
+          environment-file: pipeline_env.yaml
+
+      - name: Install Panpipes
+        shell: bash -el {0}
+        run: |
+          pip install .[spatial]
+          conda list
+
+      - name: Conda info
+        if: env.debug == 'true'
+        shell: bash -el {0}
+        run: conda info
+
+      - name: Conda list
+        if: env.debug == 'true'
+        shell: pwsh
+        run: conda list
+
+
+      - name: Preparing the data
+        run: |
+          mkdir deconvolution deconvolution/data deconvolution/data/spatial_data
+          cd deconvolution/data 
+          curl -L -o Human_Heart_reference.h5mu https://figshare.com/ndownloader/files/44969677
+          cd spatial_data
+          curl -L -o Human_Heart.h5mu https://figshare.com/ndownloader/files/44969488
+
+
+      # Note: we run the following to test that the commands works
+      - name: Preparing the configuration file
+        shell: bash -el {0}
+        run: |
+          cd deconvolution
+          panpipes deconvolution_spatial config
+
+      - name: Edit the submission file
+        run: |
+          cd deconvolution
+          curl -o pipeline.yml https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/main/docs/deconvolution/pipeline.yml
+
+      - name: Replace template contents in configuration file
+        run: |
+          cd deconvolution
+          sed -i 's+use_gpu:+use_gpu: False+g' pipeline.yml
+          sed -i 's+max_epochs: 400+max_epochs: 50+g' pipeline.yml
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree deconvolution
+
+      - name: Review pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd deconvolution
+          panpipes deconvolution_spatial show full --local
+
+      - name: Run pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd deconvolution
+          panpipes deconvolution_spatial make full --local
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree deconvolution

.github/workflows/spatial_ingestion_merfish-ci.yml

@@ -0,0 +1,103 @@
+name: Run tutorials (spatial ingest merfish)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  debug: 'true'
+
+jobs:
+  spatial_ingest_merfish:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"] # , "macos-latest", "windows-latest"
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          auto-activate-base: true
+          auto-update-conda: true
+          channels: conda-forge
+          channel-priority: strict
+          activate-environment: pipeline_env
+          environment-file: pipeline_env.yaml
+
+      - name: Install Panpipes
+        shell: bash -el {0}
+        run: |
+          pip install .[spatial]
+          conda list
+
+      - name: Conda info
+        if: env.debug == 'true'
+        shell: bash -el {0}
+        run: conda info
+
+      - name: Conda list
+        if: env.debug == 'true'
+        shell: pwsh
+        run: conda list
+
+
+      - name: Preparing the data
+        run: |
+          mkdir spatial spatial/ingestion_merfish spatial/ingestion_merfish/data
+          cd spatial/ingestion_merfish/data 
+          curl -L -o datasets_mouse_brain_map_BrainReceptorShowcase_Slice1_Replicate1_cell_by_gene_S1R1.csv https://figshare.com/ndownloader/files/45028624
+          curl -L -o datasets_mouse_brain_map_BrainReceptorShowcase_Slice1_Replicate1_cell_metadata_S1R1.csv https://figshare.com/ndownloader/files/45028621
+          mkdir images
+          cd images 
+          curl -L -o datasets_mouse_brain_map_BrainReceptorShowcase_Slice1_Replicate1_images_micron_to_mosaic_pixel_transform.csv https://figshare.com/ndownloader/files/45028645
+        
+
+      # Note: we run the following to test that the commands works
+      - name: Preparing the configuration file
+        shell: bash -el {0}
+        run: |
+          cd spatial/ingestion_merfish
+          panpipes qc_spatial config
+
+      - name: Preparing the submission file
+        run: |
+          cd spatial/ingestion_merfish
+          curl -o sample_file_qc_spatial.txt https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/main/docs/ingesting_merfish_data/sample_file_qc_spatial.txt
+
+      - name: Preparing the yaml file
+        run: |
+          cd spatial/ingestion_merfish
+          curl -o pipeline.yml https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/main/docs/ingesting_merfish_data/pipeline.yml
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree spatial/ingestion_merfish
+
+      - name: Review pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd spatial/ingestion_merfish
+          panpipes qc_spatial show full --local
+
+      - name: Run pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd spatial/ingestion_merfish
+          panpipes qc_spatial make full --local
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree spatial/ingestion_merfish

.github/workflows/spatial_ingestion_visium-ci.yml

@@ -0,0 +1,106 @@
+name: Run tutorials (spatial ingest visium)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  debug: 'true'
+
+jobs:
+  spatial_ingest_visium:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"] # , "macos-latest", "windows-latest"
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          auto-activate-base: true
+          auto-update-conda: true
+          channels: conda-forge
+          channel-priority: strict
+          activate-environment: pipeline_env
+          environment-file: pipeline_env.yaml
+
+      - name: Install Panpipes
+        shell: bash -el {0}
+        run: |
+          pip install .[spatial]
+          conda list
+
+      - name: Conda info
+        if: env.debug == 'true'
+        shell: bash -el {0}
+        run: conda info
+
+      - name: Conda list
+        if: env.debug == 'true'
+        shell: pwsh
+        run: conda list
+
+
+      - name: Preparing the data
+        run: |
+          mkdir spatial spatial/ingestion spatial/ingestion/data
+          cd spatial/ingestion/data 
+          mkdir V1_Human_Heart V1_Human_Lymph_Node
+          cd V1_Human_Heart
+          curl -O https://cf.10xgenomics.com/samples/spatial-exp/1.0.0/V1_Human_Heart/V1_Human_Heart_filtered_feature_bc_matrix.h5
+          curl -O https://cf.10xgenomics.com/samples/spatial-exp/1.0.0/V1_Human_Heart/V1_Human_Heart_spatial.tar.gz
+          tar -xf V1_Human_Heart_spatial.tar.gz
+          cd ../V1_Human_Lymph_Node
+          curl -O https://cf.10xgenomics.com/samples/spatial-exp/1.0.0/V1_Human_Lymph_Node/V1_Human_Lymph_Node_filtered_feature_bc_matrix.h5
+          curl -O https://cf.10xgenomics.com/samples/spatial-exp/1.0.0/V1_Human_Lymph_Node/V1_Human_Lymph_Node_spatial.tar.gz
+          tar -xf V1_Human_Lymph_Node_spatial.tar.gz
+
+      # Note: we run the following to test that the commands works
+      - name: Preparing the configuration file
+        shell: bash -el {0}
+        run: |
+          cd spatial/ingestion
+          panpipes qc_spatial config
+
+      - name: Preparing the submission file
+        run: |
+          cd spatial/ingestion
+          curl -o sample_file_qc_spatial.txt https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/main/docs/ingesting_visium_data/sample_file_qc_spatial.txt
+
+      - name: Preparing the yaml file
+        run: |
+          cd spatial/ingestion
+          curl -o pipeline.yml https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/main/docs/ingesting_visium_data/pipeline.yml
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree spatial/ingestion
+
+      - name: Review pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd spatial/ingestion
+          panpipes qc_spatial show full --local
+
+      - name: Run pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd spatial/ingestion
+          panpipes qc_spatial make full --local
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree spatial/ingestion

CHANGELOG.md

@@ -10,9 +10,13 @@
 ### fixed
 
 - fixed error in `vis`
-    - error occurred when only wanting to plot continuous or categorical variables (or neither), not both
+  - error occurred when only wanting to plot continuous or categorical variables (or neither), not both
+- fixed error in `refmap`
+  - high threads was not recognised, now fixed.
 
 ### dependencies
+- All the dependencies have been updated. 
+  - Python>=3.10 required
 
 ## v0.4.1
 

README.md

@@ -9,7 +9,7 @@ Panpipes allows reliable and customisable analysis and evaluation of individual
 
 **See our [documentation](https://panpipes-pipelines.readthedocs.io/en/latest/) and our [preprint](https://www.biorxiv.org/content/10.1101/2023.03.11.532085v2)**  
 
-These workflows make use of [cgat-core](https://github.com/cgat-developers/cgat-core):
+These workflows make use of [cgat-core](https://github.com/cgat-developers/cgat-core)
 
 Available workflows:
 
@@ -27,13 +27,26 @@ Available workflows:
 
 See [installation instructions here](https://panpipes-pipelines.readthedocs.io/en/latest/install.html)
 
+We recommend installing panpipes in a conda environment, we provide a minimal conda config file in `pipeline_env.yaml`
+
+```
+conda env create --file=pipeline_env.yml 
+```
+
+In this environment, you can install nightly version of panpipes, i.e. cloning this repo and installing it from main.
+
+```
+git clone https://github.com/DendrouLab/panpipes.git
+cd panpipes
+pip install -e .
+```
+
+
 Oxford BMRC Rescomp users find additional advice in [docs/installation_rescomp](https://github.com/DendrouLab/panpipes/blob/main/docs/installation_rescomp.md)
 
 ## Releases
 
-`panpipes v0.4.0` is out [now](./CHANGELOG.md)!
-
-The `ingest` workflow now expects different headers for the RNA and Protein modalities.
+Since `panpipes v0.4.0` ,the `ingest` workflow expects different headers for the RNA and Protein modalities from the previous releases.
 Check the example [submission file](https://github.com/DendrouLab/panpipes/blob/main/docs/usage/sample_file_qc_mm.md) and the [documentation](https://panpipes-pipelines.readthedocs.io/en/latest/usage/setup_for_qc_mm.html) for more detailed instructions.
 
 ## Citation
@@ -45,4 +58,4 @@ bioRxiv 2023.03.11.532085; doi: https://doi.org/10.1101/2023.03.11.532085](https
 ## Contributors
 
 Created and Maintained by Charlotte Rich-Griffin and Fabiola Curion.
-Additional contributors: Sarah Ouologuem, Devika Agarwal, Lilly May, Kevin Rue-Albrecht, Lukas Heumos.
+Additional contributors: Sarah Ouologuem, Devika Agarwal, Lilly May, Kevin Rue-Albrecht, Giulia Garcia, Lukas Heumos.

docs/install.md

@@ -16,7 +16,7 @@ conda config --add channels conda-forge
 conda config --set channel_priority strict
 # you should remove the strict priority afterwards!
 conda search r-base
-conda create --name pipeline_env python=3.9 r-base=4.3.0
+conda create --name pipeline_env python=3.10 r-base=4.3.0
 ```
 
 now we activate the environment
@@ -28,7 +28,7 @@ conda activate pipeline_env
 Let's first install the R packages
 
 ```bash
-conda install -c conda-forge r-tidyverse r-optparse r-ggforce r-ggraph r-xtable r-hdf5r r-clustree
+conda install -c conda-forge r-tidyverse r-optparse r-ggforce r-ggraph r-xtable r-hdf5r r-clustree r-cowplot
 ```
 
 Then we can install panpipes: