Merge branch 'main' into dask-sparse-mean-var

.azure-pipelines.yml

@@ -1,5 +1,5 @@
 trigger:
-- master
+- main
 - "*.*.x"
 
 variables:
@@ -52,21 +52,21 @@ jobs:
 
   - script: |
       python -m pip install --upgrade pip
-      pip install wheel coverage
+      pip install wheel
       pip install .[dev,$(TEST_EXTRA)]
     displayName: 'Install dependencies'
     condition: eq(variables['DEPENDENCIES_VERSION'], 'latest')
 
   - script: |
       python -m pip install --pre --upgrade pip
-      pip install --pre wheel coverage
+      pip install --pre wheel
       pip install --pre .[dev,$(TEST_EXTRA)]
       pip install -v "anndata[dev,test] @ git+https://github.com/scverse/anndata"
     displayName: 'Install dependencies release candidates'
     condition: eq(variables['DEPENDENCIES_VERSION'], 'pre-release')
 
   - script: |
-      python -m pip install pip wheel tomli packaging pytest-cov
+      python -m pip install pip wheel tomli packaging
       pip install `python3 ci/scripts/min-deps.py pyproject.toml --extra dev test`
       pip install --no-deps .
     displayName: 'Install dependencies minimum version'
@@ -81,8 +81,7 @@ jobs:
     condition: eq(variables['TEST_TYPE'], 'standard')
 
   - script: |
-      coverage run -m pytest
-      coverage xml
+      pytest --cov --cov-report=xml --cov-context=test
     displayName: 'PyTest (coverage)'
     condition: eq(variables['TEST_TYPE'], 'coverage')
 

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -14,7 +14,7 @@ body:
       required: true
     - label: I have confirmed this bug exists on the latest version of scanpy.
       required: true
-    - label: (optional) I have confirmed this bug exists on the master branch of scanpy.
+    - label: (optional) I have confirmed this bug exists on the main branch of scanpy.
       required: false
 - type: markdown
   attributes:

.github/ISSUE_TEMPLATE/config.yml

@@ -1,4 +1,4 @@
-blank_issues_enabled: false
+blank_issues_enabled: true
 contact_links:
   - name: Scanpy Community Forum
     url: https://discourse.scverse.org/

.github/workflows/check-pr.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - master
     types:
       # milestone changes
       - milestoned

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.2.1
+    rev: v0.2.2
     hooks:
     -   id: ruff
         types_or: [python, pyi, jupyter]
@@ -26,7 +26,7 @@ repos:
     -   id: check-merge-conflict
     -   id: detect-private-key
     -   id: no-commit-to-branch
-        args: ["--branch=master", "--branch=main"]
+        args: ["--branch=main"]
 
 ci:
     autofix_prs: false

README.md

@@ -3,7 +3,7 @@
 [![Downloads](https://static.pepy.tech/badge/scanpy)](https://pepy.tech/project/scanpy)
 [![Conda](https://img.shields.io/conda/dn/conda-forge/scanpy?logo=Anaconda)](https://anaconda.org/conda-forge/scanpy)
 [![Docs](https://readthedocs.com/projects/icb-scanpy/badge/?version=latest)](https://scanpy.readthedocs.io)
-[![Build Status](https://dev.azure.com/scverse/scanpy/_apis/build/status/theislab.scanpy?branchName=master)](https://dev.azure.com/scverse/scanpy/_build)
+[![Build Status](https://dev.azure.com/scverse/scanpy/_apis/build/status/theislab.scanpy?branchName=main)](https://dev.azure.com/scverse/scanpy/_build)
 [![Discourse topics](https://img.shields.io/discourse/posts?color=yellow&logo=discourse&server=https%3A%2F%2Fdiscourse.scverse.org)](https://discourse.scverse.org/)
 [![Chat](https://img.shields.io/badge/zulip-join_chat-%2367b08f.svg)](https://scverse.zulipchat.com)
 [![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](https://numfocus.org/)

docs/api/get.md

@@ -19,5 +19,6 @@ useful formats.
    get.obs_df
    get.var_df
    get.rank_genes_groups_df
+   get.aggregate
 
 ```

docs/dev/code.md

@@ -20,5 +20,5 @@ We ignore a couple of flake8 checks which are documented in the .flake8 file in
 To learn how to ignore checks per line please read
 [flake8 violations](https://flake8.pycqa.org/en/latest/user/violations.html).
 Additionally, we use Scanpy’s
-[EditorConfig](https://github.com/scverse/scanpy/blob/master/.editorconfig),
+[EditorConfig](https://github.com/scverse/scanpy/blob/main/.editorconfig),
 so using an editor/IDE with support for both is helpful.

docs/dev/getting-set-up.md

@@ -21,7 +21,7 @@ This is very straight forward if you're using [GitHub's CLI](https://cli.github.
 $ gh repo fork scverse/scanpy --clone --remote
 ```
 
-This will fork the repo to your github account, create a clone of the repo on your current machine, add our repository as a remote, and set the `master` development branch to track our repository.
+This will fork the repo to your github account, create a clone of the repo on your current machine, add our repository as a remote, and set the `main` development branch to track our repository.
 
 To do this manually, first make a fork of the repository by clicking the "fork" button on our main github package. Then, on your machine, run:
 
@@ -32,7 +32,7 @@ git clone https://github.com/{your-username}/scanpy.git
 cd scanpy
 # Add our repository as a remote
 git remote add upstream https://github.com/scverse/scanpy.git
-# git branch --set-upstream-to "upstream/master"
+# git branch --set-upstream-to "upstream/main"
 ```
 
 ### `pre-commit`
@@ -61,11 +61,11 @@ If you choose not to run the hooks on each commit, you can run them manually wit
 ### Creating a branch for your feature
 
 All development should occur in branches dedicated to the particular work being done.
-Additionally, unless you are a maintainer, all changes should be directed at the `master` branch.
+Additionally, unless you are a maintainer, all changes should be directed at the `main` branch.
 You can create a branch with:
 
 ```shell
-git checkout master                 # Starting from the master branch
+git checkout main                   # Starting from the main branch
 git pull                            # Syncing with the repo
 git checkout -b {your-branch-name}  # Making and changing to the new branch
 ```

docs/dev/release.md

@@ -67,4 +67,4 @@ If you want to replicate the process more exactly, make sure you are careful,
 and create a version tag before building (make sure you delete it after uploading to TestPyPI!).
 
 [hatch-build]: https://hatch.pypa.io/latest/config/build/
-[publish workflow]: https://github.com/scverse/scanpy/tree/master/.github/workflows/publish.yml
+[publish workflow]: https://github.com/scverse/scanpy/tree/main/.github/workflows/publish.yml

docs/dev/testing.md

@@ -21,7 +21,7 @@ It can take a while to run the whole test suite. There are a few ways to cut dow
 
 ## Writing tests
 
-You can refer to the [existing test suite](https://github.com/scverse/scanpy/tree/master/scanpy/tests) for examples.
+You can refer to the [existing test suite](https://github.com/scverse/scanpy/tree/main/scanpy/tests) for examples.
 If you haven't written tests before, Software Carpentry has an [in-depth guide](https://katyhuff.github.io/2016-07-11-scipy/testing/01-basics.html) on the topic.
 
 We highly recommend using [Test Driven Development](https://en.wikipedia.org/wiki/Test-driven_development) when contributing code.

docs/extensions/git_ref.py

@@ -32,7 +32,7 @@ def get() -> str | None:
         try:
             git_ref = git("rev-parse", "HEAD")
         except Exception:
-            git_ref = "master"
+            git_ref = "main"
     return git_ref
 
 

docs/release-notes/1.10.0.md

@@ -1,54 +1,60 @@
-### 1.10.0 {small}`the future`
+### 1.10.0rc2 {small}`2024-02-22`
+
+```{rubric} Bug fixes
+```
+
+* Fix pytest deprecation warning {pr}`2879` {smaller}`P Angerer`
+
+### 1.10.0rc1 {small}`2024-02-22`
 
 ```{rubric} Features
 ```
 
+* {func}`~scanpy.pp.scrublet` and {func}`~scanpy.pp.scrublet_simulate_doublets` were moved from {mod}`scanpy.external.pp` to {mod}`scanpy.pp`. The `scrublet` implementation is now maintained as part of scanpy {pr}`2703` {smaller}`P Angerer`
+* {func}`scanpy.pp.pca`, {func}`scanpy.pp.scale`, {func}`scanpy.pl.embedding`, and {func}`scanpy.experimental.pp.normalize_pearson_residuals_pca` now support a `mask` parameter {pr}`2272` {smaller}`C Bright, T Marcella, & P Angerer`
+* Enhanced dask support for some internal utilities, paving the way for more extensive dask support {pr}`2696` {smaller}`P Angerer`
+* {func}`scanpy.pp.highly_variable_genes` supports dask for the default `seurat` and `cell_ranger` flavors {pr}`2809` {smaller}`P Angerer`
+* New function {func}`scanpy.get.aggregate` which allows grouped aggregations over your data. Useful for pseudobulking! {pr}`2590` {smaller}`Isaac Virshup` {smaller}`Ilan Gold` {smaller}`Jon Bloom`
+* {func}`scanpy.pp.neighbors` now has a `transformer` argument allowing the use of different ANN/ KNN libraries {pr}`2536` {smaller}`P Angerer`
+* {func}`scanpy.experimental.pp.highly_variable_genes` using `flavor='pearson_residuals'` now uses numba for variance computation and is faster {pr}`2612` {smaller}`S Dicks & P Angerer`
+* {func}`scanpy.tl.leiden` now offers `igraph`'s implementation of the leiden algorithm via  via `flavor` when set to `igraph`. `leidenalg`'s implementation is still default, but discouraged.  {pr}`2815` {smaller}`I Gold`
+* {func}`scanpy.pp.highly_variable_genes` has new flavor `seurat_v3_paper` that is in its implementation consistent with the paper description in Stuart et al 2018. {pr}`2792` {smaller}`E Roellin`
 * {func}`scanpy.datasets.blobs` now accepts a `random_state` argument {pr}`2683` {smaller}`E Roellin`
 * {func}`scanpy.pp.pca` and {func}`scanpy.pp.regress_out` now accept a layer argument {pr}`2588` {smaller}`S Dicks`
 * {func}`scanpy.pp.subsample` with `copy=True` can now be called in backed mode {pr}`2624` {smaller}`E Roellin`
-* {func}`scanpy.pp.neighbors` now has a `transformer` argument allowing for more flexibility {pr}`2536` {smaller}`P Angerer`
-* {func}`scanpy.experimental.pp.highly_variable_genes` using `flavor='pearson_residuals'`
-  now uses numba for variance computation {pr}`2612` {smaller}`S Dicks & P Angerer`
 * {func}`scanpy.external.pp.harmony_integrate` now runs with 64 bit floats improving reproducibility {pr}`2655` {smaller}`S Dicks`
-* {func}`~scanpy.pp.scrublet` and {func}`~scanpy.pp.scrublet_simulate_doublets` were moved from {mod}`scanpy.external.pp` to {mod}`scanpy.pp`.
-  The `scrublet` implementation is now maintained as part of scanpy {pr}`2703` {smaller}`P Angerer`
-* Enhanced dask support for some internal utilities, paving the way for more extensive dask support {pr}`2696` {smaller}`P Angerer`
-* {func}`scanpy.pp.pca`, {func}`scanpy.pp.scale`, {func}`scanpy.pl.embedding`, and {func}`scanpy.experimental.pp.normalize_pearson_residuals_pca`
-  now support a `mask` parameter {pr}`2272` {smaller}`C Bright, T Marcella, & P Angerer`
 * {func}`scanpy.tl.rank_genes_groups` no longer warns that it's default was changed from t-test_overestim_var to t-test {pr}`2798` {smaller}`L Heumos`
-* {func}`scanpy.pp.highly_variable_genes` has new flavor `seurat_v3_paper` that is in its implementation consistent with the paper description in Stuart et al 2018. {pr}`2792` {smaller}`E Roellin`
-* {func}`scanpy.pp.highly_variable_genes` supports dask for the default `seurat` and `cell_ranger` flavors {pr}`2809` {smaller}`P Angerer`
-* Auto conversion of strings to collections in `scanpy.pp.calculate_qc_metrics` {pr}`2859` {smaller}`N Teyssier`
+* `scanpy.pp.calculate_qc_metrics` now allows `qc_vars` to be passed as a string {pr}`2859` {smaller}`N Teyssier`
 
 ```{rubric} Docs
 ```
+
+* Re-add search-as-you-type, this time via `readthedocs-sphinx-search` {pr}`2805` {smaller}`P Angerer`
 * Fixed a lot of broken usage examples {pr}`2605` {smaller}`P Angerer`
 * Improved harmonization of return field of `sc.pp` and `sc.tl` functions {pr}`2742` {smaller}`E Roellin`
-* Re-add search-as-you-type, this time via `readthedocs-sphinx-search` {pr}`2805` {smaller}`P Angerer`
 * Improved docs for `percent_top` argument of {func}`~scanpy.pp.calculate_qc_metrics` {pr}`2849` {smaller}`I Virshup`
 
 ```{rubric} Bug fixes
 ```
 
 * Updated {func}`~scanpy.read_visium` such that it can read spaceranger 2.0 files {smaller}`L Lehner`
-* Fix {func}`~scanpy.pp.normalize_total` {pr}`2466` {smaller}`P Angerer`
-* Fix testing package build {pr}`2468` {smaller}`P Angerer`
+* Fix {func}`~scanpy.pp.normalize_total` for dask {pr}`2466` {smaller}`P Angerer`
 * Fix setting `sc.settings.verbosity` in some cases {pr}`2605` {smaller}`P Angerer`
 * Fix all remaining pandas warnings {pr}`2789` {smaller}`P Angerer`
 * Fix some annoying plotting warnings around violin plots {pr}`2844` {smaller}`P Angerer`
 * Scanpy now has a test job which tests against the minumum versions of the dependencies. In the process of implementing this, many bugs associated with using older versions of `pandas`, `anndata`, `numpy`, and `matplotlib` were fixed. {pr}`2816` {smaller}`I Virshup`
+* Fix warnings caused by internal usage of `pandas.DataFrame.stack` with `pandas>=2.1` {pr}`2864`{smaller}`I Virshup`
 
 ```{rubric} Development
 ```
 
 * Scanpy is now tested against python 3.12 {pr}`2863` {smaller}`ivirshup`
-
-```{rubric} Ecosystem
-```
+* Fix testing package build {pr}`2468` {smaller}`P Angerer`
 
 ```{rubric} Deprecations
 ```
 
 * Dropped support for Python 3.8. [More details here](https://numpy.org/neps/nep-0029-deprecation_policy.html). {pr}`2695` {smaller}`P Angerer`
 * Deprecated specifying large numbers of function parameters by position as opposed to by name/keyword in all public APIs.
   e.g. prefer `sc.tl.umap(adata, min_dist=0.1, spread=0.8)` over `sc.tl.umap(adata, 0.1, 0.8)` {pr}`2702` {smaller}`P Angerer`
+* Dropped support for `umap<0.5` for performance reasons. {pr}`2870` {smaller}`P Angerer`

docs/release-notes/1.11.0.md

@@ -0,0 +1,13 @@
+### 1.11.0 {small}`the future`
+
+```{rubric} Features
+```
+
+```{rubric} Docs
+```
+
+```{rubric} Bug fixes
+```
+
+```{rubric} Deprecations
+```

docs/release-notes/index.md

@@ -2,6 +2,11 @@
 
 # Release notes
 
+## Version 1.11
+
+```{include} /release-notes/1.11.0.md
+```
+
 ## Version 1.10
 
 ```{include} /release-notes/1.10.0.md

pyproject.toml

@@ -63,7 +63,7 @@ dependencies = [
     "natsort",
     "joblib",
     "numba>=0.56",
-    "umap-learn>=0.3.10",
+    "umap-learn>=0.5,!=0.5.0",
     "pynndescent>=0.5",
     "packaging>=21.3",
     "session-info",
@@ -87,6 +87,7 @@ test-min = [
     "pytest>=7.4.2",
     "pytest-nunit",
     "pytest-mock",
+    "pytest-cov",
     "profimp",
 ]
 test = [
@@ -159,7 +160,6 @@ addopts = [
     "--import-mode=importlib",
     "--strict-markers",
     "--doctest-modules",
-    "-pscanpy.testing._pytest",
 ]
 testpaths = ["scanpy"]
 norecursedirs = ["scanpy/tests/_images"]

scanpy/__init__.py

@@ -1,6 +1,8 @@
 """Single-Cell Analysis in Python."""
 from __future__ import annotations
 
+import sys
+
 try:  # See https://github.com/maresb/hatch-vcs-footgun-example
     from setuptools_scm import get_version
 
@@ -21,6 +23,11 @@
 
 # the actual API
 # (start with settings as several tools are using it)
+
+from ._settings import Verbosity, settings
+
+set_figure_params = settings.set_figure_params
+
 from anndata import (
     AnnData,
     concat,
@@ -38,15 +45,10 @@
 from . import plotting as pl
 from . import preprocessing as pp
 from . import tools as tl
-from ._settings import Verbosity, settings
 from .neighbors import Neighbors
 from .readwrite import read, read_10x_h5, read_10x_mtx, read_visium, write
 
-set_figure_params = settings.set_figure_params
-
 # has to be done at the end, after everything has been imported
-import sys
-
 sys.modules.update({f"{__name__}.{m}": globals()[m] for m in ["tl", "pp", "pl"]})
 from ._utils import annotate_doc_types