Method to detect specimens in H&E images (#1044)

* mvp for function; without testgs

* added option to retain holes

* refactor + 1 test

* added missing import

* renamed test so that a plot would be generated

* added img from runner; cross-os-data-cache

* improved docstring

* added data download script to correct location

* updated hatch commands

* modified coverage combine

* removed superflous combine step

* first download data, then run tests

* attempt to simplify

* aligned testing

* updated toml

* aligned __init__ files

* no uv cache for data download

* removed download step that'd never get hit

* simplify

* parallel

* speed up tests

---------

Co-authored-by: Phil Schaf <[email protected]>

Author: timtreis

.github/workflows/test.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   schedule:
-    - cron: 00 00 * * 1 # every Monday at 00:00
+    - cron: "00 00 * * 1" # every Monday at 00:00
   push:
     branches:
       - main
@@ -14,11 +14,42 @@ env:
   FORCE_COLOR: "1"
   MPLBACKEND: agg
   # It's impossible to ignore SyntaxWarnings for a single module,
-  # so because leidenalg 0.10.0 has them, we pre-compile things: https://github.com/vtraag/leidenalg/issues/173
+  # so because leidenalg 0.10.0 has them, we pre-compile things:
+  # https://github.com/vtraag/leidenalg/issues/173
   UV_COMPILE_BYTECODE: "1"
+  COVERAGE_FILE: ${{ github.workspace }}/.coverage
 
 jobs:
+  ensure-data-is-cached:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          filter: blob:none
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: false
+          python-version: "3.13"
+          cache-dependency-glob: pyproject.toml
+
+      - name: Restore data cache
+        id: data-cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/squidpy/*.h5ad
+            ~/.cache/squidpy/*.zarr
+          key: data-${{ hashFiles('**/download_data.py') }}
+          enableCrossOsArchive: true
+
+      - name: Download datasets
+        if: steps.data-cache.outputs.cache-hit != 'true'
+        run: uvx hatch run data:download
+
   test:
+    needs: [ensure-data-is-cached]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -34,16 +65,15 @@ jobs:
             os: ubuntu-latest
             python: "3.13"
             test-type: "coverage"
+            pytest-addopts: "-v --color=yes -n auto"
           - name: hatch-test.py3.13-pre
             os: macos-latest
             python: "3.13"
-    env: # environment variable for use in codecov's env_vars tagging
+    env:
       ENV_NAME: ${{ matrix.name }}
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          filter: blob:none
+        with: { fetch-depth: 0, filter: blob:none }
 
       - uses: astral-sh/setup-uv@v6
         with:
@@ -60,12 +90,9 @@ jobs:
         with:
           path: |
             ~/.cache/squidpy/*.h5ad
+            ~/.cache/squidpy/*.zarr
           key: data-${{ hashFiles('**/download_data.py') }}
-
-      - name: Download datasets
-        if: steps.data-cache.outputs.cache-hit != 'true'
-        run: |
-          uvx hatch run ${{ matrix.name }}:download
+          enableCrossOsArchive: true
 
       - name: System dependencies (Linux)
         if: matrix.os == 'ubuntu-latest'
@@ -81,18 +108,22 @@ jobs:
         if: matrix.os == 'macos-latest'
         run: brew install automake
 
-      - name: Install dependencies
+      - name: Create env
         run: uvx hatch -v env create ${{ matrix.name }}
 
       - name: Run tests
         if: matrix.test-type == null
-        run: uvx hatch run ${{ matrix.name }}:run
+        run: uvx hatch run ${{ matrix.name }}:run -n logical
+
       - name: Run tests (coverage)
         if: matrix.test-type == 'coverage'
+        env:
+          PYTEST_ADDOPTS: ${{ matrix.pytest-addopts }}
         run: |
-          uvx hatch run ${{ matrix.name }}:run-cov
-          uvx hatch run ${{ matrix.name }}:coverage combine
-          uvx hatch run ${{ matrix.name }}:coverage xml
+          uvx hatch run ${{ matrix.name }}:cov-erase
+          uvx hatch run ${{ matrix.name }}:run-cov -n logical
+          uvx hatch run ${{ matrix.name }}:cov-combine
+          uvx hatch run ${{ matrix.name }}:cov-report
 
       - name: Archive figures generated during testing
         if: always()
@@ -111,8 +142,7 @@ jobs:
 
   check:
     if: always()
-    needs:
-      - test
+    needs: [test]
     runs-on: ubuntu-latest
     steps:
       - uses: re-actors/alls-green@release/v1

hatch.toml

@@ -2,24 +2,11 @@
 installer = "uv"
 features = ["dev"]
 
-[envs.coverage]
-extra-dependencies = [
-    "coverage[toml]",
-    "diff_cover",
-]
-
-[envs.coverage.scripts]
-clean = "coverage erase"
-report = "coverage report --omit='tox/*'"
-xml = "coverage xml --omit='tox/*' -o coverage.xml"
-diff = "diff-cover --compare-branch origin/main coverage.xml"
-
 [envs.docs]
 features = ["docs"]
 extra-dependencies = [
     "setuptools",
 ]
-
 [envs.docs.scripts]
 build = "make -C docs html {args}"
 clean = "make -C docs clean"
@@ -32,14 +19,20 @@ download = "python ./.scripts/ci/download_data.py {args}"
 
 [envs.hatch-test]
 features = ["test"]
-extra-dependencies = [
-    "pytest",
-    "pytest-xdist", 
-    "pytest-cov",
-    "pytest-mock",
-    "pytest-timeout",
+extra-dependencies = ["diff-cover"]
+[envs.hatch-test.scripts]
+# defaults (only `cov-report` is overridden)
+run = "pytest{env:HATCH_TEST_ARGS:} -p no:cov {args}"
+run-cov = "coverage run -m pytest{env:HATCH_TEST_ARGS:} -p no:cov {args}"
+cov-combine = ["coverage combine"]
+cov-report = [
+    "coverage report",
+    "coverage xml -o coverage.xml",
+    "diff-cover --compare-branch origin/main coverage.xml",
 ]
-
+# extra commands
+cov-erase = "coverage erase"
+download = "python ./.scripts/ci/download_data.py {args}"
 
 
 [[envs.hatch-test.matrix]]
@@ -72,4 +65,4 @@ extras = ["docs"]
 [envs.notebooks.scripts]
 
 setup-squidpy-kernel = "python -m ipykernel install --user --name=squidpy"
-run-notebooks = "python ./.scripts/ci/run_notebooks.py docs/notebooks"
+run-notebooks = "python ./.scripts/ci/run_notebooks.py docs/notebooks"

pyproject.toml

@@ -37,16 +37,20 @@ authors = [
     {name = "Giovanni Palla"},
     {name = "Michal Klein"},
     {name = "Hannah Spitzer"},
+    {name = "Tim Treis"},
+    {name = "Laurens Lehner"},
+    {name = "Selman Ozleyen"},
 ]
 maintainers = [
-    {name = "Giovanni Palla", email = "[email protected]"},
-    {name = "Michal Klein", email = "[email protected]"},
-    {name = "Tim Treis", email = "[email protected]"}
+    {name = "Tim Treis", email = "[email protected]"},
+    {name = "Selman Ozleyen", email = "[email protected]"}
 ]
 
 dependencies = [
    "aiohttp>=3.8.1",
    "anndata>=0.9",
+   "spatialdata>=0.2.5",
+   "spatialdata-plot",
    "cycler>=0.11.0",
    "dask-image>=0.5.0",
    "dask[array]>=2021.02.0,<=2024.11.2",
@@ -61,7 +65,7 @@ dependencies = [
    "pandas>=2.1.0",
    "Pillow>=8.0.0",
    "scanpy>=1.9.3",
-   "scikit-image>=0.20",
+   "scikit-image>=0.25",
    # due to https://github.com/scikit-image/scikit-image/issues/6850 breaks rescale ufunc
    "scikit-learn>=0.24.0",
    "statsmodels>=0.12.0",
@@ -70,14 +74,20 @@ dependencies = [
    "tqdm>=4.50.2",
    "validators>=0.18.2",
    "xarray>=2024.10.0",
+   "imagecodecs>=2025.8.2,<2026",
    "zarr>=2.6.1",
-   "spatialdata>=0.2.5",
 ]
 
 [project.optional-dependencies]
 dev = [
     "pre-commit>=3.0.0",
     "hatch>=1.9.0",
+    "jupyterlab",
+    "notebook",
+    "ipykernel",
+    "ipywidgets",
+    "jupytext",
+    "ruff",
 ]
 test = [
     "scanpy[leiden]",
@@ -262,6 +272,7 @@ omit = [
     "*/__init__.py",
     "*/_version.py",
     "squidpy/pl/_interactive/*",
+    "tox/*",
 ]
 
 [tool.coverage.paths]
@@ -282,3 +293,37 @@ show_missing = true
 precision = 2
 skip_empty = true
 sort = "Miss"
+
+[tool.pixi.workspace]
+channels = ["conda-forge"]
+platforms = ["osx-arm64", "linux-64"]
+
+[tool.pixi.dependencies]
+python = ">=3.11"
+
+[tool.pixi.pypi-dependencies]
+squidpy = { path = ".", editable = true }
+
+[tool.pixi.feature.py311.dependencies]
+python = "3.11.*"
+
+[tool.pixi.feature.py313.dependencies]
+python = "3.13.*"
+
+[tool.pixi.environments]
+dev-py311  = { features = ["dev", "test", "py311"], solve-group = "py311" }
+docs-py311 = { features = ["docs", "py311"],        solve-group = "py311" }
+
+default    = { features = ["py313"],                 solve-group = "py313" }
+dev-py313  = { features = ["dev", "test", "py313"],  solve-group = "py313" }
+docs-py313 = { features = ["docs", "py313"],         solve-group = "py313" }
+test-py313 = { features = ["test", "py313"],         solve-group = "py313" }
+
+[tool.pixi.tasks]
+lab = "jupyter lab"
+kernel-install = "python -m ipykernel install --user --name pixi-dev --display-name \"squidpy (dev)\""
+test = "pytest -v --color=yes --tb=short --durations=10"
+lint = "ruff check ."
+format = "ruff format ."
+pre-commit-install = "pre-commit install"
+pre-commit = "pre-commit run"

src/squidpy/__init__.py

@@ -3,7 +3,7 @@
 from importlib import metadata
 from importlib.metadata import PackageMetadata
 
-from squidpy import datasets, gr, im, pl, read, tl
+from squidpy import datasets, experimental, gr, im, pl, read, tl
 
 try:
     md: PackageMetadata = metadata.metadata(__name__)
@@ -14,3 +14,5 @@
     md = None  # type: ignore[assignment]
 
 del metadata, md
+
+__all__ = ["datasets", "experimental", "gr", "im", "pl", "read", "tl"]

src/squidpy/_utils.py

@@ -16,6 +16,8 @@
 import joblib as jl
 import numba
 import numpy as np
+import spatialdata as sd
+from spatialdata.models import Image2DModel, Labels2DModel
 
 __all__ = ["singledispatchmethod", "Signal", "SigQueue", "NDArray", "NDArrayA"]
 
@@ -347,3 +349,27 @@ def new_func2(*args: Any, **kwargs: Any) -> Any:
 
     else:
         raise TypeError(repr(type(reason)))
+
+
+def _get_scale_factors(
+    element: Image2DModel | Labels2DModel,
+) -> list[float]:
+    """
+    Get the scale factors of an image or labels.
+    """
+    if not hasattr(element, "keys"):
+        return []  # element isn't a datatree -> single scale
+
+    shapes = [_yx_from_shape(element[scale].image.shape) for scale in element.keys()]
+
+    factors: list[float] = [(y0 / y1 + x0 / x1) / 2 for (y0, x0), (y1, x1) in zip(shapes, shapes[1:], strict=False)]
+    return [int(f) for f in factors]
+
+
+def _yx_from_shape(shape: tuple[int, ...]) -> tuple[int, int]:
+    if len(shape) == 2:  # (y, x)
+        return shape[0], shape[1]
+    if len(shape) == 3:  # (c, y, x)
+        return shape[1], shape[2]
+
+    raise ValueError(f"Unsupported shape {shape}. Expected (y, x) or (c, y, x).")

src/squidpy/experimental/__init__.py

@@ -0,0 +1,12 @@
+"""Experimental module for Squidpy.
+
+This module contains experimental features that are still under development.
+These features may change or be removed in future releases.
+"""
+
+from __future__ import annotations
+
+from . import im
+from .im._detect_tissue import detect_tissue
+
+__all__ = ["detect_tissue", "im"]

src/squidpy/experimental/im/__init__.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from ._detect_tissue import (
+    BackgroundDetectionParams,
+    FelzenszwalbParams,
+    detect_tissue,
+)
+
+__all__ = ["detect_tissue", "BackgroundDetectionParams", "FelzenszwalbParams"]