diff --git a/benchmarks/bench_dependency_graph.py b/benchmarks/bench_dependency_graph.py index 372e9d95..026e2155 100644 --- a/benchmarks/bench_dependency_graph.py +++ b/benchmarks/bench_dependency_graph.py @@ -4,16 +4,14 @@ volume chunked into ~1M blocks. Compares daisy (Python) vs daisy (Rust). """ -import time import json +import time # --- Daisy --- from daisy import BlockwiseDependencyGraph as DaisyGraph from daisy import Roi as DaisyRoi # --- Daisy --- -from daisy import BlockwiseDependencyGraph as DaisyGraph -from daisy import Roi as DaisyRoi def bench_daisy(total_shape, block_shape, context, read_write_conflict): diff --git a/benchmarks/bench_worker_scaling.py b/benchmarks/bench_worker_scaling.py index 03037f2b..eb7ad947 100644 --- a/benchmarks/bench_worker_scaling.py +++ b/benchmarks/bench_worker_scaling.py @@ -11,12 +11,11 @@ via `daisy.Client`) """ -import time import json +import time # Ensure both packages are importable import daisy -import daisy # --- Daisy benchmark --- diff --git a/build_wrapper.py b/build_wrapper.py index 44547c16..509ce131 100644 --- a/build_wrapper.py +++ b/build_wrapper.py @@ -1,5 +1,6 @@ import shutil import sys + from maturin import * # Forward everything from maturin diff --git a/daisy-py/python/daisy/_daisy.pyi b/daisy-py/python/daisy/_daisy.pyi index 313f2109..dea2e0a7 100644 --- a/daisy-py/python/daisy/_daisy.pyi +++ b/daisy-py/python/daisy/_daisy.pyi @@ -6,9 +6,9 @@ pyright) read this `.pyi` instead of trying to introspect the compiled `.so`. """ -from collections.abc import Iterable, Iterator +from collections.abc import Callable, Iterable, Iterator from pathlib import Path -from typing import Any, Callable +from typing import Any # -- Geometry primitives ------------------------------------------------ @@ -49,7 +49,6 @@ class Roi: def translate(self, by: Coordinate | Iterable[int]) -> Roi: ... def __eq__(self, other: object) -> bool: ... def __hash__(self) -> int: ... - def __repr__(self) -> str: ... # -- Blocks ------------------------------------------------------------- @@ -149,7 +148,7 @@ class Task: fit: str = ..., max_workers: int = ..., max_retries: int = ..., - timeout: float | int | object | None = ..., + timeout: float | object | None = ..., done_marker_path: str | bool | None = ..., requires: dict[str, int] | None = ..., max_worker_restarts: int = ..., @@ -226,7 +225,6 @@ class Pipeline: def __or__(self, other: Task | Pipeline) -> Pipeline: ... def __radd__(self, other: Task | Pipeline) -> Pipeline: ... def __ror__(self, other: Task | Pipeline) -> Pipeline: ... - def __repr__(self) -> str: ... # -- Dependency graph --------------------------------------------------- @@ -290,7 +288,6 @@ class Context: def to_env(self) -> str: ... @staticmethod def from_env() -> Context: ... - def __repr__(self) -> str: ... class SyncClient: def __init__( diff --git a/daisy-py/python/daisy/_runner.py b/daisy-py/python/daisy/_runner.py index dadb899b..2c54c530 100644 --- a/daisy-py/python/daisy/_runner.py +++ b/daisy-py/python/daisy/_runner.py @@ -11,7 +11,6 @@ import daisy._daisy as _rs from daisy import logging as _worker_log - from daisy._progress import _log_resume_summary from daisy._task import ( _to_pipeline, diff --git a/daisy-py/python/daisy/_task.py b/daisy-py/python/daisy/_task.py index 1eaf2320..ffda2d58 100644 --- a/daisy-py/python/daisy/_task.py +++ b/daisy-py/python/daisy/_task.py @@ -10,11 +10,11 @@ `daisy.v1_compat`, not here. """ -from contextlib import contextmanager import copy import inspect import logging import warnings +from contextlib import contextmanager from pathlib import Path import daisy._daisy as _rs diff --git a/daisy-py/python/daisy/_worker_processes.py b/daisy-py/python/daisy/_worker_processes.py index f21440a9..3a3099d6 100644 --- a/daisy-py/python/daisy/_worker_processes.py +++ b/daisy-py/python/daisy/_worker_processes.py @@ -33,8 +33,8 @@ import pickle import struct import subprocess -import tempfile import sys +import tempfile import types #: exit code the worker child uses when it self-terminates because a block diff --git a/daisy-py/python/daisy/logging.py b/daisy-py/python/daisy/logging.py index e714251a..1dccb29f 100644 --- a/daisy-py/python/daisy/logging.py +++ b/daisy-py/python/daisy/logging.py @@ -22,15 +22,14 @@ from __future__ import annotations -import typing import logging as _py_logging import re import sys import threading import traceback as _py_traceback +import typing from pathlib import Path - # --- Public knobs ------------------------------------------------------------ LOG_BASEDIR: Path | None = Path("./daisy_logs") @@ -300,6 +299,7 @@ def __exit__(self, exc_type, exc, tb): # Path of this package, used to strip daisy-internal frames from # tracebacks before rendering. import os as _os + _DAISY_PKG_DIR = _os.path.abspath(_os.path.dirname(__file__)) @@ -410,6 +410,7 @@ def format_traceback(exc_type, exc, tb) -> str: if _TRACEBACK_STYLE == "rich": try: import io + from rich.console import Console from rich.traceback import Traceback tb_obj = Traceback.from_exception( diff --git a/daisy-py/python/daisy/v1_compat.py b/daisy-py/python/daisy/v1_compat.py index 5f991f04..e740dc1d 100644 --- a/daisy-py/python/daisy/v1_compat.py +++ b/daisy-py/python/daisy/v1_compat.py @@ -30,7 +30,7 @@ from daisy import v2 as _v2 from daisy._runner import _run_serial from daisy._task import _record_task_upstream -from daisy.v2 import ( # noqa: F401 — re-exported compat surface +from daisy.v2 import ( BlockStatus, BlockwiseDependencyGraph, Client, @@ -133,6 +133,7 @@ def _install_compat_block_boundary(): wire block so daisy's bookkeeping (auto-SUCCESS, failure reporting) is unaffected.""" from contextlib import contextmanager + from daisy._task import Client _orig_acquire = Client.acquire_block diff --git a/daisy-py/python/daisy/v2.py b/daisy-py/python/daisy/v2.py index 9f5ff69e..8fe3a116 100644 --- a/daisy-py/python/daisy/v2.py +++ b/daisy-py/python/daisy/v2.py @@ -16,7 +16,8 @@ daisy.run_blockwise([task]) """ -from importlib.metadata import PackageNotFoundError, version as _version +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _version import daisy._daisy as _rs from daisy._progress import JsonProgressObserver diff --git a/examples/astronaut_pipeline.py b/examples/astronaut_pipeline.py index e19291dd..c2550ac1 100644 --- a/examples/astronaut_pipeline.py +++ b/examples/astronaut_pipeline.py @@ -25,6 +25,8 @@ import time from pathlib import Path +import daisy.logging as gl +import daisy.v2 as daisy import matplotlib.pyplot as plt import numpy as np from scipy.ndimage import label as scipy_label @@ -32,9 +34,6 @@ from skimage.filters import sobel from skimage.segmentation import watershed -import daisy.v2 as daisy -import daisy.logging as gl - _TMP = Path(tempfile.mkdtemp(prefix="daisy_astronaut_pipeline_")) gl.set_log_basedir(_TMP / "logs") print(f"output paths under: {_TMP}") diff --git a/examples/mws.py b/examples/mws.py index eb3ac4df..59fba287 100644 --- a/examples/mws.py +++ b/examples/mws.py @@ -56,10 +56,11 @@ # %% import tempfile -import threading import time from pathlib import Path +import daisy.logging as gl +import daisy.v2 as daisy import lsd_lite import matplotlib.patches as mpatches import matplotlib.pyplot as plt @@ -67,9 +68,6 @@ import numpy as np import zarr -import daisy.v2 as daisy -import daisy.logging as gl - _TMP = Path(tempfile.mkdtemp(prefix="daisy_mws_")) gl.set_log_basedir(_TMP / "logs") # Tee worker stdout/stderr to both file and terminal so the run-3 diff --git a/examples/stress_test.py b/examples/stress_test.py index 1c3e5403..822f4ad5 100644 --- a/examples/stress_test.py +++ b/examples/stress_test.py @@ -27,8 +27,8 @@ import time from pathlib import Path -import daisy.v2 as daisy import daisy.logging as gl +import daisy.v2 as daisy _TMP = Path(tempfile.mkdtemp(prefix="daisy_stress_")) gl.set_log_basedir(_TMP / "logs") diff --git a/tests/conftest.py b/tests/conftest.py index 033beeb7..ab801a0f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,9 +11,8 @@ is idempotent. """ -import pytest - import daisy.logging as gl +import pytest @pytest.fixture(autouse=True) diff --git a/tests/daisy_compat/process_block.py b/tests/daisy_compat/process_block.py index 0f7fee83..d115430b 100644 --- a/tests/daisy_compat/process_block.py +++ b/tests/daisy_compat/process_block.py @@ -1,9 +1,8 @@ -import daisy as daisy - -import time import random import sys +import time +import daisy as daisy from filelock import FileLock tmp_path = sys.argv[1] diff --git a/tests/daisy_compat/process_block_or_die.py b/tests/daisy_compat/process_block_or_die.py index 23527b46..321dd34e 100644 --- a/tests/daisy_compat/process_block_or_die.py +++ b/tests/daisy_compat/process_block_or_die.py @@ -6,11 +6,11 @@ workers see the marker and process blocks normally. """ -import daisy as daisy - import os import sys +import daisy as daisy + tmp_path = sys.argv[1] marker = os.path.join(tmp_path, "worker_crashed") diff --git a/tests/daisy_compat/test_client.py b/tests/daisy_compat/test_client.py index dc2bb9c8..76b429c2 100644 --- a/tests/daisy_compat/test_client.py +++ b/tests/daisy_compat/test_client.py @@ -1,8 +1,7 @@ import multiprocessing as mp -import pytest - import daisy +import pytest # `daisy.messages` and `daisy.tcp` were daisy 1.x's internal protocol modules # (pickle-over-tornado). daisy v2's wire protocol is bincode over tokio TCP, diff --git a/tests/daisy_compat/test_clients_close.py b/tests/daisy_compat/test_clients_close.py index ef0f87cc..ba531351 100644 --- a/tests/daisy_compat/test_clients_close.py +++ b/tests/daisy_compat/test_clients_close.py @@ -2,11 +2,9 @@ import subprocess import sys -import pytest -from filelock import FileLock - import daisy from daisy.logging import set_log_basedir +from filelock import FileLock logging.basicConfig(level=logging.DEBUG) diff --git a/tests/daisy_compat/test_dead_workers.py b/tests/daisy_compat/test_dead_workers.py index db7f6636..57eb7df7 100644 --- a/tests/daisy_compat/test_dead_workers.py +++ b/tests/daisy_compat/test_dead_workers.py @@ -10,8 +10,6 @@ import subprocess import sys -import pytest - import daisy from daisy.logging import set_log_basedir diff --git a/tests/daisy_compat/test_dependency_graph.py b/tests/daisy_compat/test_dependency_graph.py index 02f6eb3d..d2da42b3 100644 --- a/tests/daisy_compat/test_dependency_graph.py +++ b/tests/daisy_compat/test_dependency_graph.py @@ -1,11 +1,7 @@ -from daisy import BlockwiseDependencyGraph -from daisy import Task -from daisy import Block -from daisy import Roi +import logging import pytest - -import logging +from daisy import Block, BlockwiseDependencyGraph, Roi, Task logger = logging.getLogger(__name__) diff --git a/tests/daisy_compat/test_scheduler.py b/tests/daisy_compat/test_scheduler.py index 86b3c778..f741a2ce 100644 --- a/tests/daisy_compat/test_scheduler.py +++ b/tests/daisy_compat/test_scheduler.py @@ -1,11 +1,7 @@ -from daisy import Scheduler -from daisy import Task -from daisy import Block, BlockStatus -from daisy import Roi +import logging import pytest - -import logging +from daisy import Block, BlockStatus, Roi, Scheduler, Task logger = logging.getLogger(__name__) diff --git a/tests/daisy_compat/test_server.py b/tests/daisy_compat/test_server.py index 7811e86c..bf7ca93b 100644 --- a/tests/daisy_compat/test_server.py +++ b/tests/daisy_compat/test_server.py @@ -1,8 +1,7 @@ import logging -import pytest - import daisy +import pytest logging.basicConfig(level=logging.DEBUG) diff --git a/tests/test_block_compat.py b/tests/test_block_compat.py index 93c453aa..0de967eb 100644 --- a/tests/test_block_compat.py +++ b/tests/test_block_compat.py @@ -2,11 +2,10 @@ ROIs — v1 parity for code that constructs blocks directly, e.g. "process this one ROI without a blockwise job" helpers.""" +import daisy import funlib.geometry as fg import pytest -import daisy - def test_block_constructs_from_funlib_rois(): b = daisy.Block(fg.Roi((0,), (40,)), fg.Roi((0,), (10,)), fg.Roi((5,), (5,))) diff --git a/tests/test_check_function_warning.py b/tests/test_check_function_warning.py index 8d8eac94..722ca9d8 100644 --- a/tests/test_check_function_warning.py +++ b/tests/test_check_function_warning.py @@ -2,9 +2,8 @@ import warnings -import pytest - import daisy +import pytest def _mk(**kw): diff --git a/tests/test_client_graceful.py b/tests/test_client_graceful.py index 8917da25..2677a378 100644 --- a/tests/test_client_graceful.py +++ b/tests/test_client_graceful.py @@ -12,9 +12,8 @@ import subprocess import sys -import pytest - import daisy +import pytest def _dead_port() -> int: diff --git a/tests/test_context_passing.py b/tests/test_context_passing.py index 95cd9bc5..1ed141cd 100644 --- a/tests/test_context_passing.py +++ b/tests/test_context_passing.py @@ -12,13 +12,10 @@ """ import os -import subprocess -import sys import time -import pytest - import daisy +import pytest def _mk_task(spawn_fn, n_blocks=16, max_workers=8, **kw): diff --git a/tests/test_default_timeout.py b/tests/test_default_timeout.py index e7d8aa6f..fb4279f6 100644 --- a/tests/test_default_timeout.py +++ b/tests/test_default_timeout.py @@ -4,9 +4,8 @@ import os import time -import pytest - import daisy +import pytest def _task(fn, **kw): diff --git a/tests/test_dependency_graph.py b/tests/test_dependency_graph.py index a82f8c29..fcbf437c 100644 --- a/tests/test_dependency_graph.py +++ b/tests/test_dependency_graph.py @@ -1,7 +1,7 @@ """Port of daisy's test_dependency_graph.py — 3 tests (6 parameterized variants).""" -from daisy import BlockwiseDependencyGraph, Task, Block, Roi import pytest +from daisy import Block, BlockwiseDependencyGraph, Roi, Task total_roi = Roi([0], [12]) read_roi = Roi([0], [5]) diff --git a/tests/test_done_marker.py b/tests/test_done_marker.py index b478c268..5dd39d45 100644 --- a/tests/test_done_marker.py +++ b/tests/test_done_marker.py @@ -10,7 +10,6 @@ """ import shutil -import tempfile from pathlib import Path import daisy diff --git a/tests/test_error_tracebacks.py b/tests/test_error_tracebacks.py index 1b5ff1d6..a0339f23 100644 --- a/tests/test_error_tracebacks.py +++ b/tests/test_error_tracebacks.py @@ -11,9 +11,8 @@ import io -import pytest - import daisy +import pytest def _inner_helper(block): diff --git a/tests/test_keyboard_interrupt.py b/tests/test_keyboard_interrupt.py index 1fa2cd6b..78fe3bea 100644 --- a/tests/test_keyboard_interrupt.py +++ b/tests/test_keyboard_interrupt.py @@ -16,9 +16,8 @@ import threading import time -import pytest - import daisy +import pytest def _slow_block(block): diff --git a/tests/test_pickling_native_types.py b/tests/test_pickling_native_types.py index 6174dd95..ae108963 100644 --- a/tests/test_pickling_native_types.py +++ b/tests/test_pickling_native_types.py @@ -8,9 +8,8 @@ import pickle -import pytest - import daisy +import pytest @pytest.mark.parametrize("cls_name", [ diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index f5091752..ec9f64ff 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,9 +1,8 @@ """Tests for the Rust-backed Pipeline DSL — `+` (sequential) and `|` (parallel) composition over `Task` and `Pipeline` instances.""" -import pytest - import daisy.v2 as daisy +import pytest from daisy.v2 import Pipeline diff --git a/tests/test_resources.py b/tests/test_resources.py index 60e70977..91f38675 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -14,9 +14,8 @@ import threading import time -import pytest - import daisy +import pytest def _make_task(task_id, *, max_workers, requires=None, blocks=8, hold_ms=20, diff --git a/tests/test_return_states.py b/tests/test_return_states.py index 53a4825a..6a46667a 100644 --- a/tests/test_return_states.py +++ b/tests/test_return_states.py @@ -1,9 +1,8 @@ """run_blockwise(return_states=True): TaskState access through the front door, without touching the lower-level Server class.""" -import pytest - import daisy +import pytest def _task(task_id, fn, total=40, **kw): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 4cf6fc00..e75eb0fe 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -4,8 +4,8 @@ generalized Cantor pairing function. """ -from daisy import Scheduler, Task, Block, BlockStatus, Roi import pytest +from daisy import Block, BlockStatus, Roi, Scheduler, Task def process_block(block): diff --git a/tests/test_serial.py b/tests/test_serial.py index 7b387e3f..a554921c 100644 --- a/tests/test_serial.py +++ b/tests/test_serial.py @@ -8,9 +8,8 @@ attempts in a post-run summary. """ -import pytest - import daisy +import pytest def _task(task_id, process_function): diff --git a/tests/test_server.py b/tests/test_server.py index d57f389f..1e3f5385 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1,6 +1,6 @@ """Port of daisy's test_server.py — serial server execution test.""" -from daisy import Task, Roi, run_blockwise +from daisy import Roi, Task, run_blockwise def process_block(block): diff --git a/tests/test_tcp_client.py b/tests/test_tcp_client.py index 69481d19..7b99fea6 100644 --- a/tests/test_tcp_client.py +++ b/tests/test_tcp_client.py @@ -9,10 +9,8 @@ """ import tempfile -import threading -from daisy import Task, Roi, Block, BlockStatus, Scheduler, run_blockwise -import pytest +from daisy import BlockStatus, Roi, Scheduler, Task, run_blockwise def test_client_acquire_release(): diff --git a/tests/test_types.py b/tests/test_types.py index a9f8c94f..c109eebb 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -1,7 +1,7 @@ """Tests for core types: Roi, Coordinate, Block — covering the API surface used by daisy's test_tcp.py, test_client.py, and general type operations.""" -from daisy import Roi, Coordinate, Block, BlockStatus +from daisy import Block, BlockStatus, Coordinate, Roi def test_roi_basic(): diff --git a/tests/test_worker_process_timeout.py b/tests/test_worker_process_timeout.py index b346cff2..04cd4cf6 100644 --- a/tests/test_worker_process_timeout.py +++ b/tests/test_worker_process_timeout.py @@ -12,9 +12,8 @@ import time -import pytest - import daisy +import pytest @pytest.mark.timeout(30) diff --git a/tests/test_worker_processes.py b/tests/test_worker_processes.py index d6b420b8..78878cd3 100644 --- a/tests/test_worker_processes.py +++ b/tests/test_worker_processes.py @@ -9,9 +9,8 @@ import time -import pytest - import daisy +import pytest def _busy(n): @@ -53,7 +52,7 @@ def test_lambda_gets_real_parallelism(): iters = _calibrate(0.15) def timed(workers): - fn = lambda block: _busy(iters) # noqa: E731 — the lambda IS the point + fn = lambda block: _busy(iters) # best of two: a single sample is noisy when the whole suite's # worker churn loads the machine walls = [] @@ -165,9 +164,9 @@ def test_subprocess_workers_are_the_default(tmp_path): out = str(tmp_path) def record_pid(block): + import os as _os import time as _time from pathlib import Path - import os as _os Path(out, f"pid-{_os.getpid()}-{block.block_id[1]}").touch() _time.sleep(0.05) # keep blocks around long enough for all workers @@ -200,8 +199,8 @@ def test_worker_processes_false_runs_in_server_process(tmp_path): out = str(tmp_path) def record_pid(block): - from pathlib import Path import os as _os + from pathlib import Path Path(out, f"pid-{_os.getpid()}-{block.block_id[1]}").touch() diff --git a/tests/test_worker_restarts.py b/tests/test_worker_restarts.py index 7dd840b0..ca41bbb2 100644 --- a/tests/test_worker_restarts.py +++ b/tests/test_worker_restarts.py @@ -13,9 +13,8 @@ import time -import pytest - import daisy +import pytest def _always_crashing_worker(): @@ -163,8 +162,9 @@ def _block_holding_worker(): """0-arg worker that acquires blocks one at a time and crashes while holding one. Reproduces the race between the in-flight block's release message and the worker-thread exit signal.""" - import daisy as g import time + + import daisy as g client = g.Client() while True: with client.acquire_block() as block: @@ -363,7 +363,6 @@ def test_abandonment_raises_from_run_blockwise_with_cause(): message must carry the task id, restart accounting, and the original worker error. `Server.run_blockwise` (tested above) stays non-raising for introspection.""" - import pytest def crash(block): raise ValueError("broken import on node xyz") diff --git a/tests/test_worker_serialization.py b/tests/test_worker_serialization.py index 0e6b8f07..698f7b08 100644 --- a/tests/test_worker_serialization.py +++ b/tests/test_worker_serialization.py @@ -26,15 +26,14 @@ import threading import types -import pytest - import daisy +import pytest dill = pytest.importorskip( "dill", reason="subprocess-worker payloads use dill when available" ) -from daisy._worker_processes import _serialize, read_payload # noqa: E402 +from daisy._worker_processes import _serialize, read_payload def _roundtrip(obj):