diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..22720dc --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,75 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + tests: + name: ${{ matrix.name }} + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + # Lower support boundary: oldest Python plus the minimum + # supported versions of the core dependencies + - name: Python 3.10 + minimum deps + python-version: "3.10" + pip-args: -c requirements-min.txt + # Forward boundary: newest supported Python plus current + # versions of all dependencies + - name: Python 3.14 + latest deps + python-version: "3.14" + pip-args: "" + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: | + pyproject.toml + requirements-min.txt + + - name: Install package and test dependencies + run: python -m pip install ".[test]" ${{ matrix.pip-args }} + + - name: Run tests + run: python -m pytest --cov + + quality: + name: Code quality + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.14" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install ruff + run: python -m pip install ruff + + - name: Check source + run: ruff check orca diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml deleted file mode 100644 index bc852cd..0000000 --- a/.github/workflows/code-style.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Code style - -# This workflow runs code style checks. - -on: - push: - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Check code style - run: | - pip install pycodestyle - pycodestyle orca --max-line-length 100 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index 4cdafc5..0000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Coverage - -# This workflow generates a coverage report (how much of the codebase is covered by the -# unit tests) and posts headline metrics to the PR thread. - -on: - # push: - pull_request: - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install Orca - run: | - pip install . - pip install flask pygments # for server component tests - - name: Generate coverage report - run: | - pip install pytest coverage - coverage run --source orca --module pytest --verbose - coverage report --show-missing - echo "coverage=$(coverage report | grep '^TOTAL' | grep -oE '[^ ]+$')" >> $GITHUB_ENV - - name: Post comment on PR - uses: unsplash/comment-on-pr@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - msg: "Test coverage is ${{ env.coverage }}" - check_for_duplicate_msg: true diff --git a/.github/workflows/cross-compatibility.yml b/.github/workflows/cross-compatibility.yml deleted file mode 100644 index e54e001..0000000 --- a/.github/workflows/cross-compatibility.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Cross-compatibility - -# This workflow runs the unit tests across a range of Python versions and operating -# systems. - -on: - # push: - pull_request: - workflow_dispatch: - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Orca - run: | - pip install . - pip install flask pygments # for server component tests - - name: Run unit tests - run: | - pip install pytest - pytest -s diff --git a/.github/workflows/installation.yml b/.github/workflows/installation.yml deleted file mode 100644 index 5ea4295..0000000 --- a/.github/workflows/installation.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Installation - -# This workflow tests installation from Pip and Conda across a range of Python versions -# and operating systems. You can run this manually after a new release is posted to -# confirm that it installs smoothly. This workflow also runs periodically in the -# background to catch dependency updates that cause problems. - -on: - # push: - # pull_request: - workflow_dispatch: - schedule: - - cron: '0 3 * * 1' # every Monday at 3am UTC (Sunday evening Calif time) - -jobs: - pip: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - python-version: [3.8, 3.9, '3.10'] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Orca - run: | - pip install orca - - conda: - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash -l {0} # needed for conda persistence - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - python-version: [3.8, 3.9, '3.10'] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: conda-incubator/setup-miniconda@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Orca - run: | - conda install orca --channel conda-forge diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 751c4ab..0000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Unit tests - -# This workflow runs the unit tests in a single generic environment (recent but stable -# Python version on recent but stable Ubuntu). The cross-compatibility.yml workflow runs -# the same tests across multiple platforms. - -on: - push: - # pull_request: - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install Orca - run: | - pip install . - pip install flask pygments # for server component tests - - name: Run unit tests - run: | - pip install pytest - pytest -s diff --git a/HISTORY.rst b/HISTORY.rst index e563999..e8a7c25 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,3 +1,15 @@ +Next release +============ + +* Require Python 3.10 or newer. +* Support pandas 2 and 3, NumPy 2, and current PyTables, with minimum + versions of pandas 1.5, NumPy 1.21, PyTables 3.8, and PyToolz 0.12. +* Behavior change under pandas 3: with ``copy_col=False``, pandas's + mandatory copy-on-write means that modifying a column returned from a + table no longer writes through to the table's underlying DataFrame. + Use ``update_col`` or ``update_col_from_series`` to change table values. +* Modernize Python packaging and continuous integration. + v1.8 ==== diff --git a/MANIFEST.in b/MANIFEST.in index 0d55727..52229d4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ include HISTORY.rst include LICENSE.txt include README.rst +recursive-include orca/tests *.py +recursive-include orca/utils/tests *.py diff --git a/orca/__init__.py b/orca/__init__.py index 092669a..bccb854 100644 --- a/orca/__init__.py +++ b/orca/__init__.py @@ -1,3 +1,5 @@ +__version__ = "1.9.dev0" + from .orca import * -version = __version__ = '1.8' +version = __version__ diff --git a/orca/orca.py b/orca/orca.py index 8dfecc1..682d7cd 100644 --- a/orca/orca.py +++ b/orca/orca.py @@ -2,22 +2,14 @@ # Copyright (C) 2022 UrbanSim Inc. # See full license in LICENSE. -from __future__ import print_function - -try: - from inspect import getfullargspec as getargspec -except ImportError: - from inspect import getargspec import logging import time import warnings from collections import namedtuple -try: - from collections.abc import Callable -except ImportError: # Python 2.7 - from collections import Callable +from collections.abc import Callable from contextlib import contextmanager from functools import wraps +from inspect import getfullargspec import pandas as pd @@ -191,7 +183,6 @@ def _update_scope(wrapper, new_scope=None): raise ValueError(msg) # update the cache properties - curr_cache = wrapper.cache curr_scope = wrapper.cache_scope if new_scope is None: # set to defaults, i.e. no caching @@ -328,7 +319,10 @@ class DataFrameWrapper(object): Name for the table. frame : pandas.DataFrame copy_col : bool, optional - Whether to return copies when evaluating columns. + Whether to return copies when evaluating columns. When False, + returned columns share memory with the underlying DataFrame; + note that under pandas 3's copy-on-write behavior, modifying + such a column no longer writes through to the table. Attributes ---------- @@ -558,7 +552,10 @@ class TableFuncWrapper(object): complete iteration of the pipeline, 'step' caches data for a single step of the pipeline. copy_col : bool, optional - Whether to return copies when evaluating columns. + Whether to return copies when evaluating columns. When False, + returned columns share memory with the underlying DataFrame; + note that under pandas 3's copy-on-write behavior, modifying + such a column no longer writes through to the table. Attributes ---------- @@ -575,7 +572,7 @@ def __init__( copy_col=True): self.name = name self._func = func - self._argspec = getargspec(func) + self._argspec = getfullargspec(func) self.cache = cache self.cache_scope = cache_scope self.copy_col = copy_col @@ -790,7 +787,7 @@ def __init__( self.table_name = table_name self.name = column_name self._func = func - self._argspec = getargspec(func) + self._argspec = getfullargspec(func) self.cache = cache self.cache_scope = cache_scope @@ -912,7 +909,7 @@ class _InjectableFuncWrapper(object): def __init__(self, name, func, cache=False, cache_scope=_CS_FOREVER): self.name = name self._func = func - self._argspec = getargspec(func) + self._argspec = getfullargspec(func) self.cache = cache self.cache_scope = cache_scope @@ -964,7 +961,7 @@ class _StepFuncWrapper(object): def __init__(self, step_name, func): self.name = step_name self._func = func - self._argspec = getargspec(func) + self._argspec = getfullargspec(func) def __call__(self): with log_start_finish('calling step {!r}'.format(self.name), logger): @@ -1162,7 +1159,10 @@ def add_table( complete iteration of the pipeline, 'step' caches data for a single step of the pipeline. copy_col : bool, optional - Whether to return copies when evaluating columns. + Whether to return copies when evaluating columns. When False, + returned columns share memory with the underlying DataFrame; + note that under pandas 3's copy-on-write behavior, modifying + such a column no longer writes through to the table. Returns ------- diff --git a/orca/tests/test_orca.py b/orca/tests/test_orca.py index 0ca40e6..cf120de 100644 --- a/orca/tests/test_orca.py +++ b/orca/tests/test_orca.py @@ -5,6 +5,7 @@ import os import tempfile +import numpy as np import pandas as pd import pandas.testing as pdt import pytest @@ -55,7 +56,7 @@ def test_func(test_frame): table = orca._TABLES['test_func'] assert table.index is None assert table.columns == [] - assert len(table) is 0 + assert len(table) == 0 pdt.assert_frame_equal(table.to_frame(), df / 2) pdt.assert_frame_equal(table.to_frame([]), df[[]]) pdt.assert_frame_equal(table.to_frame(columns=['a']), df[['a']] / 2) @@ -195,14 +196,18 @@ def func(col=label): if 'uncopied' in name: pdt.assert_series_equal(table['a'], df['a']) - assert table['a'] is df['a'] + assert np.shares_memory( + table['a'].to_numpy(), df['a'].to_numpy()) pdt.assert_series_equal(table['a'], table2['a']) - assert table['a'] is table2['a'] + assert np.shares_memory( + table['a'].to_numpy(), table2['a'].to_numpy()) else: pdt.assert_series_equal(table['a'], df['a']) - assert table['a'] is not df['a'] + assert not np.shares_memory( + table['a'].to_numpy(), df['a'].to_numpy()) pdt.assert_series_equal(table['a'], table2['a']) - assert table['a'] is not table2['a'] + assert not np.shares_memory( + table['a'].to_numpy(), table2['a'].to_numpy()) def test_columns_for_table(): diff --git a/orca/utils/testing.py b/orca/utils/testing.py index c8b80c8..b0d665b 100644 --- a/orca/utils/testing.py +++ b/orca/utils/testing.py @@ -6,7 +6,6 @@ Utilities used in testing of Orca. """ -import numpy as np import numpy.testing as npt import pandas as pd @@ -42,7 +41,7 @@ def assert_frames_equal(actual, expected, use_close=False): act_row = actual.loc[i] - for j, exp_item in exp_row.iteritems(): + for j, exp_item in exp_row.items(): assert j in act_row.index, \ 'Expected column {!r} not found.'.format(j) diff --git a/orca/utils/tests/test_testing.py b/orca/utils/tests/test_testing.py index 6e45f6c..b34263f 100644 --- a/orca/utils/tests/test_testing.py +++ b/orca/utils/tests/test_testing.py @@ -43,13 +43,10 @@ def test_frames_equal_mismatched_items(): with pytest.raises(AssertionError) as info: testing.assert_frames_equal(actual, expected) - assert (""" -Items are not equal: - ACTUAL: 2 - DESIRED: 1 - -Column: 'a' -Row: 0""" in str(info.value)) + message = str(info.value) + assert "Items are not equal" in message + assert "Column: 'a'" in message + assert "Row: 0" in message def test_frames_equal(): diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c58ed37 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,78 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "orca" +dynamic = ["version"] +description = "Python library for task orchestration" +readme = "README.rst" +requires-python = ">=3.10" +license = "BSD-3-Clause" +authors = [ + {name = "UrbanSim Inc.", email = "info@urbansim.com"}, +] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Operating System :: OS Independent", +] +dependencies = [ + "numpy>=1.21", + "pandas>=1.5,<4", + "tables>=3.8", + "toolz>=0.12", +] + +[project.optional-dependencies] +test = [ + "pytest>=8", + "pytest-cov>=5", +] +dev = [ + "build>=1.2", + "ruff>=0.12", + "twine>=6", +] + +[project.urls] +Documentation = "https://udst.github.io/orca/" +Repository = "https://github.com/UDST/orca" +Issues = "https://github.com/UDST/orca/issues" + +[tool.setuptools] +include-package-data = false + +[tool.setuptools.dynamic] +version = {attr = "orca.__version__"} + +[tool.setuptools.packages.find] +include = ["orca", "orca.*"] +exclude = ["orca.tests", "orca.tests.*", "orca.utils.tests", "orca.utils.tests.*"] + +[tool.pytest.ini_options] +testpaths = ["orca/tests", "orca/utils/tests"] +filterwarnings = [ + "ignore:object name is not a valid Python identifier:tables.NaturalNameWarning", +] + +[tool.coverage.run] +source = ["orca"] + +[tool.coverage.report] +show_missing = true +omit = ["orca/tests/*", "orca/utils/tests/*"] + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.ruff.lint] +select = ["E4", "E5", "E7", "E9", "F"] + +[tool.ruff.lint.per-file-ignores] +"**/__init__.py" = ["F401", "F403"] diff --git a/requirements-min.txt b/requirements-min.txt new file mode 100644 index 0000000..8971d83 --- /dev/null +++ b/requirements-min.txt @@ -0,0 +1,14 @@ +# Pip constraints pinning the oldest supported versions of the core +# dependencies (the floors declared in pyproject.toml). Used by the +# minimum-dependencies CI job: +# +# pip install ".[test]" -c requirements-min.txt +# +# numpy must be constrained alongside pandas 1.5: pandas 1.5 predates +# NumPy 2 and declares no upper bound, so without this constraint the +# resolver pairs it with NumPy 2.x and pandas fails to import. + +numpy == 1.21.* +pandas == 1.5.* +tables == 3.8.* +toolz == 0.12.* diff --git a/setup.py b/setup.py deleted file mode 100644 index efac0c3..0000000 --- a/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -from setuptools import setup, find_packages - -# read README as the long description -with open('README.rst', 'r') as f: - long_description = f.read() - -setup( - name='orca', - version='1.8', - description='Python library for task orchestration', - long_description=long_description, - author='UrbanSim Inc.', - author_email='info@urbansim.com', - license='BSD', - url='https://github.com/udst/orca', - classifiers=[ - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'License :: OSI Approved :: BSD License' - ], - packages=find_packages(exclude=['*.tests']), - install_requires=[ - 'pandas >= 0.15.0', - 'tables >= 3.1', - 'toolz >= 0.8.1' - ] -)