diff --git a/.github/workflows/build_publish.yml b/.github/workflows/build_publish.yml index f6489250c..d68f84631 100644 --- a/.github/workflows/build_publish.yml +++ b/.github/workflows/build_publish.yml @@ -10,16 +10,25 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v6 with: - python-version: '3.9' - - uses: Gr1N/setup-poetry@v8 - - run: poetry install - - run: poetry run poe tests_unit - - run: poetry run poe checks_codestyle - - run: poetry run poe checks_typing - - run: poetry run poe checks_license + python-version: '3.13' + + - name: Install Poetry + run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 + + - name: Install dependencies + run: poetry install + + - name: Unit tests + run: poetry run poe tests_unit + + - name: Code style, typing and license checks + run: poetry run poe checks_codestyle + + - name: Check typings + run: poetry run poe checks_typing build: needs: [test] @@ -27,24 +36,32 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v6 with: - python-version: '3.9' - - uses: Gr1N/setup-poetry@v8 + python-version: '3.13' + + - name: Install Poetry + run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 + - name: Get git release tag run: echo "git-release-tag=yapapi $(git describe --tags)" >> $GITHUB_OUTPUT id: git_describe + - name: Get package version run: echo "poetry-version=$(poetry version)" >> $GITHUB_OUTPUT id: poetry_version + - name: Fail on version mismatch run: exit 1 if: ${{ steps.git_describe.outputs.git-release-tag != steps.poetry_version.outputs.poetry-version }} + - name: Build the release run: poetry build + - name: Store the built package uses: actions/upload-artifact@v4 with: @@ -58,16 +75,21 @@ jobs: if: ${{ github.event.action == 'prereleased' }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v6 with: - python-version: '3.9' - - uses: Gr1N/setup-poetry@v8 + python-version: '3.13' + + - name: Install Poetry + run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 + - name: Retrieve the built package uses: actions/download-artifact@v4 with: name: dist path: dist + - name: Publish to pypi run: | poetry config repositories.testpypi https://test.pypi.org/legacy/ @@ -80,11 +102,15 @@ jobs: if: ${{ github.event.action == 'released' }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v6 with: - python-version: '3.9' - - uses: Gr1N/setup-poetry@v8 + python-version: '3.13' + + - name: Install Poetry + run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 + - name: Retrieve the built package uses: actions/download-artifact@v4 with: diff --git a/.github/workflows/goth-nightly.yml b/.github/workflows/goth-nightly.yml index 752adb4ea..6750b9bf1 100644 --- a/.github/workflows/goth-nightly.yml +++ b/.github/workflows/goth-nightly.yml @@ -47,19 +47,19 @@ jobs: run: sudo apt-get install -y libffi-dev build-essential - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Configure python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: - python-version: '3.9' + python-version: '3.13' - name: Install and configure Poetry - run: python -m pip install -U pip setuptools poetry==1.3.2 + run: python -m pip install -U pip setuptools poetry==2.2.1 - name: Install dependencies run: | - poetry env use python3.9 + poetry env use python3.13 poetry install - name: Disconnect Docker containers from default network diff --git a/.github/workflows/goth.yml b/.github/workflows/goth.yml index a153091c2..422532541 100644 --- a/.github/workflows/goth.yml +++ b/.github/workflows/goth.yml @@ -20,12 +20,12 @@ jobs: uses: actions/checkout@v4 - name: Configure python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: '3.10' + python-version: '3.13' - name: Install Poetry - run: curl -sSL https://install.python-poetry.org | python3 - --version 1.8.2 + run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 - name: Install dependencies run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 470ed5025..d3514cbf2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,31 +15,27 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.10", "3.11", "3.12", "3.13"] os: - ubuntu-latest - - macos-latest - - windows-latest - exclude: - - os: windows-latest - python-version: "3.10" - - os: macos-latest - python-version: "3.10" fail-fast: false steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - - uses: Gr1N/setup-poetry@v8 + + - name: Install Poetry + run: curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 - run: echo "ENABLE=1" >> $GITHUB_OUTPUT if: ${{ matrix.os == 'ubuntu-latest' }} name: Enable extended checks id: extended-checks - run: echo "ENABLE=1" >> $GITHUB_OUTPUT - if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.9' }} + if: ${{ matrix.os == 'ubuntu-latest' }} name: Enable sphinx check id: extended-checks-sphinx @@ -52,7 +48,5 @@ jobs: - run: poetry run poe checks_codestyle - run: poetry run poe checks_typing if: ${{ steps.extended-checks.outputs.ENABLE }} - - run: poetry run poe checks_license - if: ${{ steps.extended-checks.outputs.ENABLE }} - run: poetry run poe sphinx -W if: ${{ steps.extended-checks-sphinx.outputs.ENABLE }} diff --git a/examples/http-auth/utils.py b/examples/http-auth/utils.py new file mode 100644 index 000000000..a92810fc2 --- /dev/null +++ b/examples/http-auth/utils.py @@ -0,0 +1,114 @@ +"""Utilities for yapapi example scripts.""" + +import argparse +import asyncio +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +import colorama # type: ignore + +from yapapi import ( + Golem, + NoPaymentAccountError, +) +from yapapi import __version__ as yapapi_version +from yapapi import ( + windows_event_loop_fix, +) +from yapapi.log import enable_default_logger + +TEXT_COLOR_RED = "\033[31;1m" +TEXT_COLOR_GREEN = "\033[32;1m" +TEXT_COLOR_YELLOW = "\033[33;1m" +TEXT_COLOR_BLUE = "\033[34;1m" +TEXT_COLOR_MAGENTA = "\033[35;1m" +TEXT_COLOR_CYAN = "\033[36;1m" +TEXT_COLOR_WHITE = "\033[37;1m" + +TEXT_COLOR_DEFAULT = "\033[0m" + +colorama.init() + + +def build_parser(description: str) -> argparse.ArgumentParser: + current_time_str = datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S%z") + default_log_path = Path(tempfile.gettempdir()) / f"yapapi_{current_time_str}.log" + + parser = argparse.ArgumentParser(description=description) + parser.add_argument( + "--payment-driver", "--driver", help="Payment driver name, for example `erc20`" + ) + parser.add_argument( + "--payment-network", "--network", help="Payment network name, for example `rinkeby`" + ) + parser.add_argument("--subnet-tag", help="Subnet name, for example `devnet-beta`") + parser.add_argument( + "--log-file", + default=str(default_log_path), + help="Log file for YAPAPI; default: %(default)s", + ) + return parser + + +def format_usage(usage): + return { + "current_usage": usage.current_usage, + "timestamp": usage.timestamp.isoformat(sep=" ") if usage.timestamp else None, + } + + +def print_env_info(golem: Golem): + print( + f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n" + f"Using subnet: {TEXT_COLOR_YELLOW}{golem.subnet_tag}{TEXT_COLOR_DEFAULT}, " + f"payment driver: {TEXT_COLOR_YELLOW}{golem.payment_driver}{TEXT_COLOR_DEFAULT}, " + f"and network: {TEXT_COLOR_YELLOW}{golem.payment_network}{TEXT_COLOR_DEFAULT}\n" + ) + + +def run_golem_example(example_main, log_file=None): + # This is only required when running on Windows with Python prior to 3.8: + windows_event_loop_fix() + + if log_file: + enable_default_logger( + log_file=log_file, + debug_activity_api=True, + debug_market_api=True, + debug_payment_api=True, + debug_net_api=True, + ) + + loop = asyncio.get_event_loop() + task = loop.create_task(example_main) + + try: + loop.run_until_complete(task) + except NoPaymentAccountError as e: + handbook_url = ( + "https://handbook.golem.network/requestor-tutorials/" + "flash-tutorial-of-requestor-development" + ) + print( + f"{TEXT_COLOR_RED}" + f"No payment account initialized for driver `{e.required_driver}` " + f"and network `{e.required_network}`.\n\n" + f"See {handbook_url} on how to initialize payment accounts for a requestor node." + f"{TEXT_COLOR_DEFAULT}" + ) + except KeyboardInterrupt: + print( + f"{TEXT_COLOR_YELLOW}" + "Shutting down gracefully, please wait a short while " + "or press Ctrl+C to exit immediately..." + f"{TEXT_COLOR_DEFAULT}" + ) + task.cancel() + try: + loop.run_until_complete(task) + print( + f"{TEXT_COLOR_YELLOW}Shutdown completed, thank you for waiting!{TEXT_COLOR_DEFAULT}" + ) + except (asyncio.CancelledError, KeyboardInterrupt): + pass diff --git a/pyproject.toml b/pyproject.toml index b7a83d8c3..340e39d75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "yapapi" -version = "0.13.1" +version = "0.14.0" description = "High-level Python API for the New Golem" authors = ["GolemFactory "] license = "LGPL-3.0-or-later" @@ -19,7 +19,7 @@ requires = ["poetry_core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.dependencies] -python = "^3.8" +python = "^3.10" aiohttp = "^3.8" aiohttp-sse-client = "^0.2.1" @@ -42,28 +42,28 @@ pip = "*" alive_progress = "3.1" # Docs -sphinx = { version = "^4.0.1", optional = true } -sphinx-autodoc-typehints = { version = "^1.12.0", optional = true } -sphinx-rtd-theme = { version = "^1.0.0", optional = true} +sphinx = { version = "^8.0", optional = true } +sphinx-autodoc-typehints = { version = "^3.0", optional = true } +sphinx-rtd-theme = { version = "^3.0.2", optional = true} [tool.poetry.extras] docs = ['sphinx', 'sphinx-autodoc-typehints', 'sphinx-rtd-theme'] [tool.poetry.group.dev.dependencies] black = "^24.4.2" -factory-boy = "^3.2.0" -isort = "^5.10.1" -liccheck = "^0.4.7" -mypy = "^1.10.0" -poethepoet = "^0.8.0" -pytest = "^6.2" -pytest-asyncio = "^0.14" -pytest-cov = "^2.11" -pytest-rerunfailures = "^10.1" +factory-boy = "^3.3.3" +isort = "^7.0" +liccheck = "^0.9.2" +mypy = "^1.18.2" +poethepoet = "^0.37" +pytest = "^8.4" +pytest-asyncio = "^1.2" +pytest-cov = "^7.0" +pytest-rerunfailures = "^16.1" autoflake = "^1" -flake8 = "^5" -flake8-docstrings = "^1.6" -Flake8-pyproject = "^1.2.2" +flake8 = "^7.3" +flake8-docstrings = "^1.7" +Flake8-pyproject = "^1.2.3" pyproject-autoflake = "^1.0.2" @@ -85,9 +85,9 @@ _format_black = "black ." tests_unit = {cmd = "pytest --cov=yapapi --cov-report html --cov-report term -sv --ignore tests/goth_tests", help = "Run only unit tests"} tests_integration_init = { sequence = ["_gothv_env", "_gothv_requirements", "_gothv_assets"], help="Initialize the integration test environment"} -tests_integration = { cmd = ".envs/yapapi-goth/bin/python -m pytest -svx tests/goth_tests --config-override docker-compose.build-environment.use-prerelease=false --config-path tests/goth_tests/assets/goth-config.yml --ssh-verify-connection --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError --only-rerun requests.exceptions.ConnectionError --only-rerun OSError --only-rerun requests.exceptions.ReadTimeout", help = "Run the integration tests"} +tests_integration = { cmd = ".envs/yapapi-goth/bin/python -m pytest -svx tests/goth_tests --config-override docker-compose.build-environment.use-prerelease=false --config-path tests/goth_tests/assets/goth-config.yml --ssh-verify-connection", help = "Run the integration tests"} _gothv_env = "python -m venv .envs/yapapi-goth" -_gothv_requirements = ".envs/yapapi-goth/bin/pip install -U --extra-index-url https://test.pypi.org/simple/ goth==0.17.0 pip pytest pytest-asyncio pytest-rerunfailures pexpect" +_gothv_requirements = ".envs/yapapi-goth/bin/pip install -U --extra-index-url https://test.pypi.org/simple/ goth==0.22.1 pip pytest pytest-asyncio pytest-rerunfailures pexpect" _gothv_assets = ".envs/yapapi-goth/bin/python -m goth create-assets tests/goth_tests/assets" clean = {cmd = "rm -rf .coverage .requirements.txt dist md handbook build", help = "Clean all development related files" } @@ -177,7 +177,6 @@ line_length = 100 [tool.black] line-length = 100 -target-version = ['py38'] [tool.pytest.ini_options] asyncio_mode = "auto" diff --git a/tests/goth_tests/conftest.py b/tests/goth_tests/conftest.py index 35e88a76f..72b2924c8 100644 --- a/tests/goth_tests/conftest.py +++ b/tests/goth_tests/conftest.py @@ -1,3 +1,4 @@ +import os import asyncio from datetime import datetime, timezone from pathlib import Path @@ -100,10 +101,22 @@ def project_dir() -> Path: @pytest.fixture(scope="session") def log_dir() -> Path: - base_dir = Path("/", "tmp", "goth-tests") - date_str = datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S%z") + """Fixture providing unique directory for logs from a test run.""" + base_dir = Path("logs") + date_str = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d_%H-%M-%S") log_dir = base_dir / f"goth_{date_str}" - log_dir.mkdir(parents=True) + log_dir.mkdir(parents=True, exist_ok=True) + + # Create symlink to latest on Linux + if os.name != "nt": + latest_link = base_dir / "latest" + try: + if latest_link.is_symlink() or latest_link.exists(): + latest_link.unlink() + latest_link.symlink_to(f"goth_{date_str}", target_is_directory=True) + except OSError as e: + print(f"Warning: could not create symlink {latest_link} -> {log_dir}: {e}") + return log_dir diff --git a/tests/goth_tests/test_agreement_termination/test_agreement_termination.py b/tests/goth_tests/test_agreement_termination/test_agreement_termination.py index 58ec70820..76661b27c 100644 --- a/tests/goth_tests/test_agreement_termination/test_agreement_termination.py +++ b/tests/goth_tests/test_agreement_termination/test_agreement_termination.py @@ -48,6 +48,7 @@ async def assert_all_tasks_computed(stream): remaining_ids = {1, 2, 3, 4, 5, 6} async for line in stream: + logger.log(logging.INFO, line) m = re.search(r"TaskAccepted\(.*task=Task\(id=([0-9]+)", line) if m: task_id = int(m.group(1)) diff --git a/yapapi/__init__.py b/yapapi/__init__.py index 51a60d71c..63ca146c1 100644 --- a/yapapi/__init__.py +++ b/yapapi/__init__.py @@ -2,10 +2,10 @@ import asyncio import sys +from importlib.metadata import version from pathlib import Path import toml -from pkg_resources import get_distribution from yapapi.ctx import ExecOptions, WorkContext from yapapi.engine import NoPaymentAccountError @@ -22,7 +22,7 @@ def get_version() -> str: return pyproject["tool"]["poetry"]["version"] - return get_distribution("yapapi").version + return version("yapapi") def windows_event_loop_fix(): diff --git a/yapapi/payload/vm.py b/yapapi/payload/vm.py index 7bd1b70f1..8d0e13bef 100644 --- a/yapapi/payload/vm.py +++ b/yapapi/payload/vm.py @@ -40,9 +40,6 @@ class VmRequest(ExeUnitRequest): package_format: VmPackageFormat = prop_base.prop("golem.srv.comp.vm.package_format") -import json - - @dataclass class VmManifestRequest(ExeUnitManifestRequest): def __init__(self, **kwargs): @@ -159,44 +156,60 @@ async def manifest( Uses a signed node descriptor along with a manifest to grant access to either whitelisted domains or unrestricted access. Providers only need to trust the certificate once. - example usage:: + Example usage:: package = await vm.manifest( - manifest = open("manifest_partner_unrestricted.json", "rb").read(), - node_descriptor = json.loads(open("node-descriptor.signed.json", "r").read()), - capabilities = ["inet"], + manifest=open("manifest_partner_unrestricted.json", "rb").read(), + node_descriptor=json.loads(open("node-descriptor.signed.json", "r").read()), + capabilities=["inet"], ) 2. Alternative: Pure Manifest Scheme Requires providers to manually trust each domain listed in the manifest. More complex to set up and maintain. - example usage:: + Example usage:: package = await vm.manifest( - manifest = open("manifest_whitelist.json", "rb").read(), - manifest_sig = open("manifest.json.sha256.sig", "rb").read(), - manifest_sig_algorithm = "sha256", - manifest_cert = open("cert.der", "rb").read(), - capabilities = ["inet", "manifest-support"], + manifest=open("manifest_whitelist.json", "rb").read(), + manifest_sig=open("manifest.json.sha256.sig", "rb").read(), + manifest_sig_algorithm="sha256", + manifest_cert=open("cert.der", "rb").read(), + capabilities=["inet", "manifest-support"], ) For more information about outbound access schemes, see: https://handbook.golem.network/requestor-tutorials/vm-runtime/accessing-internet - Parameters: - :param manifest: Computation Payload Manifest as raw data or base64 encoded string - :param manifest_sig: Optional signature of manifest (required for pure manifest scheme) - :param manifest_sig_algorithm: Optional signature algorithm, e.g. "sha256" (required for pure manifest scheme) - :param manifest_cert: Optional public certificate for manifest verification (required for pure manifest scheme) - :param node_descriptor: Optional signed node descriptor (recommended for partner scheme) - :param min_mem_gib: Minimal memory required to execute application code - :param min_storage_gib: Minimal disk storage to execute tasks - :param min_cpu_threads: Minimal available logical CPU cores - :param capabilities: Optional list of required VM capabilities. Use ["inet"] for partner scheme - or ["inet", "manifest-support"] for pure manifest scheme - :return: The payload definition for the given VM image - + Parameters + ---------- + manifest : Union[str, bytes] + Computation Payload Manifest as raw data or base64 encoded string. + manifest_sig : Optional[Union[str, bytes]] + Optional signature of manifest (required for pure manifest scheme). + manifest_sig_algorithm : Optional[str] + Optional signature algorithm, e.g. "sha256" (required for pure manifest scheme). + manifest_cert : Optional[Union[str, bytes]] + Optional public certificate for manifest verification (required for pure manifest scheme). + node_descriptor : Optional[Dict[str, Any]] + Optional signed node descriptor (recommended for partner scheme). + min_mem_gib : float + Minimal memory required to execute application code. + min_storage_gib : float + Minimal disk storage to execute tasks. + min_cpu_threads : int + Minimal available logical CPU cores. + capabilities : Optional[List[VmCaps]] + Optional list of required VM capabilities. Use ["inet"] for partner scheme + or ["inet", "manifest-support"] for pure manifest scheme. + + Returns + ------- + Package + The payload definition for the given VM image. + + Notes + ----- The manifest, manifest_sig, and manifest_cert parameters can be provided either as raw data or already base64 encoded. The function will automatically handle the encoding if needed. """ diff --git a/yapapi/props/base.py b/yapapi/props/base.py index 44c91c99d..d95e6b07b 100644 --- a/yapapi/props/base.py +++ b/yapapi/props/base.py @@ -223,7 +223,7 @@ def constraint( ``` """ # the default / default_factory exception is resolved by the `field` function - return field( # type: ignore + return field( default=default, default_factory=default_factory, metadata={