E2E tests for ONNX and RVC4 NNArchive metric consistency with LuxonisTrain

.github/workflows/onnx-nnarchive-regression.yml

@@ -0,0 +1,50 @@
+name: ONNX NNArchive Regression
+
+on:
+  push:
+    branches:
+      - main
+      - "develop*"
+  pull_request:
+    branches:
+      - main
+      - "develop*"
+    types: [opened, synchronize, reopened]
+
+jobs:
+  onnx-nnarchive-regression:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
+          create_credentials_file: true
+          export_environment_variables: true
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install gcsfs
+          python -m pip install -e ".[dev]"
+
+      - name: Run ONNX NNArchive regression tests
+        env:
+          LUXONIS_TELEMETRY_ENABLED: "false"
+        run: |
+          pytest tests/test_onnx_nnarchive_regression.py -q

.github/workflows/rvc4-nnarchive-regression.yml

@@ -0,0 +1,131 @@
+name: RVC4 NNArchive Regression
+
+on:
+  workflow_dispatch:
+    inputs:
+      reservation_name:
+        description: "Reservation name"
+        required: false
+        type: string
+      os_version:
+        description: "Optional camera OS version to install before the run"
+        required: false
+        type: string
+      hold_reservation:
+        description: "Hold testbed reservation"
+        required: false
+        type: boolean
+      depthai_version:
+        description: "DepthAI version to test against"
+        required: false
+        default: "3.10.0"
+        type: string
+      additional_options:
+        description: "Additional hil_runner options"
+        required: false
+        type: string
+  push:
+    branches:
+      - main
+      - "develop*"
+  pull_request:
+    branches:
+      - main
+      - "develop*"
+    types: [opened, synchronize, reopened]
+
+jobs:
+  rvc4-nnarchive-regression:
+    runs-on: ["self-hosted", "testbed-runner"]
+    permissions:
+      contents: read
+
+    env:
+      LUXONIS_TELEMETRY_ENABLED: "false"
+      HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }}
+      PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Install HIL runner
+        shell: bash
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install --upgrade --no-cache-dir \
+            --index-url "https://__token__:${HIL_FRAMEWORK_TOKEN}@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" \
+            hil-framework
+
+      - name: Run RVC4 NNArchive regression tests
+        shell: bash
+        env:
+          DEPTHAI_VERSION_INPUT: ${{ github.event.inputs.depthai_version }}
+          RESERVATION_NAME_INPUT: ${{ github.event.inputs.reservation_name }}
+          HOLD_RESERVATION_INPUT: ${{ github.event.inputs.hold_reservation }}
+          OS_VERSION_INPUT: ${{ github.event.inputs.os_version }}
+          ADDITIONAL_OPTIONS_INPUT: ${{ github.event.inputs.additional_options }}
+          GCP_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
+        run: |
+          set -euo pipefail
+
+          CREDENTIALS_PATH="$GITHUB_WORKSPACE/.ci/gcp-credentials.json"
+          cleanup_local_credentials() {
+            rm -f "$CREDENTIALS_PATH"
+          }
+          trap cleanup_local_credentials EXIT
+
+          umask 077
+          mkdir -p "$(dirname "$CREDENTIALS_PATH")"
+          printf '%s' "$GCP_CREDENTIALS_JSON" > "$CREDENTIALS_PATH"
+
+          DEPTHAI_VERSION_VALUE="${DEPTHAI_VERSION_INPUT:-3.8.0}"
+          if [[ ! "$DEPTHAI_VERSION_VALUE" =~ ^3\.(8|9|[1-9][0-9])\.[0-9]{1,2}$ ]]; then
+            echo "DEPTHAI_VERSION must be within >=3.8.0,<4.0.0; got: $DEPTHAI_VERSION_VALUE" >&2
+            exit 1
+          fi
+
+          if [[ -n "$RESERVATION_NAME_INPUT" ]]; then
+            RESERVATION_OPTION="--reservation-name $RESERVATION_NAME_INPUT"
+          else
+            RESERVATION_OPTION="--reservation-name https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#luxonis-eval-rvc4"
+          fi
+
+          HOLD_RESERVATION_OPTION=""
+          if [[ "$HOLD_RESERVATION_INPUT" == "true" ]]; then
+            HOLD_RESERVATION_OPTION="--hold-reservation"
+          fi
+
+          OS_VERSION_OPTION=""
+          if [[ -n "$OS_VERSION_INPUT" ]]; then
+            OS_VERSION_OPTION="--os-version $OS_VERSION_INPUT"
+          fi
+
+          ADDITIONAL_OPTIONS_VALUE=""
+          if [[ -n "$ADDITIONAL_OPTIONS_INPUT" ]]; then
+            ADDITIONAL_OPTIONS_VALUE="$ADDITIONAL_OPTIONS_INPUT"
+          fi
+
+          REMOTE_HIL_FRAMEWORK_TOKEN="$(printf '%q' "$HIL_FRAMEWORK_TOKEN")"
+          REMOTE_DEPTHAI_VERSION="$(printf '%q' "$DEPTHAI_VERSION_VALUE")"
+          REMOTE_CMD="set -e; \
+            trap 'rm -f /tmp/luxonis-eval/.ci/gcp-credentials.json' EXIT; \
+            cd /tmp/luxonis-eval && \
+            chmod 600 /tmp/luxonis-eval/.ci/gcp-credentials.json && \
+            export GOOGLE_APPLICATION_CREDENTIALS=/tmp/luxonis-eval/.ci/gcp-credentials.json && \
+            export LUXONIS_TELEMETRY_ENABLED=false && \
+            export HIL_FRAMEWORK_TOKEN=$REMOTE_HIL_FRAMEWORK_TOKEN && \
+            export DEPTHAI_VERSION=$REMOTE_DEPTHAI_VERSION && \
+            bash ./tests/run_rvc4_nnarchive_hil_tests.sh"
+
+          hil_runner \
+            --models "oak4_pro or oak4_d or oak4_s" \
+            $HOLD_RESERVATION_OPTION \
+            $ADDITIONAL_OPTIONS_VALUE \
+            --wait \
+            $OS_VERSION_OPTION \
+            $RESERVATION_OPTION \
+            --sync-workspace --rsync-args="--exclude=venv" \
+            --commands "$REMOTE_CMD"

luxonis_eval/core/runtime.py

@@ -123,26 +123,27 @@ def resolve_luxonis_loader_class_mapping(
             "`BaseEvalLoader`."
         )
     loader_task_name = loader_task_name or ""
-    dataset_classes = dataloader.dataset.get_classes()
+    dataset_classes = dataloader._classes
     ldf_class_map = {
         v: k for k, v in dataset_classes[loader_task_name].items()
     }
+    dataset_name = dataloader.dataset._dataset_name
 
-    if "imagenet" in dataloader.dataset.dataset_name:
+    if "imagenet" in dataset_name:
         native_class_map = get_dataset_class_mapping("imagenet")
-    elif "coco" in dataloader.dataset.dataset_name:
+    elif "coco" in dataset_name:
         native_class_map = get_dataset_class_mapping("coco")
     else:
         native_class_map = class_mapping
         if native_class_map:
             logger.info(
-                f"Dataset '{dataloader.dataset.dataset_name}' does not match "
+                f"Dataset '{dataset_name}' does not match "
                 "known datasets for automatic class mapping. Using the "
                 "provided 'loader.params.class_mapping' argument."
             )
         else:
             logger.warning(
-                f"Dataset '{dataloader.dataset.dataset_name}' does not match "
+                f"Dataset '{dataset_name}' does not match "
                 "known datasets for automatic class mapping and no "
                 "'loader.params.class_mapping' was provided. Falling back to "
                 "the dataset's LDF class order as the native class mapping."

requirements-dev.txt

@@ -1 +1,2 @@
-pre-commit==3.2.1
+pytest>=8.0.0,<9.0.0
+pre-commit>=3.2.1,<4.0.0

requirements.txt

@@ -1,6 +1,6 @@
 cyclopts
 depthai>=3.7.1,<4
-luxonis-ml[data,nn_archive,utils]>=0.8.2
+luxonis-ml[data,nn_archive,utils]>=0.9.0
 loguru
 # TODO: Switch to a released depthai-nodes version once the required changes are published.
 depthai-nodes @ git+https://github.com/luxonis/depthai-nodes.git@main

tests/__init__.py

@@ -0,0 +1 @@
+"""Test suite for luxonis-eval."""

tests/conftest.py

@@ -0,0 +1,136 @@
+import os
+from pathlib import Path
+
+import pytest
+from luxonis_ml.utils import LuxonisFileSystem
+
+os.environ.setdefault("LUXONIS_TELEMETRY_ENABLED", "false")
+
+NORMALIZED_TESTDATA_REMOTE_DIR = (
+    "gs://luxonis-test-bucket/luxonis-eval-test-data/NormalizedTestData"
+)
+LOCAL_TESTDATA_ENV_VAR = "LUXONIS_EVAL_TESTDATA_DIR"
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(
+        "--device-ip",
+        action="store",
+        default=None,
+        help=(
+            "RVC4 device IP or MXID. If omitted, uses RVC4_IP or "
+            "testbed resolution when configured."
+        ),
+    )
+    parser.addoption(
+        "--testbed-name",
+        action="store",
+        default=None,
+        help="Optional HIL testbed name used to resolve an RVC4 device.",
+    )
+    parser.addoption(
+        "--require-device",
+        action="store_true",
+        default=False,
+        help="Fail instead of skipping when no RVC4 device can be resolved.",
+    )
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line(
+        "markers", "device: mark test as requiring a physical device"
+    )
+    config.addinivalue_line(
+        "markers", "rvc4: mark test as requiring an RVC4 device"
+    )
+
+
+@pytest.fixture(scope="session")
+def nnarchive_testdata_root(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    configured_dir = os.environ.get(LOCAL_TESTDATA_ENV_VAR)
+    if configured_dir:
+        return Path(configured_dir)
+
+    data_dir = tmp_path_factory.mktemp("nnarchive_testdata")
+    return LuxonisFileSystem.download(
+        NORMALIZED_TESTDATA_REMOTE_DIR,
+        dest=data_dir,
+    )
+
+
+@pytest.fixture(scope="session")
+def rvc4_device_ip(request: pytest.FixtureRequest) -> str | None:
+    configured_device_ip = request.config.getoption("--device-ip")
+    if configured_device_ip:
+        return configured_device_ip
+
+    env_device_ip = os.environ.get("RVC4_IP")
+    if env_device_ip:
+        return env_device_ip
+
+    testbed_name = request.config.getoption(
+        "--testbed-name"
+    ) or os.environ.get("HIL_TESTBED")
+    if not testbed_name:
+        return None
+
+    return _resolve_rvc4_device_ip_from_testbed(testbed_name)
+
+
+@pytest.fixture(scope="session")
+def required_rvc4_device_ip(
+    request: pytest.FixtureRequest, rvc4_device_ip: str | None
+) -> str:
+    if rvc4_device_ip is not None:
+        return rvc4_device_ip
+
+    message = (
+        "No RVC4 device configured. Re-run with --device-ip <ip-or-mxid>, "
+        "set RVC4_IP, or provide --testbed-name / HIL_TESTBED."
+    )
+    if request.config.getoption("--require-device"):
+        pytest.fail(message, pytrace=False)
+
+    pytest.skip(message)
+
+
+def _resolve_rvc4_device_ip_from_testbed(testbed_name: str) -> str:
+    try:
+        from hil_framework.lib_testbed.config.Config import Config
+        from hil_framework.lib_testbed.utils.Testbed import Testbed
+    except ImportError:
+        pytest.exit(
+            "hil_framework is required when --testbed-name or HIL_TESTBED is used.",
+            returncode=1,
+        )
+
+    testbed = Testbed(Config(testbed_name))
+    target_matches = [
+        camera
+        for camera in testbed.cameras
+        if str(getattr(camera, "platform", "")).lower() == "rvc4"
+    ]
+    if len(target_matches) == 1:
+        hostname = getattr(target_matches[0], "hostname", None)
+        if not hostname:
+            pytest.exit(
+                f"Selected RVC4 camera in testbed {testbed_name!r} does not expose a hostname.",
+                returncode=1,
+            )
+        return str(hostname)
+
+    available_cameras = ", ".join(
+        f"{camera.name}:{getattr(camera, 'hostname', None)}:{getattr(camera, 'platform', None)}"
+        for camera in testbed.cameras
+    )
+    if not target_matches:
+        pytest.exit(
+            f"No RVC4 camera found in testbed {testbed_name!r}. Available cameras: {available_cameras}",
+            returncode=1,
+        )
+
+    pytest.exit(
+        "Unable to choose a unique RVC4 camera from testbed "
+        f"{testbed_name!r}. Available cameras: {available_cameras}",
+        returncode=1,
+    )