TRR tests implementation in the core nectarchain module (#173)

* move TRR test suite into nectarchain

* add h5py dependency

* add lmfit to pyproject.toml

* fix docstring

* rebase on main

* Fix Sphinx intermapping with PyQt documentation

* Switch back to PyQt5 instead of PyQt6, to be able to launch the TRR GUI, on Linux at least

* To try to ensure that the TRR GUI is launchable both from Linux and macOS

* very simple unit test for TRR

* change test_gui to work on macos

* remove pyqt6

* install pyqt5 in CI for mac

* sqlite issue

* check pyqt installation in CI

* install pyqt with brew before mamba installation on mac

* add --no-deps to pip install nectarchain for  pytest

* add cleaner pip installation of nectarchain

* clean separation of the installation through pip and mamba

* tomlib -> toml for python < 3.11

* fix pip install

* clean

* Remove any use of PyQt6 in TRR GUI

* Try to re-simplify the installation of dependencies for CI tests

* Upload codecov report only for 1 platform/python version/installation method, not for all of them.

---------

Co-authored-by: guillaume.grolleron <[email protected]>
Co-authored-by: jlenain <[email protected]>

Author: 3 people

.github/workflows/ci.yml

@@ -4,12 +4,10 @@ name: CI
 on:
   workflow_dispatch:
   push:
-  # TODO
-  # Remove comment: just a test to force CI on push
-  # branches:
-  #   - main
-  # tags:
-  #   - '**'
+    branches:
+      - main
+    tags:
+      - "**"
   pull_request:
 
 env:
@@ -61,6 +59,7 @@ jobs:
             python-version: "3.11"
             ctapipe-version: "v0.19.3"
             install-method: "pip"
+            extra-args: ["codecov"]
           # macos 14 image is arm64 based
           - os: "macos-14"
             python-version: "3.11"
@@ -70,6 +69,7 @@ jobs:
     env :
       DISPLAY: ":99.0"
       QT_SELECT: "qt5"
+
     defaults:
       run:
         # We need login shells (-l) for micromamba to work.
@@ -82,6 +82,10 @@ jobs:
           sudo apt-get update
           sudo apt-get install freeglut3-dev
 
+      - name: Install dependencies for Qt on mac-OS
+        if: matrix.os == 'macos-14'
+        run: |
+          brew install pyqt
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
@@ -92,63 +96,85 @@ jobs:
         run: |
           sed -i -e "s/- python=.*/- python=$PYTHON_VERSION/g" environment.yml
 
+      - name: Setup xvfb for X11 display
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          # Stuff copied wildly from several stackoverflow posts
+          sudo apt-get install -y xvfb libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0 libxcb-shape0 libglib2.0-0 libgl1-mesa-dev
+          sudo apt-get install '^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev
+          # start xvfb in the background
+          sudo /usr/bin/Xvfb $DISPLAY -screen 0 1280x1024x24 &
+
       - name: Mamba setup
         if: matrix.install-method == 'mamba'
         uses: mamba-org/setup-micromamba@v2
         with:
           environment-name: "ci"
           environment-file: environment.yml
 
-
       - name: Python setup
         if: matrix.install-method == 'pip'
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           check-latest: true
-      
-      - name: Setup xvfb for X11 display
-        if: matrix.os == 'ubuntu-latest'
-        run: |
-          # Stuff copied wildly from several stackoverflow posts
-          sudo apt-get install -y xvfb libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0 libxcb-shape0 libglib2.0-0 libgl1-mesa-dev
-          sudo apt-get install '^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev
-          # start xvfb in the background
-          sudo /usr/bin/Xvfb $DISPLAY -screen 0 1280x1024x24 &
-      
-      - name: Instal QT and setup environment on macOS
-        if: matrix.os == 'macos-14'
-        run: |
-          brew install pyqt@6
-          echo "DYLD_FRAMEWORK_PATH=$(brew --prefix qt)/lib" >> $GITHUB_ENV
 
       - name: Install dependencies
         env:
           CTAPIPE_VERSION: ${{ matrix.ctapipe-version }}
-
         run: |
           python --version
           echo "Installing additional pip packages"
           # we install ctapipe using pip to be able to select any commit, e.g. the current main
           pip install \
              "git+https://github.com/cta-observatory/ctapipe@$CTAPIPE_VERSION"
 
-          echo "pip install -e ."
-          pip install -e .[test]
-
-      - name: Check pyqt version
+          if [ "${{ matrix.os }}" == "macos-14" ]; then
+            # nectarchain installation with pip
+            if [ "${{ matrix.install-method }}" == "pip" ]; then
+              pip install -e .
+            else
+              pip install --no-deps -e .
+            fi
+            # install with pip dependencies for pytest (NB : when we will use python > 3.10 for all configs, use tomllib instead of toml)
+            pip install toml
+            python -c "import toml, sys; print('\n'.join(toml.load('pyproject.toml')['project']['optional-dependencies']['test']))" | xargs pip install 
+          else
+            echo "pip install -e ."
+            pip install -e .[test]
+          fi
+
+      - name: Check pyqt version and installation
         run: |
-          pip list | grep PyQt
+          if [ "${{ matrix.install-method }}" == "pip" ]; then
+            pip list | grep PyQt
+          else
+            micromamba list -n ci qt
+            pip list | grep PyQt
+          fi
+
+      - name: Check mac-os installation
+        if: matrix.os == 'macos-14'
+        run: |
+          micromamba list -n ci | grep qt
+          eval "$(micromamba shell hook --shell=bash)"
+          micromamba activate ci
+          otool -L $(python -c "import PyQt5.QtCore; print(PyQt5.QtCore.__file__)")
 
       - name: Tests
         run: |
           pytest -n auto --dist loadscope --cov=nectarchain --cov-report=xml --ignore=src/nectarchain/user_scripts
 
       - name: Upload coverage reports to Codecov with GitHub Action
         uses: codecov/codecov-action@v5
+        if: contains(matrix.extra-args, 'codecov')
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+      # - name: Start SSH Debugging Session
+      #   if: ${{ failure() }}
+      #   uses: mxschmitt/action-tmate@v3
+
 
   docs:
     needs: lint

docs/conf.py

@@ -51,6 +51,7 @@
     # class signature)
     "numpydoc",
     "sphinx_design",
+    "sphinx_qt_documentation",
 ]
 
 autosummary_generate = True  # Turn on sphinx.ext.autosummary

environment.yml

@@ -17,11 +17,13 @@ dependencies:
   - sphinx
   - sphinx-automodapi
   - pydata-sphinx-theme
-  - pyqt # [linux]
+  - lmfit # needed into TRR
+  - h5py # needed into TRR (should be removed to use I/O methods of containers)
+  - pyqt
   - pip:
     - zeo
     - zodb
     - mechanize
     - browser-cookie3
     - pyqtgraph
-    - pyqt6 # [osx and arm64]
+    - sphinx-qt-documentation

pyproject.toml

@@ -27,8 +27,11 @@ dependencies = [
     "scipy==1.11.4",
     "zodb",
     "zeo",
-    "pyqt6",
+    "lmfit",
+    "h5py",
+    "pyqt5",
     "pyqtgraph",
+    "tomli; python_version < '3.11'",
 ]
 
 # needed for setuptools_scm, we don't define a static version
@@ -52,9 +55,9 @@ docs = [
     "sphinx-autodoc-typehints",
     "sphinx-automodapi",
     "sphinx-design",
+    "sphinx-qt-documentation",
     "pydata_sphinx_theme",
     "numpydoc",
-    "tomli; python_version < '3.11'"
 ]
 # we can use self-references to simplify all
 all = [

src/nectarchain/makers/tests/test_core_makers.py

@@ -1,5 +1,4 @@
 import logging
-import os
 import pathlib
 from unittest.mock import MagicMock, patch
 
@@ -58,22 +57,22 @@ class TestEventsLoopNectarCAMCalibrationTool(TestBaseNectarCAMCalibrationTool):
 
     @pytest.fixture
     def tool_instance(self):
-        return EventsLoopNectarCAMCalibrationTool(run_number=self.RUN_NUMBER)
+        return EventsLoopNectarCAMCalibrationTool(
+            run_number=self.RUN_NUMBER,
+            output_path=pathlib.Path("/tmp/test_output.h5"),
+        )
 
     @pytest.fixture
     def tool_instance_run_file(self):
         return EventsLoopNectarCAMCalibrationTool(
             run_number=self.RUN_NUMBER,
             run_file=self.RUN_FILE,
-            output_path=pathlib.Path(f"/tmp/{np.random.random()}test_output.h5")
+            output_path=pathlib.Path(f"/tmp/{np.random.random()}test_output.h5"),
             # to avoid I/O conflicts between tests
         )
 
     def test_init_output_path(self, tool_instance):
-        expected_path = pathlib.Path(
-            f"{os.environ.get('NECTARCAMDATA', '/tmp')}/runs"
-            f"/EventsLoopNectarCAMCalibration_run{self.RUN_NUMBER}.h5"
-        )
+        expected_path = pathlib.Path("/tmp/test_output.h5")
         assert tool_instance.output_path == expected_path
         assert tool_instance.run_number == self.RUN_NUMBER
         assert tool_instance.max_events is None

…r_scripts/dmousadi/TRR_scripts/README.md src/nectarchain/trr_test_suite/README.mdsrc/nectarchain/user_scripts/dmousadi/TRR_scripts/README.md renamed to src/nectarchain/trr_test_suite/README.md src/nectarchain/user_scripts/dmousadi/TRR_scripts/README.md renamed to src/nectarchain/trr_test_suite/README.md

@@ -0,0 +1,5 @@
+from .gui import TestRunner
+
+__all__ = [
+    "TestRunner",
+]