diff --git a/docs/agents/smurf_hammer.rst b/docs/agents/smurf_hammer.rst new file mode 100644 index 000000000..bf09d8c82 --- /dev/null +++ b/docs/agents/smurf_hammer.rst @@ -0,0 +1,85 @@ +.. highlight:: rst + +.. _smurf_hammer: + +================== +SMuRF Hammer Agent +================== + +The SMuRF Hammer Agent wraps sodetlib's ``jackhammer hammer`` CLI command as an +OCS agent. It operates on the crate controlled by the SMuRF server to which it +is deployed and accepts that usual ``jackhammer`` options. Slots for which the +hammer fails will be reported in the session data. There is also a monitoring +process to expose the configuration state of each slot. + +.. argparse:: + :filename: ../socs/agents/smurf_hammer/agent.py + :func: add_agent_args + :prog: python3 agent.py + +Dependencies +-------------- + +The SMuRF Hammer Agent requires the following packages: + + - `sodetlib `_ + +Additionally, ``socs`` should be installed with the ``pysmurf`` group: + +.. code-block:: bash + + $ pip install -U socs[pysmurf] + +Configuration File Examples +------------------------------ +Below are configuration examples for the ocs config file and for the +docker compose service. + +OCS Site Config +````````````````` +Here is an example of an agent configuration block for the ocs-site-config +file:: + + {'agent-class': 'SmurfHammerAgent', + 'instance-id': 'smurf-hammer', + 'arguments': []}, + +To suppress the auto-starting monitor process:: + + {'agent-class': 'SmurfHammerAgent', + 'instance-id': 'smurf-hammer', + 'arguments': ['--no-processes']}, + +Description +-------------- + +The agent exposes two operations: + +**hammer** (task) + Resets and reconfigures the specified SMuRF slots by calling + ``sodetlib.hammers.jackhammer.hammer()``. The sequence reboots the + carriers, waits for EPICS connectivity, and runs pysmurf setup. Failures + at each stage are caught per-slot, and the remaining slots continue. The + task is protected by a ``TimeoutLock`` and requires privilege level 2. + +**monitor** (process) + Continuously polls each slot's EPICS registers using ``epics.caget()`` + directly (bypassing pysmurf, so it works even when the system is + degraded). Two registers are queried per slot every 10 seconds: + + - ``AMCc.SmurfApplication.SystemConfigured`` -- whether pysmurf setup + has completed. + - ``AMCc.SmurfApplication.ConfiguringInProgress`` -- whether setup is + currently running. + + Results are published to the ``system_configured`` OCS feed for HK + archival and Grafana. The monitor auto-starts on agent launch unless + ``--no-processes`` is passed. + + Unreachable slots are recorded as ``None`` in session data and ``-1`` + in the feed. + +Agent API +----------- +.. autoclass:: socs.agents.smurf_hammer.agent.SmurfHammerAgent + :members: diff --git a/docs/index.rst b/docs/index.rst index 6cbf27e65..dca8b6a31 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -78,6 +78,7 @@ API Reference Full API documentation for core parts of the SOCS library. agents/rtsp_camera agents/scpi_psu agents/smurf_crate_monitor + agents/smurf_hammer agents/smurf_timing_card agents/srs_cg635 agents/stimulator_encoder diff --git a/socs/agents/smurf_hammer/__init__.py b/socs/agents/smurf_hammer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/socs/agents/smurf_hammer/agent.py b/socs/agents/smurf_hammer/agent.py new file mode 100644 index 000000000..9b73d316c --- /dev/null +++ b/socs/agents/smurf_hammer/agent.py @@ -0,0 +1,257 @@ +import argparse +import os +import time +import traceback + +import epics +import txaio +from ocs import ocs_agent, site_config +from ocs.ocs_twisted import TimeoutLock + + +class SmurfHammerAgent: + """Agent to execute the sodetlib jackhammer hammer sequence and + monitor the configured status of each SMuRF slot. + + Parameters + ---------- + agent : OCSAgent + OCSAgent object which forms this Agent. + """ + + def __init__(self, agent): + self.agent = agent + self.log = agent.log + self.lock = TimeoutLock() + self._monitor_running = False + + # get slots from config file + from sodetlib.hammers.jackhammer import sys_config + self.slot_order = list(sys_config['slot_order']) + + self.agent.register_feed('system_configured', + record=True, + buffer_time=0) + + @ocs_agent.param('slots', default=None) + @ocs_agent.param('no_reboot', default=False, type=bool) + @ocs_agent.param('dump_logs', default=False, type=bool) + @ocs_agent.param('skip_setup', default=False, type=bool) + @ocs_agent.param('dump_rogue', default=False, type=bool) + def hammer(self, session, params): + """hammer(slots=None, no_reboot=False, dump_logs=False, \ + skip_setup=False, dump_rogue=False) + + **Task** - Execute the jackhammer hammer sequence to reset and + reconfigure SMuRF slots. This replicates the ``jackhammer hammer`` + CLI command. + + Individual slot failures are isolated so that the remaining + slots can still complete successfully. + + Parameters + ---------- + slots : list of int, optional + Slot numbers to hammer. Defaults to all slots defined in + the sys_config. + no_reboot : bool + If True, perform a soft reset without rebooting the carriers. + dump_logs : bool + If True, dump docker logs before hammering. + skip_setup : bool + If True, skip pysmurf setup after reboot. + dump_rogue : bool + If True, dump the rogue tree before hammering. + + Notes + ----- + The session data object reports per-slot results:: + + >>> response.session['data'] + {'slots': [2, 3], + 'reboot': True, + 'succeeded_slots': [2], + 'failed_slots': {3: 'EPICS connection timed out ...'}, + 'error': None} + """ + from sodetlib.hammers.jackhammer import hammer + + with self.lock.acquire_timeout(10, job='hammer') as acquired: + if not acquired: + return False, "Could not acquire lock" + + session.data = { + 'slots': params.get('slots'), + 'reboot': not params['no_reboot'], + 'succeeded_slots': [], + 'failed_slots': {}, + 'error': None, + 'timestamp': time.time(), + } + + try: + result = hammer( + slots=params['slots'], + no_reboot=params['no_reboot'], + no_dump=not params['dump_logs'], + skip_setup=params['skip_setup'], + dump_rogue=params['dump_rogue'], + ) + except Exception as e: + self.log.error("Hammer failed: {error}", error=e) + session.data['error'] = traceback.format_exc() + return False, f"Hammer failed: {e}" + + session.data['succeeded_slots'] = result['succeeded'] + session.data['failed_slots'] = result['failed'] + + succeeded = result['succeeded'] + failed = result['failed'] + + if not succeeded: + return False, f"All slots failed: {failed}" + if failed: + return True, ( + f"Partial success: slots {succeeded} succeeded, " + f"slots {list(failed.keys())} failed" + ) + return True, f"Successfully hammered slots {succeeded}" + + @ocs_agent.param('_') + def monitor(self, session, params): + """monitor() + + **Process** - Continuously monitor the configuration status of + each SMuRF slot by querying two EPICS registers: + + - ``AMCc.SmurfApplication.SystemConfigured`` — whether setup + has completed successfully. + - ``AMCc.SmurfApplication.ConfiguringInProgress`` — whether + setup is currently running. + + Notes + ----- + The session data object reports per-slot status:: + + >>> response.session['data'] + {'timestamp': 1721234567.0, + 'slots': { + 2: {'configured': True, 'configuring': False}, + 3: {'configured': False, 'configuring': True}, + 5: {'configured': None, 'configuring': None}, + }} + + For each register, ``True``/``False`` reflect the register + value and ``None`` means the EPICS query timed out (slot + unreachable). + """ + self._monitor_running = True + session.data = {} + + while self._monitor_running: + slot_status = {} + start = time.time() + feed_data = { + 'block_name': 'system_configured', + 'timestamp': start, + 'data': {}, + } + + pv_names = [] + for slot in self.slot_order: + epics_root = f'smurf_server_s{slot}' + pv_names.append(f'{epics_root}:AMCc:SmurfApplication:SystemConfigured') + pv_names.append(f'{epics_root}:AMCc:SmurfApplication:ConfiguringInProgress') + + try: + values = epics.caget_many(pv_names, timeout=5, connection_timeout=5) + except Exception as e: + self.log.error(f"EPICS query failed with: {e}") + values = [None] * len(pv_names) + + for i, slot in enumerate(self.slot_order): + val_configured = values[2 * i] + val_configuring = values[2 * i + 1] + + if val_configured is None: + configured = None + feed_data['data'][f'configured_s{slot}'] = -1 + else: + configured = bool(int(val_configured)) + feed_data['data'][f'configured_s{slot}'] = int(configured) + + if val_configuring is None: + configuring = None + feed_data['data'][f'configuring_s{slot}'] = -1 + else: + configuring = bool(int(val_configuring)) + feed_data['data'][f'configuring_s{slot}'] = int(configuring) + + slot_status[slot] = { + 'configured': configured, + 'configuring': configuring, + } + + session.data = { + 'timestamp': feed_data['timestamp'], + 'slots': slot_status, + } + + if feed_data['data']: + self.agent.publish_to_feed('system_configured', feed_data) + + self.log.debug( + "Slot status: {status}", status=slot_status + ) + + # aim for 10s between samples + wait = 10 - (time.time() - start) + if wait > 0: + time.sleep(wait) + + return True, 'Monitor exited cleanly.' + + def _stop_monitor(self, session, params): + self._monitor_running = False + session.set_status('stopping') + return True, 'Stopping monitor.' + + +def add_agent_args(parser_in=None): + if parser_in is None: + parser_in = argparse.ArgumentParser() + pgroup = parser_in.add_argument_group('Agent Options') + pgroup.add_argument('--no-processes', action='store_true', + default=False, + help="Do not auto-start the monitor process.") + return parser_in + + +def main(args=None): + # set up logging + txaio.use_twisted() + txaio.start_logging(level=os.environ.get("LOGLEVEL", "info")) + + parser = add_agent_args() + args = site_config.parse_args(agent_class='SmurfHammerAgent', + parser=parser, + args=args) + + startup = not args.no_processes + + agent, runner = ocs_agent.init_site_agent(args) + p = SmurfHammerAgent(agent) + + agent.register_process('monitor', + p.monitor, + p._stop_monitor, + blocking=True, + startup=startup) + + agent.register_task('hammer', p.hammer) + + runner.run(agent, auto_reconnect=True) + + +if __name__ == "__main__": + main() diff --git a/socs/plugin.py b/socs/plugin.py index 9f7873e3b..18a6a485b 100644 --- a/socs/plugin.py +++ b/socs/plugin.py @@ -41,6 +41,7 @@ 'RTSPCameraAgent': {'module': 'socs.agents.rtsp_camera.agent', 'entry_point': 'main'}, 'ScpiPsuAgent': {'module': 'socs.agents.scpi_psu.agent', 'entry_point': 'main'}, 'SmurfFileEmulator': {'module': 'socs.agents.smurf_file_emulator.agent', 'entry_point': 'main'}, + 'SmurfHammerAgent': {'module': 'socs.agents.smurf_hammer.agent', 'entry_point': 'main'}, 'SmurfStreamSimulator': {'module': 'socs.agents.smurf_stream_simulator.agent', 'entry_point': 'main'}, 'SmurfTimingCardAgent': {'module': 'socs.agents.smurf_timing_card.agent', 'entry_point': 'main'}, 'SRSCG635Agent': {'module': 'socs.agents.srs_cg635.agent', 'entry_point': 'main'},