From 649074e22d5ef0a19d2182071ec74efcc58bc46a Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Thu, 12 Nov 2026 14:13:56 +0000 Subject: [PATCH 01/22] add connection state and disconnect_on decorator --- src/fastcs/connections/__init__.py | 2 + src/fastcs/connections/connection.py | 112 ++++++++++++++++++++++++++- tests/test_connection.py | 64 +++++++++++++++ 3 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 tests/test_connection.py diff --git a/src/fastcs/connections/__init__.py b/src/fastcs/connections/__init__.py index fa4c5653..f84d26d7 100644 --- a/src/fastcs/connections/__init__.py +++ b/src/fastcs/connections/__init__.py @@ -3,7 +3,9 @@ from fastcs.util import optional_dependency from .connection import Connection as Connection +from .connection import ConnectionState as ConnectionState from .connection import DisconnectedError as DisconnectedError +from .connection import disconnect_on as disconnect_on from .ip_connection import IPConnection as IPConnection with optional_dependency("httpx"): diff --git a/src/fastcs/connections/connection.py b/src/fastcs/connections/connection.py index 54dbefa0..ac1abf6b 100644 --- a/src/fastcs/connections/connection.py +++ b/src/fastcs/connections/connection.py @@ -1,8 +1,11 @@ from __future__ import annotations +import asyncio +import functools +import inspect from abc import ABC, abstractmethod -from collections.abc import Sequence -from typing import ClassVar, Generic, cast +from collections.abc import Callable, Coroutine, Sequence +from typing import Any, ClassVar, Generic, cast, overload from typing_extensions import TypeVar @@ -14,6 +17,48 @@ class DisconnectedError(ConnectionError): """Raised when using a connection whose link isn't open""" +class ConnectionState: + """Whether a connection's link is up, set by ``@disconnect_on`` and the framework. + + The framework gives each connection it opens its own live state. It starts up. + """ + + def __init__(self) -> None: + self._up = asyncio.Event() + self._up.set() + self._down = asyncio.Event() + + @property + def up(self) -> bool: + """Whether the link is up""" + return self._up.is_set() + + def mark_down(self) -> None: + """Mark the link down, waking anything waiting for that""" + self._up.clear() + self._down.set() + + def mark_up(self) -> None: + """Mark the link up, waking anything waiting for that""" + self._down.clear() + self._up.set() + + async def wait_down(self) -> None: + """Wait until the link is marked down""" + await self._down.wait() + + async def wait_up(self) -> None: + """Wait until the link is marked up""" + await self._up.wait() + + +class _InertConnectionState(ConnectionState): + """The state of a connection the framework hasn't opened, which is always up""" + + def mark_down(self) -> None: + pass + + class Connection(ABC, Generic[IntrospectionT]): """A link to a device, which can optionally introspect it. @@ -26,6 +71,9 @@ async def introspect(self) -> EigerIntrospection: ... # Connection type(s) in the same ControllerSpec to open before this one depends_on: ClassVar[type[Connection] | Sequence[type[Connection]]] = () + # Replaced with a live state by the framework when it opens this connection + state: ConnectionState = _InertConnectionState() + _introspection: IntrospectionT @property @@ -67,3 +115,63 @@ async def introspect(self) -> IntrospectionT: @abstractmethod async def close(self) -> None: """Close the link. Called by the framework on stop, or if build fails.""" + + +# What every @disconnect_on treats as the link going down, as well as its errors +_ALWAYS_DISCONNECT_ON: tuple[type[BaseException], ...] = ( + ConnectionError, + DisconnectedError, +) + +_MethodT = TypeVar("_MethodT", bound=Callable[..., Coroutine[Any, Any, Any]]) + + +@overload +def disconnect_on(method: _MethodT, /) -> _MethodT: ... + + +@overload +def disconnect_on( + *errors: type[BaseException], +) -> Callable[[_MethodT], _MethodT]: ... + + +def disconnect_on(*errors: Any) -> Any: + """Mark a connection IO method whose failure means the link has gone. + + If the method raises one of ``errors``, a `ConnectionError` or a + `DisconnectedError`, the connection's ``state`` is marked down and the exception + is re-raised. If it is called while the state is down, it raises + `DisconnectedError` without calling the method. + + Example:: + + class EigerConnection(Connection): + @disconnect_on(TimeoutError) + async def get(self, key: str) -> Any: ... + + Args: + errors: Other exceptions that mean the link has gone + + """ + if len(errors) == 1 and inspect.iscoroutinefunction(errors[0]): + # Used bare, as @disconnect_on, so the only argument is the method itself + return disconnect_on()(errors[0]) + + link_errors = (*_ALWAYS_DISCONNECT_ON, *errors) + + def decorator(method: _MethodT) -> _MethodT: + @functools.wraps(method) + async def wrapper(self: Connection, *args: Any, **kwargs: Any) -> Any: + if not self.state.up: + raise DisconnectedError(f"{type(self).__name__} is disconnected") + + try: + return await method(self, *args, **kwargs) + except link_errors: + self.state.mark_down() + raise + + return cast(_MethodT, wrapper) + + return decorator diff --git a/tests/test_connection.py b/tests/test_connection.py new file mode 100644 index 00000000..c90d5452 --- /dev/null +++ b/tests/test_connection.py @@ -0,0 +1,64 @@ +import pytest + +from fastcs.connections import ( + Connection, + ConnectionState, + DisconnectedError, + disconnect_on, +) + + +class _FlakyConnection(Connection): + def __init__(self) -> None: + self.calls = 0 + + async def connect(self) -> None: + pass + + async def close(self) -> None: + pass + + @disconnect_on(TimeoutError) + async def get(self, error: Exception | None = None) -> int: + self.calls += 1 + if error is not None: + raise error + return self.calls + + @disconnect_on + async def send(self) -> None: + raise ConnectionResetError() + + +@pytest.mark.asyncio +async def test_listed_error_marks_down_and_gates_later_calls(): + connection = _FlakyConnection() + connection.state = ConnectionState() + + with pytest.raises(ValueError): + await connection.get(ValueError()) + assert connection.state.up + + with pytest.raises(TimeoutError): + await connection.get(TimeoutError()) + assert not connection.state.up + + with pytest.raises(DisconnectedError): + await connection.get() + assert connection.calls == 2 + + connection.state.mark_up() + assert await connection.get() == 3 + + with pytest.raises(ConnectionResetError): + await connection.get(ConnectionResetError()) + assert not connection.state.up + + +@pytest.mark.asyncio +async def test_bare_decorator_and_inert_state(): + connection = _FlakyConnection() + + with pytest.raises(ConnectionResetError): + await connection.send() + assert connection.state.up From 0ff9d0f3d848fd66605b2917471850484623b81d Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Thu, 12 Nov 2026 14:21:09 +0000 Subject: [PATCH 02/22] update builtin connections with a @disconnect_on decorator --- src/fastcs/connections/http_connection.py | 8 +++++++- src/fastcs/connections/ip_connection.py | 11 ++++++++++- src/fastcs/connections/serial_connection.py | 11 ++++++++++- tests/test_http_connection.py | 6 ++++-- tests/test_ip_connection.py | 6 ++++-- tests/test_serial_connection.py | 6 ++++-- 6 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/fastcs/connections/http_connection.py b/src/fastcs/connections/http_connection.py index f7aea021..b3662299 100644 --- a/src/fastcs/connections/http_connection.py +++ b/src/fastcs/connections/http_connection.py @@ -6,7 +6,12 @@ import httpx -from fastcs.connections.connection import Connection, DisconnectedError, IntrospectionT +from fastcs.connections.connection import ( + Connection, + DisconnectedError, + IntrospectionT, + disconnect_on, +) from fastcs.tracer import Tracer @@ -165,6 +170,7 @@ async def close(self) -> None: self._client = None await client.aclose() + @disconnect_on(TimeoutError) async def _send(self, method: str, path: str, **kwargs: Any) -> httpx.Response: client = self._client if client is None: diff --git a/src/fastcs/connections/ip_connection.py b/src/fastcs/connections/ip_connection.py index 587e0aa4..d9fb8ed2 100644 --- a/src/fastcs/connections/ip_connection.py +++ b/src/fastcs/connections/ip_connection.py @@ -2,7 +2,12 @@ from collections.abc import AsyncGenerator from contextlib import asynccontextmanager -from fastcs.connections.connection import Connection, DisconnectedError, IntrospectionT +from fastcs.connections.connection import ( + Connection, + DisconnectedError, + IntrospectionT, + disconnect_on, +) from fastcs.tracer import Tracer @@ -62,6 +67,7 @@ async def connect(self) -> None: self.ip, self.port, limit=self.limit ) + @disconnect_on(OSError, asyncio.IncompleteReadError) async def write(self, message: str) -> None: """Send a message, adding the terminator. @@ -78,6 +84,7 @@ async def write(self, message: str) -> None: self.log_event("Sent message", message=message) + @disconnect_on(OSError, asyncio.IncompleteReadError) async def query(self, message: str) -> str: """Send a message and read the reply. @@ -107,6 +114,7 @@ async def query(self, message: str) -> str: self.log_event("Received query response", query=message, response=response) return response + @disconnect_on(OSError, asyncio.IncompleteReadError) async def write_bytes(self, data: bytes) -> None: """Send raw bytes, without a terminator. @@ -122,6 +130,7 @@ async def write_bytes(self, data: bytes) -> None: self.log_event("Sent bytes", data=data) + @disconnect_on(OSError, asyncio.IncompleteReadError) async def query_bytes(self, data: bytes, size: int) -> bytes: """Send raw bytes and read a reply of a fixed size. diff --git a/src/fastcs/connections/serial_connection.py b/src/fastcs/connections/serial_connection.py index 367f1884..d2804354 100644 --- a/src/fastcs/connections/serial_connection.py +++ b/src/fastcs/connections/serial_connection.py @@ -6,7 +6,12 @@ import aioserial import serial -from fastcs.connections.connection import Connection, DisconnectedError, IntrospectionT +from fastcs.connections.connection import ( + Connection, + DisconnectedError, + IntrospectionT, + disconnect_on, +) from fastcs.tracer import Tracer @@ -87,6 +92,7 @@ async def connect(self) -> None: exclusive=True, ) + @disconnect_on(OSError) async def write(self, message: str) -> None: """Send a message, adding the terminator. @@ -102,6 +108,7 @@ async def write(self, message: str) -> None: self.log_event("Sent message", message=message) + @disconnect_on(OSError) async def query(self, message: str) -> str: """Send a message and read the reply. @@ -129,6 +136,7 @@ async def query(self, message: str) -> str: self.log_event("Received query response", query=message, response=response) return response + @disconnect_on(OSError) async def write_bytes(self, data: bytes) -> None: """Send raw bytes, without a terminator. @@ -143,6 +151,7 @@ async def write_bytes(self, data: bytes) -> None: self.log_event("Sent bytes", data=data) + @disconnect_on(OSError) async def query_bytes(self, data: bytes, size: int) -> bytes: """Send raw bytes and read a reply of a fixed size. diff --git a/tests/test_http_connection.py b/tests/test_http_connection.py index cd4ec88c..6bc33c88 100644 --- a/tests/test_http_connection.py +++ b/tests/test_http_connection.py @@ -8,7 +8,7 @@ import pytest_asyncio import fastcs.connections -from fastcs.connections import DisconnectedError, HTTPConnection +from fastcs.connections import ConnectionState, DisconnectedError, HTTPConnection class _RateDevice(BaseHTTPRequestHandler): @@ -76,14 +76,16 @@ async def test_connect_when_connected_raises(connection: HTTPConnection): @pytest.mark.asyncio -async def test_link_failure_closes_connection( +async def test_link_failure_marks_down_and_closes_connection( server: ThreadingHTTPServer, connection: HTTPConnection ): + connection.state = ConnectionState() server.shutdown() server.server_close() with pytest.raises(ConnectionError): await connection.get("/rate") + assert not connection.state.up with pytest.raises(DisconnectedError): await connection.get("/rate") diff --git a/tests/test_ip_connection.py b/tests/test_ip_connection.py index dff8ac1e..f127caaf 100644 --- a/tests/test_ip_connection.py +++ b/tests/test_ip_connection.py @@ -3,7 +3,7 @@ import pytest import pytest_asyncio -from fastcs.connections import DisconnectedError, IPConnection +from fastcs.connections import ConnectionState, DisconnectedError, IPConnection async def _reply_to_r(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): @@ -40,9 +40,11 @@ async def test_connect_when_connected_raises(connection: IPConnection): @pytest.mark.asyncio -async def test_timeout_closes_connection(connection: IPConnection): +async def test_timeout_marks_down_and_closes_connection(connection: IPConnection): + connection.state = ConnectionState() with pytest.raises(TimeoutError): await connection.query("NO_REPLY?") + assert not connection.state.up with pytest.raises(DisconnectedError): await connection.write("R=5") diff --git a/tests/test_serial_connection.py b/tests/test_serial_connection.py index a5fc91c0..97816181 100644 --- a/tests/test_serial_connection.py +++ b/tests/test_serial_connection.py @@ -5,7 +5,7 @@ import pytest import pytest_asyncio -from fastcs.connections import DisconnectedError, SerialConnection +from fastcs.connections import ConnectionState, DisconnectedError, SerialConnection def _reply_to_r(device: int): @@ -50,9 +50,11 @@ async def test_connect_when_connected_raises(connection: SerialConnection): @pytest.mark.asyncio -async def test_timeout_closes_connection(connection: SerialConnection): +async def test_timeout_marks_down_and_closes_connection(connection: SerialConnection): + connection.state = ConnectionState() with pytest.raises(TimeoutError): await connection.query("NO_REPLY?") + assert not connection.state.up with pytest.raises(DisconnectedError): await connection.write("R=5") From 2727833e89767cc47e5d3e6e0331b93fe38617e3 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Thu, 12 Nov 2026 14:56:40 +0000 Subject: [PATCH 03/22] remove old controller connection methods, and add new connected attribute on init --- src/fastcs/controllers/base_controller.py | 45 ++++++++++++++++++----- src/fastcs/controllers/controller.py | 23 ------------ src/fastcs/demo/controllers.py | 11 ------ src/fastcs/runner.py | 40 ++++++++++++-------- tests/test_controller_runner.py | 20 ++++++++++ tests/test_controllers.py | 38 ++++++------------- 6 files changed, 91 insertions(+), 86 deletions(-) diff --git a/src/fastcs/controllers/base_controller.py b/src/fastcs/controllers/base_controller.py index 7307359a..f74707ea 100755 --- a/src/fastcs/controllers/base_controller.py +++ b/src/fastcs/controllers/base_controller.py @@ -13,6 +13,7 @@ from fastcs.attributes import AnyAttributeIO, Attribute, AttrR, AttrW, HintedAttribute from fastcs.controllers.controller_api import ControllerAPI +from fastcs.datatypes import Bool from fastcs.logging import logger from fastcs.methods import Command, Method, Scan, UnboundCommand, UnboundScan from fastcs.tracer import Tracer @@ -35,10 +36,14 @@ class BaseController(Tracer): # These class attributes can be overridden on child classes to define default # behaviour of instantiated controllers root_attribute: Attribute | None = None + # Added to every controller by the framework in __init__ + connected: AttrR[bool] description: str | None = None _sealed: bool = False _registered: bool = False + # Whether the name dicts exist, so assignments can be checked against them + _names_ready: bool = False def __init__( self, @@ -59,6 +64,7 @@ def __init__( self.__sub_controllers: dict[str, BaseController] = {} self.__command_methods: dict[str, Command] = {} self.__scan_methods: dict[str, Scan] = {} + self._names_ready = True self.__hinted_attributes: dict[str, HintedAttribute] = {} self.__hinted_methods: dict[str, type[Method]] = {} @@ -67,6 +73,16 @@ def __init__( self._bind_attrs() + # Added to every controller by the framework, so drivers can't use the name + self.add_attribute( + "connected", + AttrR( + Bool(), + initial_value=True, + description="Whether its connections are up", + ), + ) + ios = ios or [] self._attribute_ref_io_map = {io.ref_type: io for io in ios} self._validate_io(ios) @@ -186,6 +202,8 @@ def __setattr__(self, name, value): elif isinstance(value, BaseController): self.add_sub_controller(name, value) else: + if self._names_ready: + self._check_name_free(name, value) super().__setattr__(name, value) async def setup(self): @@ -194,8 +212,8 @@ async def setup(self): pass def seal(self): - """Validate and connect this controller and its sub controllers, then stop - anything being added to them. Called by the framework after construction. + """Validate and connect this controller, then stop anything being added to it. + Called by the framework on every controller in the tree after construction. Raises: RuntimeError: If the controller is already sealed @@ -207,8 +225,6 @@ def seal(self): self._validate_type_hints() self._connect_attribute_ios() self._sealed = True - for controller in self.sub_controllers.values(): - controller.seal() def _validate_type_hints(self): """Validate all type-hints were introspected""" @@ -317,6 +333,19 @@ def _check_can_add(self, name: str, member: Attribute | BaseController | Method) RuntimeError: If the controller has been sealed ValueError: If ``name`` is already used by another member + """ + if self._sealed: + raise RuntimeError( + f"Cannot add {member} as {name} to {self} after it has been sealed" + ) + self._check_name_free(name, member) + + def _check_name_free(self, name: str, member: object): + """Check that ``name`` is not already used by a member of this controller. + + Raises: + ValueError: If ``name`` is already used by another member + """ namespaces = { Attribute: ("attribute", self.__attributes), @@ -325,14 +354,10 @@ def _check_can_add(self, name: str, member: Attribute | BaseController | Method) Command: ("command method", self.__command_methods), } kind = next( - kind for cls, (kind, _) in namespaces.items() if isinstance(member, cls) + (kind for cls, (kind, _) in namespaces.items() if isinstance(member, cls)), + "value", ) - if self._sealed: - raise RuntimeError( - f"Cannot add {kind} {name} to {self} after it has been sealed" - ) - for existing_kind, namespace in namespaces.values(): if name in namespace: raise ValueError( diff --git a/src/fastcs/controllers/controller.py b/src/fastcs/controllers/controller.py index aae91fd2..a6241172 100755 --- a/src/fastcs/controllers/controller.py +++ b/src/fastcs/controllers/controller.py @@ -13,8 +13,6 @@ def __init__( ios: Sequence[AnyAttributeIO] | None = None, ) -> None: super().__init__(description=description, ios=ios) - # Connections are open before any controller is constructed - self._connected = True def add_sub_controller(self, name: str, sub_controller: BaseController): if name.isdigit(): @@ -23,24 +21,3 @@ def add_sub_controller(self, name: str, sub_controller: BaseController): "Numeric-only names are not allowed; use ControllerVector instead" ) return super().add_sub_controller(name, sub_controller) - - @property - def connected(self) -> bool: - """Whether the controller is connected and its scan tasks should run""" - return self._connected - - def mark_disconnected(self) -> None: - """Mark the controller disconnected, pausing its scan tasks until reconnect""" - self._connected = False - - async def reconnect(self): - """Hook to reconnect to device after an error - - This should set ``_connected`` to ``True`` if the connection was successful to - enable scan tasks. - - If the connection cannot be re-established it should log an error with the - reason. It should not raise an exception. - - """ - self._connected = True diff --git a/src/fastcs/demo/controllers.py b/src/fastcs/demo/controllers.py index 4d0f16e0..837799e4 100755 --- a/src/fastcs/demo/controllers.py +++ b/src/fastcs/demo/controllers.py @@ -10,7 +10,6 @@ from fastcs.connections import IPConnection from fastcs.controllers import Controller from fastcs.datatypes import Enum, Float, Int, Waveform -from fastcs.logging import logger from fastcs.methods import command, scan NumberT = TypeVar("NumberT", int, float) @@ -91,16 +90,6 @@ async def cancel_all(self) -> None: # TODO: The requests all get concatenated and the sim doesn't handle it await asyncio.sleep(0.1) - async def reconnect(self): - try: - await self.sensor.close() - await self.sensor.connect() - except BaseException: - logger.exception("Reconnect failed") - return - - self._connected = True - @scan(0.1) async def update_voltages(self): query = "V?" diff --git a/src/fastcs/runner.py b/src/fastcs/runner.py index dc0b6098..c6d4932f 100644 --- a/src/fastcs/runner.py +++ b/src/fastcs/runner.py @@ -21,6 +21,10 @@ class ControllerRunner: def __init__(self) -> None: self._connections: list[Connection] = [] self._controllers: list[Controller] = [] + # id of each connection -> the controllers that hold it, in tree order + self._controllers_by_connection: dict[int, list[BaseController]] = defaultdict( + list + ) self._controller_apis: list[ControllerAPI] = [] self._initial_coros: list[ScanCallback] = [] @@ -54,8 +58,7 @@ async def build(cls, specs: list[ControllerSpec]) -> Self: connection.introspection = await connection.introspect() runner._controllers = [spec.construct() for spec in specs] - for controller in runner._controllers: - controller.seal() + runner._seal() for controller in runner._controllers: api = controller.build_api(controller.path) @@ -97,6 +100,18 @@ async def stop(self) -> None: await asyncio.gather(*tasks, return_exceptions=True) await self._close_connections() + def _seal(self) -> None: + """Seal every controller, recording which controllers hold each connection.""" + opened = {id(connection) for connection in self._connections} + for controller in self._controllers: + for sub_controller in _walk(controller): + sub_controller.seal() + for connection in _held_connections(sub_controller): + if id(connection) in opened: + self._controllers_by_connection[id(connection)].append( + sub_controller + ) + async def _close_connections(self) -> None: """Close the open connections, in reverse order.""" while self._connections: @@ -136,8 +151,7 @@ def _collect_scans( ) scan_coros = [ - _periodic_scan_coro(controller, period, scans) - for period, scans in scan_dict.items() + _periodic_scan_coro(period, scans) for period, scans in scan_dict.items() ] return initial_coros, scan_coros @@ -159,6 +173,12 @@ def _walk(controller: BaseController) -> Iterator[BaseController]: yield from _walk(sub_controller) +def _held_connections(controller: BaseController) -> list[Connection]: + """The connections stored as attributes of a controller, once each""" + values = vars(controller).values() + return list({id(v): v for v in values if isinstance(v, Connection)}.values()) + + def _check_connections_not_shared(specs: list[ControllerSpec]) -> None: """Check that no connection is used by more than one spec. @@ -220,13 +240,10 @@ def visit(connection: Connection) -> None: return ordered -def _periodic_scan_coro( - controller: Controller, period: float, scans: Sequence[ScanCallback] -) -> ScanCallback: +def _periodic_scan_coro(period: float, scans: Sequence[ScanCallback]) -> ScanCallback: """Create a coroutine to run scans at a given period Args: - controller: The top-level controller whose connection gates the scans period: The period to run the scans at scans: A list of `ScanCallback` to run periodically @@ -237,17 +254,10 @@ def _periodic_scan_coro( async def scan_coro() -> None: while True: - if not controller.connected: - await asyncio.sleep(1) - continue - try: await asyncio.gather(asyncio.sleep(period), *[scan() for scan in scans]) except Exception: logger.exception("Exception in scan task", period=period) - controller.mark_disconnected() - await asyncio.sleep(1) - logger.error("Pausing scan tasks and waiting for reconnect") return scan_coro diff --git a/tests/test_controller_runner.py b/tests/test_controller_runner.py index d4e9b6ec..de3359bc 100644 --- a/tests/test_controller_runner.py +++ b/tests/test_controller_runner.py @@ -272,6 +272,26 @@ def test_introspection_set_twice_raises(): connection.introspection = DeviceIntrospection({"gain": 3}) +@pytest.mark.asyncio +async def test_build_maps_each_connection_to_the_controllers_holding_it(): + class Holder(Controller): + def __init__(self, connection: Connection) -> None: + super().__init__() + self.connection = connection + + class Parent(Controller): + def __init__(self, connection: Connection) -> None: + super().__init__() + self.A = Holder(connection) + self.B = Holder(connection) + + connection = RecordingConnection([]) + runner = await ControllerRunner.build([ControllerSpec("P", Parent, connection)]) + + holders = list(runner.controllers[0].sub_controllers.values()) + assert runner._controllers_by_connection[id(connection)] == holders + + @pytest.mark.asyncio async def test_controller_tree_sealed_after_build(): class ParentController(Controller): diff --git a/tests/test_controllers.py b/tests/test_controllers.py index e1bb4034..0e3a0c91 100644 --- a/tests/test_controllers.py +++ b/tests/test_controllers.py @@ -1,13 +1,11 @@ -import asyncio import enum import pytest from fastcs.attributes import AttrR, AttrRW from fastcs.controllers import Controller, ControllerSpec, ControllerVector -from fastcs.datatypes import Enum, Float, Int +from fastcs.datatypes import Bool, Enum, Float, Int from fastcs.methods import Command, Scan, command, scan -from fastcs.runner import ControllerRunner def test_controller_nesting(): @@ -80,6 +78,7 @@ def test_attribute_parsing(): controller = SomeController(sub_controller) assert set(controller.attributes.keys()) == { + "connected", "_attributes_attr", "attr_on_object", "_attributes_attr_equal", @@ -95,6 +94,7 @@ def test_attribute_parsing(): ) assert sub_controller.attributes == { + "connected": sub_controller.connected, "sub_attribute": sub_controller.sub_attribute, } @@ -273,32 +273,16 @@ async def scan_nothing(self): api = controller.build_api([]) assert api.description == controller.description - assert list(api.attributes) == ["attr1", "attr2"] + assert list(api.attributes) == ["attr1", "connected", "attr2"] assert list(api.command_methods) == ["do_nothing"] assert list(api.scan_methods) == ["scan_nothing"] -@pytest.mark.asyncio -async def test_scan_exception_sets_disconnected_and_reconnect_resumes(): - class MyTestController(Controller): - @scan(0.01) - async def failing_scan(self): - raise RuntimeError("scan error") - - runner = await ControllerRunner.build([ControllerSpec("A", MyTestController)]) - controller = runner.controllers[0] - - controller._connected = True - task = asyncio.create_task(runner._scan_coros[0]()) - - # Wait long enough for the scan to run and raise, marking it disconnected - await asyncio.sleep(0.1) - assert not controller.connected - - # Trigger reconnect - connected resumes scan tasks - await controller.reconnect() - assert controller.connected +def test_connected_attribute_added_and_name_reserved(): + controller = Controller() + assert controller.connected.get() is True - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task + with pytest.raises(ValueError, match="Cannot add attribute"): + controller.add_attribute("connected", AttrR(Bool())) + with pytest.raises(ValueError, match="has existing attribute connected"): + controller.connected = False # pyright: ignore[reportAttributeAccessIssue] From 6013ff9a297d905f1e1fd6c2e49bd79f6c95fead Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Thu, 12 Nov 2026 15:16:21 +0000 Subject: [PATCH 04/22] add a gate to scans that is set on disconnect, group scans by controller --- src/fastcs/demo/controllers.py | 1 + src/fastcs/runner.py | 74 +++++++++++++++++---------------- tests/test_controller_runner.py | 30 +++++++++++++ 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/src/fastcs/demo/controllers.py b/src/fastcs/demo/controllers.py index 837799e4..28a536ec 100755 --- a/src/fastcs/demo/controllers.py +++ b/src/fastcs/demo/controllers.py @@ -122,3 +122,4 @@ def __init__(self, index: int, sensor: IPConnection) -> None: super().__init__( f"Ramp{suffix}", ios=[TemperatureControllerAttributeIO(sensor, suffix)] ) + self.sensor = sensor diff --git a/src/fastcs/runner.py b/src/fastcs/runner.py index c6d4932f..f5bbc0bb 100644 --- a/src/fastcs/runner.py +++ b/src/fastcs/runner.py @@ -4,7 +4,6 @@ from typing import Self from fastcs.attributes.attr_r import AttrR -from fastcs.attributes.attribute_io_ref import AttributeIORef from fastcs.connections import Connection from fastcs.controllers.base_controller import BaseController from fastcs.controllers.controller import Controller @@ -29,6 +28,8 @@ def __init__(self) -> None: self._controller_apis: list[ControllerAPI] = [] self._initial_coros: list[ScanCallback] = [] self._scan_coros: list[ScanCallback] = [] + # id of each controller -> its scan gate, set while its scans may run + self._scan_gates: dict[int, asyncio.Event] = {} self._tasks: set[asyncio.Task] = set() @classmethod @@ -61,11 +62,8 @@ async def build(cls, specs: list[ControllerSpec]) -> Self: runner._seal() for controller in runner._controllers: - api = controller.build_api(controller.path) - initial_coros, scan_coros = runner._collect_scans(controller, api) - runner._controller_apis.append(api) - runner._initial_coros.extend(initial_coros) - runner._scan_coros.extend(scan_coros) + runner._controller_apis.append(controller.build_api(controller.path)) + runner._collect_scans(controller) except BaseException: await runner._close_connections() raise @@ -121,39 +119,28 @@ async def _close_connections(self) -> None: except Exception: logger.exception("Exception closing connection", connection=connection) - def _collect_scans( - self, controller: Controller, api: ControllerAPI - ) -> tuple[list[ScanCallback], list[ScanCallback]]: + def _collect_scans(self, controller: Controller) -> None: """Find the initial reads and periodic scans in a controller's tree. - Returns: - The initial coroutines, and one periodic coroutine per period + Each controller in the tree gets its own scan gate, and one periodic + coroutine per period that waits on that gate before each cycle. """ - initial_coros: list[ScanCallback] = [] - scan_dict: dict[float, list[ScanCallback]] = defaultdict(list) - - for sub_api in api.walk_api(): - for method in sub_api.scan_methods.values(): - if method.period is ONCE: - initial_coros.append(method.fn) + for sub_controller in _walk(controller): + scans_by_period: dict[float, list[ScanCallback]] = defaultdict(list) + for period, scan in _scans(sub_controller): + if period is ONCE: + self._initial_coros.append(scan) else: - scan_dict[method.period].append(method.fn) - - for attribute in sub_api.attributes.values(): - match attribute: - case AttrR(_io_ref=AttributeIORef(update_period=update_period)): - if update_period is ONCE: - initial_coros.append(attribute.bind_update_callback()) - elif update_period is not None: - scan_dict[update_period].append( - attribute.bind_update_callback() - ) - - scan_coros = [ - _periodic_scan_coro(period, scans) for period, scans in scan_dict.items() - ] - return initial_coros, scan_coros + scans_by_period[period].append(scan) + + gate = asyncio.Event() + gate.set() + self._scan_gates[id(sub_controller)] = gate + self._scan_coros.extend( + _periodic_scan_coro(period, scans, gate) + for period, scans in scans_by_period.items() + ) def _cancel_tasks(self) -> None: for task in self._tasks: @@ -173,6 +160,19 @@ def _walk(controller: BaseController) -> Iterator[BaseController]: yield from _walk(sub_controller) +def _scans(controller: BaseController) -> Iterator[tuple[float, ScanCallback]]: + """The period and callback of each scan method and polled attribute of a + controller, not including its sub controllers""" + for method in controller.scan_methods.values(): + yield method.period, method.fn + + for attribute in controller.attributes.values(): + if isinstance(attribute, AttrR) and attribute.has_io_ref(): + period = attribute.io_ref.update_period + if period is not None: + yield period, attribute.bind_update_callback() + + def _held_connections(controller: BaseController) -> list[Connection]: """The connections stored as attributes of a controller, once each""" values = vars(controller).values() @@ -240,12 +240,15 @@ def visit(connection: Connection) -> None: return ordered -def _periodic_scan_coro(period: float, scans: Sequence[ScanCallback]) -> ScanCallback: +def _periodic_scan_coro( + period: float, scans: Sequence[ScanCallback], gate: asyncio.Event +) -> ScanCallback: """Create a coroutine to run scans at a given period Args: period: The period to run the scans at scans: A list of `ScanCallback` to run periodically + gate: Waited on before each cycle, so clearing it pauses the scans Returns: A wrapper `ScanCallback` that runs all of the callbacks at a given period @@ -254,6 +257,7 @@ def _periodic_scan_coro(period: float, scans: Sequence[ScanCallback]) -> ScanCal async def scan_coro() -> None: while True: + await gate.wait() try: await asyncio.gather(asyncio.sleep(period), *[scan() for scan in scans]) except Exception: diff --git a/tests/test_controller_runner.py b/tests/test_controller_runner.py index de3359bc..0c299b89 100644 --- a/tests/test_controller_runner.py +++ b/tests/test_controller_runner.py @@ -17,6 +17,7 @@ def __init__(self) -> None: self.scan_count = 0 self.setup_before_read_once = False self.multiple_scans = asyncio.Event() + self.scanned = asyncio.Event() self.started = asyncio.Event() self.stopped = asyncio.Event() @@ -30,6 +31,7 @@ async def read_once(self) -> None: @scan(0.01) async def counter(self) -> None: self.scan_count += 1 + self.scanned.set() if self.scan_count > 1: self.multiple_scans.set() @@ -69,6 +71,34 @@ async def test_start_runs_scans_and_stop_stops_them(): assert controller.stopped.is_set() +async def _wait_for_scans(controller: ScanController, count: int) -> None: + """Wait until the controller's counter scan has run at least ``count`` times""" + while controller.scan_count < count: + controller.scanned.clear() + await controller.scanned.wait() + + +@pytest.mark.asyncio +async def test_clearing_a_scan_gate_pauses_only_that_controllers_scans(): + paused, running = ScanController(), ScanController() + runner = await ControllerRunner.build( + [ControllerSpec("A", lambda: paused), ControllerSpec("B", lambda: running)] + ) + await runner.start() + await asyncio.wait_for(_wait_for_scans(paused, 2), timeout=1) + + runner._scan_gates[id(paused)].clear() + # Same period, so two cycles of the other let a paused cycle under way finish + await asyncio.wait_for(_wait_for_scans(running, running.scan_count + 2), timeout=1) + paused_count = paused.scan_count + await asyncio.wait_for(_wait_for_scans(running, running.scan_count + 3), timeout=1) + assert paused.scan_count == paused_count + + runner._scan_gates[id(paused)].set() + await asyncio.wait_for(_wait_for_scans(paused, paused_count + 1), timeout=1) + await runner.stop() + + class RecordingConnection(Connection): def __init__(self, events: list[str], name: str = "") -> None: self.events = events From a54c67cb385721126ec4932168792f85dd24b562 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Thu, 12 Nov 2026 15:34:20 +0000 Subject: [PATCH 05/22] check connection holders, and implement reconnect logic --- src/fastcs/connections/connection.py | 4 + src/fastcs/runner.py | 149 +++++++++++++++++++++++---- tests/test_control_system.py | 3 +- tests/test_controller_runner.py | 116 +++++++++++++++++++++ 4 files changed, 252 insertions(+), 20 deletions(-) diff --git a/src/fastcs/connections/connection.py b/src/fastcs/connections/connection.py index ac1abf6b..6423147c 100644 --- a/src/fastcs/connections/connection.py +++ b/src/fastcs/connections/connection.py @@ -74,6 +74,10 @@ async def introspect(self) -> EigerIntrospection: ... # Replaced with a live state by the framework when it opens this connection state: ConnectionState = _InertConnectionState() + # How many times, and how many seconds apart, the framework tries to reconnect + reconnect_attempts: int = 10 + reconnect_period: float = 5.0 + _introspection: IntrospectionT @property diff --git a/src/fastcs/runner.py b/src/fastcs/runner.py index f5bbc0bb..26bf66e4 100644 --- a/src/fastcs/runner.py +++ b/src/fastcs/runner.py @@ -4,7 +4,7 @@ from typing import Self from fastcs.attributes.attr_r import AttrR -from fastcs.connections import Connection +from fastcs.connections import Connection, ConnectionState from fastcs.controllers.base_controller import BaseController from fastcs.controllers.controller import Controller from fastcs.controllers.controller_api import ControllerAPI @@ -25,12 +25,18 @@ def __init__(self) -> None: list ) + # id of each connection -> the connections it waits for when reconnecting + self._dependencies: dict[int, list[Connection]] = {} + self._controller_apis: list[ControllerAPI] = [] - self._initial_coros: list[ScanCallback] = [] + # id of each controller -> its read-once reads + self._initial_coros: dict[int, list[ScanCallback]] = defaultdict(list) self._scan_coros: list[ScanCallback] = [] # id of each controller -> its scan gate, set while its scans may run self._scan_gates: dict[int, asyncio.Event] = {} self._tasks: set[asyncio.Task] = set() + # id of each connection -> its watcher + self._watchers: dict[int, asyncio.Task] = {} @classmethod async def build(cls, specs: list[ControllerSpec]) -> Self: @@ -54,12 +60,17 @@ async def build(cls, specs: list[ControllerSpec]) -> Self: try: for spec in specs: for connection in _dependency_order(spec.connections): + connection.state = ConnectionState() await connection.connect() runner._connections.append(connection) connection.introspection = await connection.introspect() + runner._dependencies[id(connection)] = _dependencies( + connection, spec.connections + ) runner._controllers = [spec.construct() for spec in specs] runner._seal() + runner._map_connection_holders() for controller in runner._controllers: runner._controller_apis.append(controller.build_api(controller.path)) @@ -86,29 +97,122 @@ async def start(self) -> None: for sub_controller in _walk(controller): await sub_controller.setup() - for coro in self._initial_coros: - await coro() + for reads in self._initial_coros.values(): + for read in reads: + await read() self._tasks = {asyncio.create_task(coro()) for coro in self._scan_coros} + self._watchers = { + id(connection): asyncio.create_task(self._watch(connection)) + for connection in self._connections + } async def stop(self) -> None: - """Stop the scan tasks and close the connections.""" - tasks = list(self._tasks) + """Stop the scan tasks and watchers, and close the connections.""" + tasks = [*self._tasks, *self._watchers.values()] self._cancel_tasks() await asyncio.gather(*tasks, return_exceptions=True) await self._close_connections() def _seal(self) -> None: - """Seal every controller, recording which controllers hold each connection.""" - opened = {id(connection) for connection in self._connections} + """Seal every controller in every tree.""" for controller in self._controllers: for sub_controller in _walk(controller): sub_controller.seal() - for connection in _held_connections(sub_controller): - if id(connection) in opened: - self._controllers_by_connection[id(connection)].append( - sub_controller + + def _map_connection_holders(self) -> None: + """Record which controllers hold each connection. + + Raises: + ValueError: If a controller holds a connection the runner didn't open, or + more than one connection, or if no controller holds a connection + + """ + opened = {id(connection) for connection in self._connections} + for controller in self._controllers: + for sub_controller in _walk(controller): + held = _held_connections(sub_controller) + for connection in held: + if id(connection) not in opened: + raise ValueError( + f"{sub_controller} holds a {type(connection).__name__} " + "that wasn't opened by the framework. Declare it as an " + "argument of the top-level controller instead." ) + if len(held) > 1: + names = ", ".join(type(c).__name__ for c in held) + raise ValueError( + f"{sub_controller} holds more than one connection ({names}). " + "Give each connection its own sub controller instead." + ) + if held: + self._controllers_by_connection[id(held[0])].append(sub_controller) + + for connection in self._connections: + if id(connection) not in self._controllers_by_connection: + raise ValueError( + f"No controller holds {type(connection).__name__}. Store it on " + "the controller that uses it, e.g. self.connection = connection." + ) + + async def _watch(self, connection: Connection) -> None: + """Each time a connection is marked down, mark its controllers down, then + reconnect it and mark them up, or give up and leave them down.""" + while True: + await connection.state.wait_down() + await self._mark_down(connection) + if not await self._reconnect(connection): + logger.error("Giving up reconnecting", connection=connection) + return + + await self._mark_up(connection) + + async def _mark_down(self, connection: Connection) -> None: + """Pause the scans of the controllers holding a connection, and mark them + disconnected.""" + for controller in self._controllers_by_connection[id(connection)]: + self._scan_gates[id(controller)].clear() + await controller.connected.update(False) + + async def _reconnect(self, connection: Connection) -> bool: + """Try to reconnect, once its dependencies are up. + + Returns: + Whether it reconnected within ``reconnect_attempts`` + + """ + for attempt in range(connection.reconnect_attempts): + if attempt: + await asyncio.sleep(connection.reconnect_period) + + for dependency in self._dependencies[id(connection)]: + await dependency.state.wait_up() + + try: + await connection.close() + await connection.connect() + except Exception: + logger.exception( + "Reconnect failed", connection=connection, attempt=attempt + 1 + ) + else: + return True + + return False + + async def _mark_up(self, connection: Connection) -> None: + """Mark a connection up, then mark connected each controller holding it, + re-run its read-once reads and resume its scans.""" + connection.state.mark_up() + for controller in self._controllers_by_connection[id(connection)]: + await controller.connected.update(True) + for read in self._initial_coros[id(controller)]: + try: + await read() + except Exception: + logger.exception("Exception in read-once read after reconnect") + + self._scan_gates[id(controller)].set() async def _close_connections(self) -> None: """Close the open connections, in reverse order.""" @@ -130,7 +234,7 @@ def _collect_scans(self, controller: Controller) -> None: scans_by_period: dict[float, list[ScanCallback]] = defaultdict(list) for period, scan in _scans(sub_controller): if period is ONCE: - self._initial_coros.append(scan) + self._initial_coros[id(sub_controller)].append(scan) else: scans_by_period[period].append(scan) @@ -143,11 +247,12 @@ def _collect_scans(self, controller: Controller) -> None: ) def _cancel_tasks(self) -> None: - for task in self._tasks: + for task in [*self._tasks, *self._watchers.values()]: if not task.done(): task.cancel() self._tasks.clear() + self._watchers.clear() def __del__(self): self._cancel_tasks() @@ -207,6 +312,15 @@ def _dependency_types(connection: Connection) -> tuple[type[Connection], ...]: return tuple(depends_on) +def _dependencies( + connection: Connection, connections: list[Connection] +) -> list[Connection]: + """The connections, from the same spec, that a connection depends on""" + types = _dependency_types(connection) + unique = {id(c): c for c in connections}.values() + return [c for c in unique if c is not connection and isinstance(c, types)] + + def _dependency_order(connections: list[Connection]) -> list[Connection]: """Order connections so that each comes after the connections it depends on. @@ -226,11 +340,8 @@ def visit(connection: Connection) -> None: raise ValueError(f"Cycle in connection dependencies: {cycle}") visiting.append(connection) - for other in unique: - if other is not connection and isinstance( - other, _dependency_types(connection) - ): - visit(other) + for dependency in _dependencies(connection, unique): + visit(dependency) visiting.pop() ordered.append(connection) diff --git a/tests/test_control_system.py b/tests/test_control_system.py index b1b392d8..8cc6dccf 100644 --- a/tests/test_control_system.py +++ b/tests/test_control_system.py @@ -97,7 +97,7 @@ class MyController(Controller): assert controller.update_never.get() == 0 assert len(fastcs._runner._tasks) == 1 - assert len(fastcs._runner._initial_coros) == 1 + assert sum(map(len, fastcs._runner._initial_coros.values())) == 1 class ClosingConnection(Connection): @@ -113,6 +113,7 @@ async def close(self) -> None: class FailingSetupController(Controller): def __init__(self, connection: ClosingConnection) -> None: super().__init__() + self.connection = connection async def setup(self) -> None: raise RuntimeError("setup failed") diff --git a/tests/test_controller_runner.py b/tests/test_controller_runner.py index 0c299b89..09213569 100644 --- a/tests/test_controller_runner.py +++ b/tests/test_controller_runner.py @@ -111,6 +111,17 @@ async def close(self) -> None: self.events.append(f"close{self.suffix}") +class HoldingController(Controller): + def __init__(self, connection: Connection) -> None: + super().__init__() + self.connection = connection + self.reads = 0 + + @scan(ONCE) + async def read_once(self) -> None: + self.reads += 1 + + class ConnectedController(Controller): def __init__(self, connection: RecordingConnection) -> None: super().__init__() @@ -138,6 +149,48 @@ async def test_connection_opened_before_construction_and_closed_on_stop(): assert events == ["connect", "construct", "close"] +class UnheldController(Controller): + def __init__(self, connection: Connection) -> None: + super().__init__() + + +class TwoHeldController(Controller): + def __init__(self, a: RecordingConnection, b: RecordingConnection) -> None: + super().__init__() + self.a = a + self.b = b + + +class UnopenedController(Controller): + def __init__(self) -> None: + super().__init__() + self.connection = RecordingConnection([]) + + +@pytest.mark.parametrize( + "spec, message", + [ + ( + ControllerSpec("A", UnheldController, RecordingConnection([])), + "No controller holds", + ), + ( + ControllerSpec( + "A", TwoHeldController, RecordingConnection([]), RecordingConnection([]) + ), + "more than one connection", + ), + (ControllerSpec("A", UnopenedController), "wasn't opened by the framework"), + ], +) +@pytest.mark.asyncio +async def test_build_raises_if_connections_break_the_holding_rules( + spec: ControllerSpec, message: str +): + with pytest.raises(ValueError, match=message): + await ControllerRunner.build([spec]) + + class FailingController(Controller): def __init__(self, connection: RecordingConnection) -> None: raise RuntimeError("construction failed") @@ -200,6 +253,8 @@ class OdinConnection(RecordingConnection): class TwoConnectionController(Controller): def __init__(self, odin: OdinConnection, eiger: EigerConnection) -> None: super().__init__() + self.ODIN = HoldingController(odin) + self.EIGER = HoldingController(eiger) @pytest.mark.asyncio @@ -274,6 +329,7 @@ async def close(self) -> None: class IntrospectedController(Controller): def __init__(self, connection: IntrospectingConnection) -> None: super().__init__() + self.connection = connection self.parameters = connection.introspection.parameters @@ -341,3 +397,63 @@ def test_seal_twice_raises(): controller.seal() with pytest.raises(RuntimeError, match="already sealed"): controller.seal() + + +class FlakyConnection(Connection): + reconnect_period = 0 + + def __init__(self, failures: int) -> None: + self.failures = failures + self.connects = 0 + self.reconnected = asyncio.Event() + + async def connect(self) -> None: + self.connects += 1 + if self.connects == 1: + return + if self.failures: + self.failures -= 1 + raise ConnectionError("Device is off") + self.reconnected.set() + + async def close(self) -> None: + pass + + +@pytest.mark.asyncio +async def test_watcher_reconnects_then_marks_controllers_up(): + connection = FlakyConnection(failures=1) + runner = await ControllerRunner.build( + [ControllerSpec("A", HoldingController, connection)] + ) + await runner.start() + controller = runner.controllers[0] + assert isinstance(controller, HoldingController) + + connection.state.mark_down() + await asyncio.wait_for(connection.reconnected.wait(), timeout=1) + await asyncio.wait_for(runner._scan_gates[id(controller)].wait(), timeout=1) + + assert connection.connects == 3 + assert controller.connected.get() is True + assert controller.reads == 2 + await runner.stop() + + +@pytest.mark.asyncio +async def test_watcher_gives_up_and_leaves_controllers_down(): + connection = FlakyConnection(failures=0) + connection.reconnect_attempts = 0 + runner = await ControllerRunner.build( + [ControllerSpec("A", HoldingController, connection)] + ) + await runner.start() + controller = runner.controllers[0] + + connection.state.mark_down() + await asyncio.wait_for(runner._watchers[id(connection)], timeout=1) + + assert connection.connects == 1 + assert controller.connected.get() is False + assert not runner._scan_gates[id(controller)].is_set() + await runner.stop() From a97a07876ab868b3aedb30774d1f358153d61a28 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Thu, 12 Nov 2026 15:35:14 +0000 Subject: [PATCH 06/22] remove unhelpful comments --- src/fastcs/runner.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/fastcs/runner.py b/src/fastcs/runner.py index 26bf66e4..38dfc2db 100644 --- a/src/fastcs/runner.py +++ b/src/fastcs/runner.py @@ -20,22 +20,17 @@ class ControllerRunner: def __init__(self) -> None: self._connections: list[Connection] = [] self._controllers: list[Controller] = [] - # id of each connection -> the controllers that hold it, in tree order self._controllers_by_connection: dict[int, list[BaseController]] = defaultdict( list ) - # id of each connection -> the connections it waits for when reconnecting self._dependencies: dict[int, list[Connection]] = {} self._controller_apis: list[ControllerAPI] = [] - # id of each controller -> its read-once reads self._initial_coros: dict[int, list[ScanCallback]] = defaultdict(list) self._scan_coros: list[ScanCallback] = [] - # id of each controller -> its scan gate, set while its scans may run self._scan_gates: dict[int, asyncio.Event] = {} self._tasks: set[asyncio.Task] = set() - # id of each connection -> its watcher self._watchers: dict[int, asyncio.Task] = {} @classmethod From 1c826e34900834b0876e304b29b640f104fb37e5 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Thu, 12 Nov 2026 15:43:34 +0000 Subject: [PATCH 07/22] add reconnect attempts and period, class and instane overridable --- src/fastcs/connections/http_connection.py | 4 +++ src/fastcs/connections/ip_connection.py | 4 +++ src/fastcs/connections/serial_connection.py | 4 +++ src/fastcs/demo/schema.json | 32 +++++++++++++++++++++ src/fastcs/launch.py | 27 +++++++++++++++-- tests/test_launch.py | 10 +++++++ 6 files changed, 79 insertions(+), 2 deletions(-) diff --git a/src/fastcs/connections/http_connection.py b/src/fastcs/connections/http_connection.py index b3662299..73f1eb5a 100644 --- a/src/fastcs/connections/http_connection.py +++ b/src/fastcs/connections/http_connection.py @@ -27,6 +27,10 @@ class HTTPConnection(Connection[IntrospectionT], Tracer): """ + # Long enough to ride out a slow server restart + reconnect_attempts = 60 + reconnect_period = 5.0 + def __init__( self, url: str, diff --git a/src/fastcs/connections/ip_connection.py b/src/fastcs/connections/ip_connection.py index d9fb8ed2..924d77a4 100644 --- a/src/fastcs/connections/ip_connection.py +++ b/src/fastcs/connections/ip_connection.py @@ -27,6 +27,10 @@ class IPConnection(Connection[IntrospectionT], Tracer): """ + # Long enough to ride out a device being power cycled + reconnect_attempts = 30 + reconnect_period = 2.0 + def __init__( self, ip: str, diff --git a/src/fastcs/connections/serial_connection.py b/src/fastcs/connections/serial_connection.py index d2804354..eb28e992 100644 --- a/src/fastcs/connections/serial_connection.py +++ b/src/fastcs/connections/serial_connection.py @@ -36,6 +36,10 @@ class SerialConnection(Connection[IntrospectionT], Tracer): """ + # Reopening a local port is quick, and one gone for longer needs a person + reconnect_attempts = 10 + reconnect_period = 1.0 + def __init__( self, port: str, diff --git a/src/fastcs/demo/schema.json b/src/fastcs/demo/schema.json index 12768125..1b8bc982 100644 --- a/src/fastcs/demo/schema.json +++ b/src/fastcs/demo/schema.json @@ -186,6 +186,38 @@ "default": 25565, "title": "Port", "type": "integer" + }, + "terminator": { + "default": "\r\n", + "title": "Terminator", + "type": "string" + }, + "encoding": { + "default": "utf-8", + "title": "Encoding", + "type": "string" + }, + "timeout": { + "default": 5.0, + "title": "Timeout", + "type": "number" + }, + "limit": { + "default": 65536, + "title": "Limit", + "type": "integer" + }, + "reconnect_attempts": { + "default": 30, + "minimum": 0, + "title": "Reconnect Attempts", + "type": "integer" + }, + "reconnect_period": { + "default": 2.0, + "minimum": 0, + "title": "Reconnect Period", + "type": "number" } }, "title": "IPConnectionSettings", diff --git a/src/fastcs/launch.py b/src/fastcs/launch.py index f121066f..47e97a74 100644 --- a/src/fastcs/launch.py +++ b/src/fastcs/launch.py @@ -215,7 +215,12 @@ def _controller_specs( for name, connection_type in registered.connection_types.items(): settings = getattr(entry.connections, name) values = {k: getattr(settings, k) for k in type(settings).model_fields} - kwargs[name] = connection_type(**values) + attempts = values.pop("reconnect_attempts") + period = values.pop("reconnect_period") + connection = connection_type(**values) + connection.reconnect_attempts = attempts + connection.reconnect_period = period + kwargs[name] = connection if registered.options_name is not None: field_values = { @@ -233,8 +238,12 @@ def _controller_specs( def _connection_settings_model(connection_type: type[Connection]) -> type[BaseModel]: """Build the Pydantic model for one connection's block under `connections:`. + The block has the constructor's arguments, and the framework's + ``reconnect_attempts`` and ``reconnect_period``. + Raises: - LaunchError: If a constructor argument has no type hint + LaunchError: If a constructor argument has no type hint, or has the name of a + framework setting """ sig = inspect.signature(connection_type.__init__) @@ -248,9 +257,23 @@ def _connection_settings_model(connection_type: type[Connection]) -> type[BaseMo f"Expected typehinting in '{connection_type.__name__}.__init__' " f"but received {sig}. Add a typehint for `{name}`." ) + if name in ("reconnect_attempts", "reconnect_period"): + raise LaunchError( + f"'{connection_type.__name__}.__init__' can't take `{name}`, which is " + "a framework setting." + ) default = ... if param.default is inspect.Parameter.empty else param.default fields[name] = (hints[name], default) + fields["reconnect_attempts"] = ( + int, + Field(default=connection_type.reconnect_attempts, ge=0), + ) + fields["reconnect_period"] = ( + float, + Field(default=connection_type.reconnect_period, ge=0), + ) + return create_model( f"{connection_type.__name__}Settings", __config__={"extra": "forbid"}, **fields ) diff --git a/tests/test_launch.py b/tests/test_launch.py index 2b48a0d0..40939dfa 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -18,6 +18,7 @@ from fastcs.exceptions import LaunchError from fastcs.launch import ( _build_options_model, + _connection_settings_model, _launch, get_controller_schema, launch, @@ -359,6 +360,8 @@ def test_connections_built_from_config(mocker: MockerFixture, tmp_path): " connection:\n" " ip: localhost\n" " port: 8000\n" + " reconnect_attempts: 0\n" + " reconnect_period: 2.0\n" " name: a\n" "transport:\n" " - rest: {}\n" @@ -370,9 +373,16 @@ def test_connections_built_from_config(mocker: MockerFixture, tmp_path): connection = spec.kwargs["connection"] assert isinstance(connection, IPConnection) assert (connection.ip, connection.port) == ("localhost", 8000) + assert (connection.reconnect_attempts, connection.reconnect_period) == (0, 2.0) assert spec.kwargs["arg"] == SomeConfig(name="a") +def test_reconnect_setting_defaults_come_from_the_connection_class(): + fields = _connection_settings_model(IPConnection).model_fields + assert fields["reconnect_attempts"].default == IPConnection.reconnect_attempts + assert fields["reconnect_period"].default == IPConnection.reconnect_period + + def test_parametrised_and_no_settings_connections(): options_model = _build_options_model([WithGenericConnections]) options_model.model_validate( From 09ee695464664a63db33137cd18ce334fb8f1491 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Thu, 12 Nov 2026 16:05:04 +0000 Subject: [PATCH 08/22] add fatal error handling and reconnect introspection comparison --- src/fastcs/control_system.py | 5 +++ src/fastcs/exceptions.py | 8 ++++- src/fastcs/runner.py | 60 ++++++++++++++++++++++++++++----- tests/test_controller_runner.py | 37 +++++++++++++++++++- 4 files changed, 100 insertions(+), 10 deletions(-) diff --git a/src/fastcs/control_system.py b/src/fastcs/control_system.py index f8895202..1571ca83 100644 --- a/src/fastcs/control_system.py +++ b/src/fastcs/control_system.py @@ -8,6 +8,7 @@ from IPython.terminal.embed import InteractiveShellEmbed from fastcs.controllers import ControllerSpec +from fastcs.exceptions import FatalError from fastcs.logging import logger from fastcs.runner import ControllerRunner from fastcs.tracer import Tracer @@ -129,10 +130,14 @@ async def block_forever(): coros.append(block_forever()) + coros.append(self._runner.wait_for_fatal_error()) try: await asyncio.gather(*coros) except asyncio.CancelledError: pass + except FatalError: + logger.exception("Stopping FastCS after a fatal error") + raise except Exception: logger.exception("Unhandled exception in serve") finally: diff --git a/src/fastcs/exceptions.py b/src/fastcs/exceptions.py index 45fbad3d..84a708cf 100644 --- a/src/fastcs/exceptions.py +++ b/src/fastcs/exceptions.py @@ -1,5 +1,11 @@ class FastCSError(Exception): - """Base class for general problems in the running of a FastCS transport.""" + """Base class for problems in running FastCS.""" + + +class FatalError(FastCSError): + """For when something has happened that means FastCS must stop, so that it can be + restarted. + """ class LaunchError(FastCSError): diff --git a/src/fastcs/runner.py b/src/fastcs/runner.py index 38dfc2db..f4bc79ab 100644 --- a/src/fastcs/runner.py +++ b/src/fastcs/runner.py @@ -1,7 +1,10 @@ import asyncio +import dataclasses from collections import defaultdict -from collections.abc import Iterator, Sequence -from typing import Self +from collections.abc import Iterator, Mapping, Sequence +from typing import Any, Self + +from pydantic import BaseModel from fastcs.attributes.attr_r import AttrR from fastcs.connections import Connection, ConnectionState @@ -9,6 +12,7 @@ from fastcs.controllers.controller import Controller from fastcs.controllers.controller_api import ControllerAPI from fastcs.controllers.controller_spec import ControllerSpec +from fastcs.exceptions import FatalError from fastcs.logging import logger from fastcs.methods import ScanCallback from fastcs.util import ONCE @@ -102,6 +106,15 @@ async def start(self) -> None: for connection in self._connections } + async def wait_for_fatal_error(self) -> None: + """Wait on the connection watchers, which only stop by giving up or failing. + + Raises: + FatalError: If a watcher hits one + + """ + await asyncio.gather(*self._watchers.values()) + async def stop(self) -> None: """Stop the scan tasks and watchers, and close the connections.""" tasks = [*self._tasks, *self._watchers.values()] @@ -170,11 +183,15 @@ async def _mark_down(self, connection: Connection) -> None: await controller.connected.update(False) async def _reconnect(self, connection: Connection) -> bool: - """Try to reconnect, once its dependencies are up. + """Try to reconnect and introspect, once its dependencies are up, marking the + connection up so that ``introspect`` can use it. Returns: Whether it reconnected within ``reconnect_attempts`` + Raises: + FatalError: If the new introspection's structure is different + """ for attempt in range(connection.reconnect_attempts): if attempt: @@ -186,19 +203,28 @@ async def _reconnect(self, connection: Connection) -> bool: try: await connection.close() await connection.connect() + connection.state.mark_up() + introspection = await connection.introspect() except Exception: + connection.state.mark_down() logger.exception( "Reconnect failed", connection=connection, attempt=attempt + 1 ) - else: - return True + continue + + if _structure(introspection) != _structure(connection.introspection): + raise FatalError( + f"{type(connection).__name__} introspection changed structure on " + "reconnect, so the controllers built from it are out of date" + ) + + return True return False async def _mark_up(self, connection: Connection) -> None: - """Mark a connection up, then mark connected each controller holding it, - re-run its read-once reads and resume its scans.""" - connection.state.mark_up() + """Mark connected each controller holding a connection, re-run its read-once + reads and resume its scans.""" for controller in self._controllers_by_connection[id(connection)]: await controller.connected.update(True) for read in self._initial_coros[id(controller)]: @@ -273,6 +299,24 @@ def _scans(controller: BaseController) -> Iterator[tuple[float, ScanCallback]]: yield period, attribute.bind_update_callback() +def _structure(value: Any) -> Any: + """The shape of a value: the fields, keys and lengths it's made of, but not its + leaf values or their types""" + if isinstance(value, BaseModel): + fields = type(value).model_fields + return {name: _structure(getattr(value, name)) for name in fields} + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + field.name: _structure(getattr(value, field.name)) + for field in dataclasses.fields(value) + } + if isinstance(value, Mapping): + return {key: _structure(item) for key, item in value.items()} + if isinstance(value, list | tuple): + return [_structure(item) for item in value] + return None + + def _held_connections(controller: BaseController) -> list[Connection]: """The connections stored as attributes of a controller, once each""" values = vars(controller).values() diff --git a/tests/test_controller_runner.py b/tests/test_controller_runner.py index 09213569..9cbe7a41 100644 --- a/tests/test_controller_runner.py +++ b/tests/test_controller_runner.py @@ -5,8 +5,9 @@ from fastcs.connections import Connection from fastcs.controllers import Controller, ControllerSpec +from fastcs.exceptions import FatalError from fastcs.methods import scan -from fastcs.runner import ControllerRunner +from fastcs.runner import ControllerRunner, _structure from fastcs.util import ONCE @@ -346,6 +347,40 @@ async def test_introspection_available_when_controller_constructed(): assert connection.introspection == DeviceIntrospection({"gain": 2}) +def test_structure_counts_fields_and_keys_but_not_values(): + assert _structure(DeviceIntrospection({"gain": 2})) == _structure( + DeviceIntrospection({"gain": 3}) + ) + assert _structure(DeviceIntrospection({"gain": 2})) != _structure( + DeviceIntrospection({"offset": 2}) + ) + + +class ChangingConnection(IntrospectingConnection): + def __init__(self) -> None: + self.introspections = 0 + + async def introspect(self) -> DeviceIntrospection: + self.introspections += 1 + if self.introspections == 1: + return DeviceIntrospection({"gain": 2}) + return DeviceIntrospection({"offset": 2}) + + +@pytest.mark.asyncio +async def test_introspection_changing_structure_on_reconnect_is_fatal(): + connection = ChangingConnection() + runner = await ControllerRunner.build( + [ControllerSpec("A", IntrospectedController, connection)] + ) + await runner.start() + + connection.state.mark_down() + with pytest.raises(FatalError, match="changed structure"): + await asyncio.wait_for(runner.wait_for_fatal_error(), timeout=1) + await runner.stop() + + def test_introspection_before_connecting_raises(): with pytest.raises(RuntimeError, match="no introspection until the framework"): _ = IntrospectingConnection().introspection From ddb5bff63d2cbadc73c365e2c1fb1f38e5766603 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Thu, 12 Nov 2026 16:09:19 +0000 Subject: [PATCH 09/22] update docs with stale connection descriptions --- docs/conf.py | 1 + docs/explanations/connections.md | 79 ++++++++++++++++++++ docs/explanations/controllers.md | 33 ++++---- docs/explanations/what-is-fastcs.md | 6 +- docs/how-to/update-attributes-from-device.md | 4 +- 5 files changed, 99 insertions(+), 24 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index ec941e43..87dc0621 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -107,6 +107,7 @@ nitpick_ignore_regex = [ ("py:class", r"fastcs.*.DType_T"), ("py:class", r"fastcs.*.Numeric_T"), + ("py:class", r"fastcs.*._MethodT"), ("py:obj", r"fastcs.*.DType_T"), (r"py:.*", r"fastcs\.demo.*"), (r"py:.*", r"tickit.*"), diff --git a/docs/explanations/connections.md b/docs/explanations/connections.md index 3260120e..68364432 100644 --- a/docs/explanations/connections.md +++ b/docs/explanations/connections.md @@ -45,3 +45,82 @@ Like the other connections, it raises `TimeoutError` when the device doesn't ans time, but leaves the link open, as one slow request doesn't mean the server has gone. It raises `ConnectionError` when the server can't be reached, and closes the link so later calls raise `DisconnectedError`. + +## When the link goes down + +A connection marks the IO methods whose failure means the link has gone with +`@disconnect_on`: + +```python +from fastcs.connections import Connection, disconnect_on + + +class EigerConnection(Connection): + ... + + @disconnect_on(TimeoutError) + async def get(self, key: str) -> Any: + return await self.client.get(f"/detector/{key}") +``` + +If the method raises a `ConnectionError`, a `DisconnectedError` or one of the listed +exceptions, the connection is marked down and the exception carries on to the caller. +While it's down, decorated methods raise `DisconnectedError` without touching the +device. Other exceptions, like an error reply from the device, leave the link alone. The +built-in connections already decorate their IO methods. + +When a connection is marked down, FastCS: + +1. Sets `connected` to `False` on every controller holding it, and pauses their scans. +2. Waits for the connections it `depends_on` to be up, then calls `close()`, `connect()` + and `introspect()`. The first try is straight away, and later ones are + `reconnect_period` seconds apart. +3. Once that works, sets `connected` back to `True`, re-runs the controllers' `ONCE` + scans and resumes the others. +4. After `reconnect_attempts` failed tries, gives up. The controllers stay disconnected + while the rest of the application carries on. + +If the new introspection has different fields or keys from the one the controllers were +built from, FastCS stops with a `FatalError`, so that it can be restarted and build them +again. Changed values are fine. + +### Reconnect settings + +Each connection's block in `fastcs.yaml` can set how it reconnects: + +```yaml +connections: + connection: + ip: 10.0.0.5 + port: 4000 + reconnect_attempts: 5 + reconnect_period: 2.0 +``` + +`reconnect_attempts: 0` means never reconnect. A connection class can set its own +defaults: + +```python +class EigerConnection(Connection): + reconnect_attempts = 20 + reconnect_period = 3.0 +``` + +| Connection | `reconnect_attempts` | `reconnect_period` | +|---|---|---| +| `Connection` | 10 | 5.0 | +| `IPConnection` | 30 | 2.0 | +| `HTTPConnection` | 60 | 5.0 | +| `SerialConnection` | 10 | 1.0 | + +### Which controllers a connection belongs to + +FastCS finds the controllers to mark down by looking for the connection on them, so: + +- The controller that uses a connection stores it, e.g. `self.connection = connection`, + and passes it to any sub controllers that need it, which store it too. +- Each controller stores at most one connection. +- Every connection a controller stores was opened by FastCS, from the top-level + controller's arguments. + +FastCS raises a `ValueError` at startup if any of these is broken. diff --git a/docs/explanations/controllers.md b/docs/explanations/controllers.md index b1cb7ddc..aa105a31 100644 --- a/docs/explanations/controllers.md +++ b/docs/explanations/controllers.md @@ -12,28 +12,31 @@ role, FastCS opens the `Connection`s it is given, constructs it, calls its lifec hooks and runs the scan tasks it creates on the event loop. **Sub controller:** attached to a parent controller via `add_sub_controller()` or by -assigning it as an attribute. In this role, the sub controller's `reconnect` hook is -not called automatically by FastCS. The parent controller is responsible for calling it -as part of its own lifecycle, if required. +assigning it as an attribute. Its attributes are exposed under a prefixed path. ### Lifecycle hooks | Method | Purpose | |---|---| | `setup` | Called by the framework after all attributes are added, before serving the application. Called on every controller in the tree | -| `reconnect` | Re-open connection after scan error | Connections to the device are not opened by the controller. They are `Connection`s -passed to its constructor, which FastCS opens before constructing it and closes on -shutdown. +passed to its constructor, which FastCS opens before constructing it, reconnects if the +link goes down, and closes on shutdown. A controller stores the connection it uses, and +passes it on to any sub controllers that need it. See +[Connections](connections.md). ### Scan task behaviour -When used as the root controller, FastCS collects all `@scan` methods and readable -attributes with `update_period` set, across the whole controller hierarchy to be run as -background tasks by FastCS. Scan tasks are gated on the `_connected` flag: if a scan -raises an exception, `_connected` is set to `False` and tasks pause until `reconnect` -sets it back to `True`. +FastCS collects the `@scan` methods and readable attributes with `update_period` set +from every controller in the tree, and runs them as background tasks. A scan that +raises an exception is logged, and runs again on its next period. + +Every controller has a read-only `connected` attribute. When the connection a controller +holds goes down, FastCS sets `connected` to `False` and pauses that controller's scans. +Once the connection is back, it sets `connected` to `True`, re-runs the controller's +`ONCE` scans and resumes the others. Controllers that hold no connection, or a different +one, carry on. ```python from fastcs.controllers import Controller @@ -50,14 +53,6 @@ class TemperatureController(Controller): super().__init__() self._connection = connection - async def reconnect(self): - try: - await self._connection.close() - await self._connection.connect() - self._connected = True - except Exception: - logger.error("Failed to reconnect") - @scan(period=1.0) async def update_temperature(self): value = await self._connection.get_temperature() diff --git a/docs/explanations/what-is-fastcs.md b/docs/explanations/what-is-fastcs.md index cbcce272..be59ab2d 100644 --- a/docs/explanations/what-is-fastcs.md +++ b/docs/explanations/what-is-fastcs.md @@ -55,7 +55,7 @@ using `ControllerVector`. See [Controllers](controllers.md) for details. To create a driver, subclass `Controller`, declare attributes and commands as typed class members, take the `Connection`s it needs as constructor arguments, and implement -lifecycle hooks (`setup`, `reconnect`) and periodic scan tasks. The [tutorials](../tutorials.md) walk through a complete example. +the `setup` lifecycle hook and periodic scan tasks. The [tutorials](../tutorials.md) walk through a complete example. ## Deploying as an application @@ -77,8 +77,8 @@ device logic does not require a running IOC or device server. **Multiple transports from a single driver** — the same controller can serve EPICS CA, EPICS PVA, Tango, REST, and GraphQL simultaneously or in any combination. -**Automatic reconnection** — if a scan task raises an exception, FastCS pauses all -tasks and calls `reconnect()` automatically, resuming once the connection is restored. +**Automatic reconnection** — when a connection's link goes down, FastCS pauses the scans +of the controllers using it and reconnects it, resuming them once the link is back. **Auto-generated OPI screens** — EPICS transports generate CSS-Phoebus screen files from the controller's attribute metadata, with no additional effort from the driver diff --git a/docs/how-to/update-attributes-from-device.md b/docs/how-to/update-attributes-from-device.md index 451b51e8..e829efd0 100644 --- a/docs/how-to/update-attributes-from-device.md +++ b/docs/how-to/update-attributes-from-device.md @@ -175,8 +175,8 @@ class MultiChannelController(Controller): await channel.voltage.update(float(voltage)) ``` -The scan period (here `0.1` seconds) sets how often the batched query runs. Scans that -raise an exception will pause and wait for `reconnect()` to be called before resuming. +The scan period (here `0.1` seconds) sets how often the batched query runs. If the +connection goes down, the scan pauses until FastCS has reconnected it. ### Scan as a cache for `AttributeIO.update` From 31a04bd0085eab49f2b6d751348d5cb4006ee02f Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Thu, 12 Nov 2026 17:08:28 +0000 Subject: [PATCH 10/22] add Connected attribute to epics tests --- tests/transports/epics/ca/test_gui.py | 22 ++++++++++++++++++++-- tests/transports/epics/ca/test_softioc.py | 9 ++++++++- tests/transports/epics/pva/test_p4p.py | 6 ++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/transports/epics/ca/test_gui.py b/tests/transports/epics/ca/test_gui.py index 492a0402..5511cf69 100644 --- a/tests/transports/epics/ca/test_gui.py +++ b/tests/transports/epics/ca/test_gui.py @@ -116,7 +116,13 @@ def test_get_components(controller): name="ReadInt", read_pv="DEVICE:SubController01:ReadInt", read_widget=TextRead(), - ) + ), + SignalR( + name="Connected", + description="Whether its connections are up", + read_pv="DEVICE:SubController01:Connected", + read_widget=LED(), + ), ], ), Group( @@ -127,7 +133,13 @@ def test_get_components(controller): name="ReadInt", read_pv="DEVICE:SubController02:ReadInt", read_widget=TextRead(), - ) + ), + SignalR( + name="Connected", + description="Whether its connections are up", + read_pv="DEVICE:SubController02:Connected", + read_widget=LED(), + ), ], ), SignalR( @@ -164,6 +176,12 @@ def test_get_components(controller): read_pv="DEVICE:ReadString_RBV", write_pv="DEVICE:ReadString", ), + SignalR( + name="Connected", + description="Whether its connections are up", + read_pv="DEVICE:Connected", + read_widget=LED(), + ), SignalX( name="Go", write_pv="DEVICE:Go", diff --git a/tests/transports/epics/ca/test_softioc.py b/tests/transports/epics/ca/test_softioc.py index b270bca6..610b1345 100644 --- a/tests/transports/epics/ca/test_softioc.py +++ b/tests/transports/epics/ca/test_softioc.py @@ -354,13 +354,20 @@ def test_ioc(mocker: MockerFixture, epics_controller_api: ControllerAPI): EpicsCAIOC([epics_controller_api], {}) # Check records are created - util_builder.boolIn.assert_called_once_with( + util_builder.boolIn.assert_any_call( f"{DEVICE}:ReadBool", DESC=None, ZNAM="False", ONAM="True", initial_value=False, ) + util_builder.boolIn.assert_any_call( + f"{DEVICE}:Connected", + DESC="Whether its connections are up", + ZNAM="False", + ONAM="True", + initial_value=True, + ) util_builder.longIn.assert_any_call( f"{DEVICE}:ReadInt", DESC=None, diff --git a/tests/transports/epics/pva/test_p4p.py b/tests/transports/epics/pva/test_p4p.py index b6d3c4d5..121993c2 100644 --- a/tests/transports/epics/pva/test_p4p.py +++ b/tests/transports/epics/pva/test_p4p.py @@ -40,6 +40,7 @@ async def test_ioc(p4p_subprocess: tuple[str, Queue]): "r": f"{pv_prefix}:Table_RBV", "w": f"{pv_prefix}:Table", }, + "connected": {"r": f"{pv_prefix}:Connected"}, } child_vector_pvi_pv = parent_pvi["value"]["child"]["d"] @@ -56,6 +57,7 @@ async def test_ioc(p4p_subprocess: tuple[str, Queue]): }, "__1": {"d": f"{pv_prefix}:Child:1:PVI"}, "__2": {"d": f"{pv_prefix}:Child:2:PVI"}, + "connected": {"r": f"{pv_prefix}:Child:Connected"}, } child_pvi_pv = _child_vector_pvi["value"]["__1"]["d"] @@ -73,6 +75,7 @@ async def test_ioc(p4p_subprocess: tuple[str, Queue]): "h": {"w": f"{pv_prefix}:Child:1:H", "r": f"{pv_prefix}:Child:1:H_RBV"}, "i": {"x": f"{pv_prefix}:Child:1:I"}, "j": {"r": f"{pv_prefix}:Child:1:J"}, + "connected": {"r": f"{pv_prefix}:Child:1:Connected"}, } @@ -401,6 +404,7 @@ class SomeController(Controller): "child0": {"d": f"{pv_prefix}:Child0:PVI"}, "child1": {"d": f"{pv_prefix}:Child1:PVI"}, "child2": {"d": f"{pv_prefix}:Child2:PVI"}, + "connected": {"r": f"{pv_prefix}:Connected"}, }, } assert len(child_vector_controller_pvi) == 1 @@ -416,6 +420,7 @@ class SomeController(Controller): "__0": {"d": f"{pv_prefix}:Child:0:PVI"}, "__1": {"d": f"{pv_prefix}:Child:1:PVI"}, "__2": {"d": f"{pv_prefix}:Child:2:PVI"}, + "connected": {"r": f"{pv_prefix}:Child:Connected"}, }, } assert len(child_child_controller_pvi) == 1 @@ -432,6 +437,7 @@ class SomeController(Controller): "attr_d": { "w": f"{pv_prefix}:Child:0:AttrD", }, + "connected": {"r": f"{pv_prefix}:Child:0:Connected"}, }, } From 699c2d23c31e6e0831f625397f943b0747025c15 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Thu, 12 Nov 2026 17:11:36 +0000 Subject: [PATCH 11/22] tests for depends_on ordering --- tests/test_control_system.py | 39 +++++++++++++++++++++++++++++++++ tests/test_controller_runner.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/tests/test_control_system.py b/tests/test_control_system.py index 8cc6dccf..9391823c 100644 --- a/tests/test_control_system.py +++ b/tests/test_control_system.py @@ -8,6 +8,7 @@ from fastcs.control_system import FastCS from fastcs.controllers import Controller, ControllerSpec from fastcs.datatypes import Int +from fastcs.exceptions import FatalError from fastcs.methods import Command, command from fastcs.util import ONCE from tests.assertable_controller import MyTestController @@ -128,3 +129,41 @@ async def test_connections_closed_if_startup_fails(): await fastcs.serve(interactive=False) assert connection.closed + + +class ChangingConnection(Connection[dict[str, int]]): + def __init__(self) -> None: + self.introspections = 0 + self.closes = 0 + + async def connect(self) -> None: + pass + + async def introspect(self) -> dict[str, int]: + self.introspections += 1 + return {"gain": 1} if self.introspections == 1 else {"offset": 1} + + async def close(self) -> None: + self.closes += 1 + + +class DroppingController(Controller): + def __init__(self, connection: ChangingConnection) -> None: + super().__init__() + self.connection = connection + + async def setup(self) -> None: + # The watcher reconnects as soon as it starts + self.connection.state.mark_down() + + +@pytest.mark.asyncio +async def test_serve_raises_fatal_error_and_closes_connections(): + connection = ChangingConnection() + fastcs = FastCS([ControllerSpec("A", DroppingController, connection)], []) + + with pytest.raises(FatalError, match="changed structure"): + await asyncio.wait_for(fastcs.serve(interactive=False), timeout=1) + + # Once on reconnect, once on shutdown + assert connection.closes == 2 diff --git a/tests/test_controller_runner.py b/tests/test_controller_runner.py index 9cbe7a41..e6a08f61 100644 --- a/tests/test_controller_runner.py +++ b/tests/test_controller_runner.py @@ -492,3 +492,40 @@ async def test_watcher_gives_up_and_leaves_controllers_down(): assert controller.connected.get() is False assert not runner._scan_gates[id(controller)].is_set() await runner.stop() + + +class HeldEigerConnection(EigerConnection): + def __init__(self, events: list[str], name: str = "") -> None: + super().__init__(events, name) + self.connecting = asyncio.Event() + self.release = asyncio.Event() + self.release.set() + + async def connect(self) -> None: + self.connecting.set() + await self.release.wait() + await super().connect() + + +@pytest.mark.asyncio +async def test_reconnect_waits_for_dependencies_to_be_up(): + events: list[str] = [] + eiger = HeldEigerConnection(events, "eiger") + odin = OdinConnection(events, "odin") + runner = await ControllerRunner.build( + [ControllerSpec("A", TwoConnectionController, odin=odin, eiger=eiger)] + ) + await runner.start() + events.clear() + eiger.connecting.clear() + eiger.release.clear() + + eiger.state.mark_down() + odin.state.mark_down() + await asyncio.wait_for(eiger.connecting.wait(), timeout=1) + assert events == ["close eiger"] + + eiger.release.set() + await asyncio.wait_for(odin.state.wait_up(), timeout=1) + assert events == ["close eiger", "connect eiger", "close odin", "connect odin"] + await runner.stop() From 54d97b05773e4c646ed3ba350f0f04ef9d7fc843 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Fri, 13 Nov 2026 13:27:53 +0000 Subject: [PATCH 12/22] update disconnect_on errors, now that standard connection error handling is improved --- src/fastcs/connections/http_connection.py | 2 +- src/fastcs/connections/ip_connection.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/fastcs/connections/http_connection.py b/src/fastcs/connections/http_connection.py index 73f1eb5a..97cfc2e1 100644 --- a/src/fastcs/connections/http_connection.py +++ b/src/fastcs/connections/http_connection.py @@ -174,7 +174,7 @@ async def close(self) -> None: self._client = None await client.aclose() - @disconnect_on(TimeoutError) + @disconnect_on async def _send(self, method: str, path: str, **kwargs: Any) -> httpx.Response: client = self._client if client is None: diff --git a/src/fastcs/connections/ip_connection.py b/src/fastcs/connections/ip_connection.py index 924d77a4..276abeab 100644 --- a/src/fastcs/connections/ip_connection.py +++ b/src/fastcs/connections/ip_connection.py @@ -71,7 +71,7 @@ async def connect(self) -> None: self.ip, self.port, limit=self.limit ) - @disconnect_on(OSError, asyncio.IncompleteReadError) + @disconnect_on(OSError) async def write(self, message: str) -> None: """Send a message, adding the terminator. @@ -88,7 +88,7 @@ async def write(self, message: str) -> None: self.log_event("Sent message", message=message) - @disconnect_on(OSError, asyncio.IncompleteReadError) + @disconnect_on(OSError) async def query(self, message: str) -> str: """Send a message and read the reply. @@ -118,7 +118,7 @@ async def query(self, message: str) -> str: self.log_event("Received query response", query=message, response=response) return response - @disconnect_on(OSError, asyncio.IncompleteReadError) + @disconnect_on(OSError) async def write_bytes(self, data: bytes) -> None: """Send raw bytes, without a terminator. @@ -134,7 +134,7 @@ async def write_bytes(self, data: bytes) -> None: self.log_event("Sent bytes", data=data) - @disconnect_on(OSError, asyncio.IncompleteReadError) + @disconnect_on(OSError) async def query_bytes(self, data: bytes, size: int) -> bytes: """Send raw bytes and read a reply of a fixed size. From d087058ef329eb82ee9998154a002b0d46022753 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Fri, 13 Nov 2026 13:32:44 +0000 Subject: [PATCH 13/22] tango test fails because of Connected fix --- tests/transports/tango/test_dsr.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/transports/tango/test_dsr.py b/tests/transports/tango/test_dsr.py index 61a8f73e..1e5b8390 100644 --- a/tests/transports/tango/test_dsr.py +++ b/tests/transports/tango/test_dsr.py @@ -78,8 +78,11 @@ def test_list_attributes(self, tango_context: DeviceProxy): "Enum", "OneDWaveform", "TwoDWaveform", + "Connected", "SubController01_ReadInt", + "SubController01_Connected", "SubController02_ReadInt", + "SubController02_Connected", "State", "Status", ] From ea169c52430025a42a45d6b20503cd35aff000d9 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Fri, 13 Nov 2026 17:56:29 +0000 Subject: [PATCH 14/22] move structure comparison to the connection to avoid mutation of introspection (ie using .pop()) --- src/fastcs/connections/connection.py | 29 +++++++++++++++++++++++++++- src/fastcs/runner.py | 27 +++----------------------- tests/test_controller_runner.py | 25 +++++++++++++++++++++++- 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/src/fastcs/connections/connection.py b/src/fastcs/connections/connection.py index 6423147c..0ccea0e5 100644 --- a/src/fastcs/connections/connection.py +++ b/src/fastcs/connections/connection.py @@ -1,12 +1,14 @@ from __future__ import annotations import asyncio +import dataclasses import functools import inspect from abc import ABC, abstractmethod -from collections.abc import Callable, Coroutine, Sequence +from collections.abc import Callable, Coroutine, Mapping, Sequence from typing import Any, ClassVar, Generic, cast, overload +from pydantic import BaseModel from typing_extensions import TypeVar IntrospectionT = TypeVar("IntrospectionT", default=None) @@ -79,6 +81,7 @@ async def introspect(self) -> EigerIntrospection: ... reconnect_period: float = 5.0 _introspection: IntrospectionT + _introspection_structure: Any @property def introspection(self) -> IntrospectionT: @@ -102,6 +105,12 @@ def introspection(self, value: IntrospectionT) -> None: raise RuntimeError(f"{type(self).__name__} introspection is already set") self._introspection = value + self._introspection_structure = _structure(value) + + def introspection_changed(self, introspection: IntrospectionT) -> bool: + """Whether an introspection's fields, keys or list lengths differ from those + of the one this connection was opened with""" + return _structure(introspection) != self._introspection_structure @abstractmethod async def connect(self) -> None: @@ -179,3 +188,21 @@ async def wrapper(self: Connection, *args: Any, **kwargs: Any) -> Any: return cast(_MethodT, wrapper) return decorator + + +def _structure(value: Any) -> Any: + """The shape of a value: the fields, keys and lengths it's made of, but not its + leaf values or their types""" + if isinstance(value, BaseModel): + fields = type(value).model_fields + return {name: _structure(getattr(value, name)) for name in fields} + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + field.name: _structure(getattr(value, field.name)) + for field in dataclasses.fields(value) + } + if isinstance(value, Mapping): + return {key: _structure(item) for key, item in value.items()} + if isinstance(value, list | tuple): + return [_structure(item) for item in value] + return None diff --git a/src/fastcs/runner.py b/src/fastcs/runner.py index f4bc79ab..bd91b437 100644 --- a/src/fastcs/runner.py +++ b/src/fastcs/runner.py @@ -1,10 +1,7 @@ import asyncio -import dataclasses from collections import defaultdict -from collections.abc import Iterator, Mapping, Sequence -from typing import Any, Self - -from pydantic import BaseModel +from collections.abc import Iterator, Sequence +from typing import Self from fastcs.attributes.attr_r import AttrR from fastcs.connections import Connection, ConnectionState @@ -212,7 +209,7 @@ async def _reconnect(self, connection: Connection) -> bool: ) continue - if _structure(introspection) != _structure(connection.introspection): + if connection.introspection_changed(introspection): raise FatalError( f"{type(connection).__name__} introspection changed structure on " "reconnect, so the controllers built from it are out of date" @@ -299,24 +296,6 @@ def _scans(controller: BaseController) -> Iterator[tuple[float, ScanCallback]]: yield period, attribute.bind_update_callback() -def _structure(value: Any) -> Any: - """The shape of a value: the fields, keys and lengths it's made of, but not its - leaf values or their types""" - if isinstance(value, BaseModel): - fields = type(value).model_fields - return {name: _structure(getattr(value, name)) for name in fields} - if dataclasses.is_dataclass(value) and not isinstance(value, type): - return { - field.name: _structure(getattr(value, field.name)) - for field in dataclasses.fields(value) - } - if isinstance(value, Mapping): - return {key: _structure(item) for key, item in value.items()} - if isinstance(value, list | tuple): - return [_structure(item) for item in value] - return None - - def _held_connections(controller: BaseController) -> list[Connection]: """The connections stored as attributes of a controller, once each""" values = vars(controller).values() diff --git a/tests/test_controller_runner.py b/tests/test_controller_runner.py index e6a08f61..31399fe5 100644 --- a/tests/test_controller_runner.py +++ b/tests/test_controller_runner.py @@ -4,10 +4,11 @@ import pytest from fastcs.connections import Connection +from fastcs.connections.connection import _structure from fastcs.controllers import Controller, ControllerSpec from fastcs.exceptions import FatalError from fastcs.methods import scan -from fastcs.runner import ControllerRunner, _structure +from fastcs.runner import ControllerRunner from fastcs.util import ONCE @@ -381,6 +382,28 @@ async def test_introspection_changing_structure_on_reconnect_is_fatal(): await runner.stop() +class PoppingController(Controller): + def __init__(self, connection: IntrospectingConnection) -> None: + super().__init__() + self.connection = connection + connection.introspection.parameters.pop("gain") + + +@pytest.mark.asyncio +async def test_controller_mutating_introspection_still_reconnects(): + connection = IntrospectingConnection() + runner = await ControllerRunner.build( + [ControllerSpec("A", PoppingController, connection)] + ) + await runner.start() + controller = runner.controllers[0] + + connection.state.mark_down() + await controller.connected.wait_for_value(False, timeout=1) + await controller.connected.wait_for_value(True, timeout=1) + await runner.stop() + + def test_introspection_before_connecting_raises(): with pytest.raises(RuntimeError, match="no introspection until the framework"): _ = IntrospectingConnection().introspection From 11bfdddb42fbcfa9628db8e5ea7c1f70235452f1 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Fri, 13 Nov 2026 17:59:46 +0000 Subject: [PATCH 15/22] wrap serve coroutines in tasks so we can cleanup on FatalErrors --- src/fastcs/control_system.py | 7 ++++++- tests/test_control_system.py | 24 +++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/fastcs/control_system.py b/src/fastcs/control_system.py index 1571ca83..266c945c 100644 --- a/src/fastcs/control_system.py +++ b/src/fastcs/control_system.py @@ -131,8 +131,9 @@ async def block_forever(): coros.append(block_forever()) coros.append(self._runner.wait_for_fatal_error()) + tasks = [asyncio.create_task(coro) for coro in coros] try: - await asyncio.gather(*coros) + await asyncio.gather(*tasks) except asyncio.CancelledError: pass except FatalError: @@ -140,6 +141,10 @@ async def block_forever(): raise except Exception: logger.exception("Unhandled exception in serve") + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) finally: logger.info("Shutting down FastCS") await self._runner.stop() diff --git a/tests/test_control_system.py b/tests/test_control_system.py index 9391823c..281d7fc8 100644 --- a/tests/test_control_system.py +++ b/tests/test_control_system.py @@ -10,6 +10,7 @@ from fastcs.datatypes import Int from fastcs.exceptions import FatalError from fastcs.methods import Command, command +from fastcs.transports import Transport from fastcs.util import ONCE from tests.assertable_controller import MyTestController @@ -157,13 +158,34 @@ async def setup(self) -> None: self.connection.state.mark_down() +class ForeverTransport(Transport): + cancelled = False + + def connect(self, controller_apis, loop) -> None: + pass + + async def serve(self) -> None: + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + self.cancelled = True + raise + + +# Keep the test transport out of the launcher's transport schema +Transport.subclasses.remove(ForeverTransport) + + @pytest.mark.asyncio async def test_serve_raises_fatal_error_and_closes_connections(): connection = ChangingConnection() - fastcs = FastCS([ControllerSpec("A", DroppingController, connection)], []) + transport = ForeverTransport() + fastcs = FastCS([ControllerSpec("A", DroppingController, connection)], [transport]) with pytest.raises(FatalError, match="changed structure"): await asyncio.wait_for(fastcs.serve(interactive=False), timeout=1) # Once on reconnect, once on shutdown assert connection.closes == 2 + assert transport.cancelled + assert asyncio.all_tasks() == {asyncio.current_task()} From 03f788271ce1052cbad802dcd02bab0abcc851b8 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Fri, 13 Nov 2026 18:03:45 +0000 Subject: [PATCH 16/22] tidy up disconnect errors --- src/fastcs/connections/connection.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/fastcs/connections/connection.py b/src/fastcs/connections/connection.py index 0ccea0e5..e4cce62f 100644 --- a/src/fastcs/connections/connection.py +++ b/src/fastcs/connections/connection.py @@ -130,12 +130,6 @@ async def close(self) -> None: """Close the link. Called by the framework on stop, or if build fails.""" -# What every @disconnect_on treats as the link going down, as well as its errors -_ALWAYS_DISCONNECT_ON: tuple[type[BaseException], ...] = ( - ConnectionError, - DisconnectedError, -) - _MethodT = TypeVar("_MethodT", bound=Callable[..., Coroutine[Any, Any, Any]]) @@ -166,14 +160,26 @@ async def get(self, key: str) -> Any: ... Args: errors: Other exceptions that mean the link has gone + Raises: + TypeError: If the method isn't async, or an error isn't an exception class + """ - if len(errors) == 1 and inspect.iscoroutinefunction(errors[0]): + if len(errors) == 1 and not isinstance(errors[0], type): # Used bare, as @disconnect_on, so the only argument is the method itself return disconnect_on()(errors[0]) - link_errors = (*_ALWAYS_DISCONNECT_ON, *errors) + link_errors = (ConnectionError, *errors) def decorator(method: _MethodT) -> _MethodT: + name = getattr(method, "__qualname__", repr(method)) + if not inspect.iscoroutinefunction(method): + raise TypeError(f"@disconnect_on needs an async method, but {name} isn't") + for error in errors: + if not (isinstance(error, type) and issubclass(error, BaseException)): + raise TypeError( + f"@disconnect_on on {name} takes exception classes, not {error!r}" + ) + @functools.wraps(method) async def wrapper(self: Connection, *args: Any, **kwargs: Any) -> Any: if not self.state.up: From fa6e63f1e8e2946d3ebc65f2d29e35c846753e72 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Fri, 13 Nov 2026 18:07:16 +0000 Subject: [PATCH 17/22] add context so that connect can use other methods with disconnect_on without disconnecting --- docs/explanations/connections.md | 10 +++--- src/fastcs/connections/connection.py | 6 +++- src/fastcs/runner.py | 11 ++++--- tests/test_controller_runner.py | 46 +++++++++++++++++++++++++++- 4 files changed, 63 insertions(+), 10 deletions(-) diff --git a/docs/explanations/connections.md b/docs/explanations/connections.md index 68364432..babb528e 100644 --- a/docs/explanations/connections.md +++ b/docs/explanations/connections.md @@ -66,8 +66,9 @@ class EigerConnection(Connection): If the method raises a `ConnectionError`, a `DisconnectedError` or one of the listed exceptions, the connection is marked down and the exception carries on to the caller. While it's down, decorated methods raise `DisconnectedError` without touching the -device. Other exceptions, like an error reply from the device, leave the link alone. The -built-in connections already decorate their IO methods. +device, except when called from the connection's own `connect()` or `introspect()` while +FastCS reconnects it. Other exceptions, like an error reply from the device, leave the +link alone. The built-in connections already decorate their IO methods. When a connection is marked down, FastCS: @@ -75,8 +76,9 @@ When a connection is marked down, FastCS: 2. Waits for the connections it `depends_on` to be up, then calls `close()`, `connect()` and `introspect()`. The first try is straight away, and later ones are `reconnect_period` seconds apart. -3. Once that works, sets `connected` back to `True`, re-runs the controllers' `ONCE` - scans and resumes the others. +3. Once that works and the introspection's structure is unchanged, marks the connection + up, sets `connected` back to `True`, re-runs the controllers' `ONCE` scans and + resumes the others. 4. After `reconnect_attempts` failed tries, gives up. The controllers stay disconnected while the rest of the application carries on. diff --git a/src/fastcs/connections/connection.py b/src/fastcs/connections/connection.py index e4cce62f..d10d05bd 100644 --- a/src/fastcs/connections/connection.py +++ b/src/fastcs/connections/connection.py @@ -6,6 +6,7 @@ import inspect from abc import ABC, abstractmethod from collections.abc import Callable, Coroutine, Mapping, Sequence +from contextvars import ContextVar from typing import Any, ClassVar, Generic, cast, overload from pydantic import BaseModel @@ -132,6 +133,9 @@ async def close(self) -> None: _MethodT = TypeVar("_MethodT", bound=Callable[..., Coroutine[Any, Any, Any]]) +# The connection the framework is reconnecting in this task, which may use its own IO +_reconnecting: ContextVar[Connection | None] = ContextVar("_reconnecting", default=None) + @overload def disconnect_on(method: _MethodT, /) -> _MethodT: ... @@ -182,7 +186,7 @@ def decorator(method: _MethodT) -> _MethodT: @functools.wraps(method) async def wrapper(self: Connection, *args: Any, **kwargs: Any) -> Any: - if not self.state.up: + if not self.state.up and _reconnecting.get() is not self: raise DisconnectedError(f"{type(self).__name__} is disconnected") try: diff --git a/src/fastcs/runner.py b/src/fastcs/runner.py index bd91b437..3c75effc 100644 --- a/src/fastcs/runner.py +++ b/src/fastcs/runner.py @@ -5,6 +5,7 @@ from fastcs.attributes.attr_r import AttrR from fastcs.connections import Connection, ConnectionState +from fastcs.connections.connection import _reconnecting from fastcs.controllers.base_controller import BaseController from fastcs.controllers.controller import Controller from fastcs.controllers.controller_api import ControllerAPI @@ -180,8 +181,8 @@ async def _mark_down(self, connection: Connection) -> None: await controller.connected.update(False) async def _reconnect(self, connection: Connection) -> bool: - """Try to reconnect and introspect, once its dependencies are up, marking the - connection up so that ``introspect`` can use it. + """Try to reconnect and introspect, once its dependencies are up, and mark the + connection up if the introspection's structure is unchanged. Returns: Whether it reconnected within ``reconnect_attempts`` @@ -197,17 +198,18 @@ async def _reconnect(self, connection: Connection) -> bool: for dependency in self._dependencies[id(connection)]: await dependency.state.wait_up() + token = _reconnecting.set(connection) try: await connection.close() await connection.connect() - connection.state.mark_up() introspection = await connection.introspect() except Exception: - connection.state.mark_down() logger.exception( "Reconnect failed", connection=connection, attempt=attempt + 1 ) continue + finally: + _reconnecting.reset(token) if connection.introspection_changed(introspection): raise FatalError( @@ -215,6 +217,7 @@ async def _reconnect(self, connection: Connection) -> bool: "reconnect, so the controllers built from it are out of date" ) + connection.state.mark_up() return True return False diff --git a/tests/test_controller_runner.py b/tests/test_controller_runner.py index 31399fe5..47aebb3a 100644 --- a/tests/test_controller_runner.py +++ b/tests/test_controller_runner.py @@ -3,7 +3,7 @@ import pytest -from fastcs.connections import Connection +from fastcs.connections import Connection, DisconnectedError, disconnect_on from fastcs.connections.connection import _structure from fastcs.controllers import Controller, ControllerSpec from fastcs.exceptions import FatalError @@ -552,3 +552,47 @@ async def test_reconnect_waits_for_dependencies_to_be_up(): await asyncio.wait_for(odin.state.wait_up(), timeout=1) assert events == ["close eiger", "connect eiger", "close odin", "connect odin"] await runner.stop() + + +class HandshakeConnection(Connection): + reconnect_period = 0 + + def __init__(self) -> None: + self.handshakes = 0 + self.introspecting = asyncio.Event() + self.release = asyncio.Event() + + async def connect(self) -> None: + await self.handshake() + + @disconnect_on + async def handshake(self) -> None: + self.handshakes += 1 + + async def introspect(self) -> None: + if self.handshakes > 1: + self.introspecting.set() + await self.release.wait() + + async def close(self) -> None: + pass + + +@pytest.mark.asyncio +async def test_only_the_reconnect_can_use_a_down_connection(): + connection = HandshakeConnection() + runner = await ControllerRunner.build( + [ControllerSpec("A", HoldingController, connection)] + ) + await runner.start() + controller = runner.controllers[0] + + connection.state.mark_down() + await asyncio.wait_for(connection.introspecting.wait(), timeout=1) + with pytest.raises(DisconnectedError): + await connection.handshake() + + connection.release.set() + await controller.connected.wait_for_value(True, timeout=1) + assert connection.handshakes == 2 + await runner.stop() From 79425c81fd473c0df5687773de626a649d6838a5 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Mon, 16 Nov 2026 10:45:27 +0000 Subject: [PATCH 18/22] fix scan explanation --- docs/explanations/controllers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/explanations/controllers.md b/docs/explanations/controllers.md index aa105a31..0feddd2f 100644 --- a/docs/explanations/controllers.md +++ b/docs/explanations/controllers.md @@ -30,7 +30,7 @@ passes it on to any sub controllers that need it. See FastCS collects the `@scan` methods and readable attributes with `update_period` set from every controller in the tree, and runs them as background tasks. A scan that -raises an exception is logged, and runs again on its next period. +raises an exception is logged, and runs again a second later. Every controller has a read-only `connected` attribute. When the connection a controller holds goes down, FastCS sets `connected` to `False` and pauses that controller's scans. From c73ec1ae08a9a05866aeaa9205dd600ae1dad917 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Mon, 16 Nov 2026 10:56:26 +0000 Subject: [PATCH 19/22] improve fatal error handling --- docs/explanations/connections.md | 11 ++++++----- src/fastcs/control_system.py | 4 ++-- src/fastcs/exceptions.py | 10 ++++++---- src/fastcs/runner.py | 33 ++++++++++++++++---------------- tests/test_control_system.py | 4 ++-- tests/test_controller_runner.py | 16 +++++++++------- 6 files changed, 42 insertions(+), 36 deletions(-) diff --git a/docs/explanations/connections.md b/docs/explanations/connections.md index babb528e..25a3354c 100644 --- a/docs/explanations/connections.md +++ b/docs/explanations/connections.md @@ -79,12 +79,13 @@ When a connection is marked down, FastCS: 3. Once that works and the introspection's structure is unchanged, marks the connection up, sets `connected` back to `True`, re-runs the controllers' `ONCE` scans and resumes the others. -4. After `reconnect_attempts` failed tries, gives up. The controllers stay disconnected - while the rest of the application carries on. +4. After `reconnect_attempts` failed tries, stops with a `ReconnectFailedError`. If the new introspection has different fields or keys from the one the controllers were -built from, FastCS stops with a `FatalError`, so that it can be restarted and build them -again. Changed values are fine. +built from, FastCS stops with an `IntrospectionMismatchError`. Changed values are fine. + +Both are a `FastCSError`: FastCS shuts down, so that it can be restarted and build its +controllers again. ### Reconnect settings @@ -99,7 +100,7 @@ connections: reconnect_period: 2.0 ``` -`reconnect_attempts: 0` means never reconnect. A connection class can set its own +`reconnect_attempts: 0` means stop as soon as the connection goes down. A connection class can set its own defaults: ```python diff --git a/src/fastcs/control_system.py b/src/fastcs/control_system.py index 266c945c..c98360c5 100644 --- a/src/fastcs/control_system.py +++ b/src/fastcs/control_system.py @@ -8,7 +8,7 @@ from IPython.terminal.embed import InteractiveShellEmbed from fastcs.controllers import ControllerSpec -from fastcs.exceptions import FatalError +from fastcs.exceptions import FastCSError from fastcs.logging import logger from fastcs.runner import ControllerRunner from fastcs.tracer import Tracer @@ -136,7 +136,7 @@ async def block_forever(): await asyncio.gather(*tasks) except asyncio.CancelledError: pass - except FatalError: + except FastCSError: logger.exception("Stopping FastCS after a fatal error") raise except Exception: diff --git a/src/fastcs/exceptions.py b/src/fastcs/exceptions.py index 84a708cf..a385c401 100644 --- a/src/fastcs/exceptions.py +++ b/src/fastcs/exceptions.py @@ -2,10 +2,12 @@ class FastCSError(Exception): """Base class for problems in running FastCS.""" -class FatalError(FastCSError): - """For when something has happened that means FastCS must stop, so that it can be - restarted. - """ +class ReconnectFailedError(FastCSError): + """Raised when a connection doesn't reconnect within its ``reconnect_attempts``""" + + +class IntrospectionMismatchError(FastCSError): + """Raised when a connection's introspection changes structure on reconnect""" class LaunchError(FastCSError): diff --git a/src/fastcs/runner.py b/src/fastcs/runner.py index 3c75effc..0cb2e489 100644 --- a/src/fastcs/runner.py +++ b/src/fastcs/runner.py @@ -10,7 +10,7 @@ from fastcs.controllers.controller import Controller from fastcs.controllers.controller_api import ControllerAPI from fastcs.controllers.controller_spec import ControllerSpec -from fastcs.exceptions import FatalError +from fastcs.exceptions import IntrospectionMismatchError, ReconnectFailedError from fastcs.logging import logger from fastcs.methods import ScanCallback from fastcs.util import ONCE @@ -105,10 +105,12 @@ async def start(self) -> None: } async def wait_for_fatal_error(self) -> None: - """Wait on the connection watchers, which only stop by giving up or failing. + """Wait on the connection watchers, which only stop by failing. Raises: - FatalError: If a watcher hits one + ReconnectFailedError: If a connection doesn't reconnect + IntrospectionMismatchError: If a connection's introspection changes + structure on reconnect """ await asyncio.gather(*self._watchers.values()) @@ -163,14 +165,11 @@ def _map_connection_holders(self) -> None: async def _watch(self, connection: Connection) -> None: """Each time a connection is marked down, mark its controllers down, then - reconnect it and mark them up, or give up and leave them down.""" + reconnect it and mark them up.""" while True: await connection.state.wait_down() await self._mark_down(connection) - if not await self._reconnect(connection): - logger.error("Giving up reconnecting", connection=connection) - return - + await self._reconnect(connection) await self._mark_up(connection) async def _mark_down(self, connection: Connection) -> None: @@ -180,15 +179,14 @@ async def _mark_down(self, connection: Connection) -> None: self._scan_gates[id(controller)].clear() await controller.connected.update(False) - async def _reconnect(self, connection: Connection) -> bool: + async def _reconnect(self, connection: Connection) -> None: """Try to reconnect and introspect, once its dependencies are up, and mark the connection up if the introspection's structure is unchanged. - Returns: - Whether it reconnected within ``reconnect_attempts`` - Raises: - FatalError: If the new introspection's structure is different + ReconnectFailedError: If it doesn't reconnect within ``reconnect_attempts`` + IntrospectionMismatchError: If the new introspection's structure is + different """ for attempt in range(connection.reconnect_attempts): @@ -212,15 +210,18 @@ async def _reconnect(self, connection: Connection) -> bool: _reconnecting.reset(token) if connection.introspection_changed(introspection): - raise FatalError( + raise IntrospectionMismatchError( f"{type(connection).__name__} introspection changed structure on " "reconnect, so the controllers built from it are out of date" ) connection.state.mark_up() - return True + return - return False + raise ReconnectFailedError( + f"{type(connection).__name__} went down and didn't reconnect within " + f"{connection.reconnect_attempts} attempts" + ) async def _mark_up(self, connection: Connection) -> None: """Mark connected each controller holding a connection, re-run its read-once diff --git a/tests/test_control_system.py b/tests/test_control_system.py index 281d7fc8..32fb39f2 100644 --- a/tests/test_control_system.py +++ b/tests/test_control_system.py @@ -8,7 +8,7 @@ from fastcs.control_system import FastCS from fastcs.controllers import Controller, ControllerSpec from fastcs.datatypes import Int -from fastcs.exceptions import FatalError +from fastcs.exceptions import IntrospectionMismatchError from fastcs.methods import Command, command from fastcs.transports import Transport from fastcs.util import ONCE @@ -182,7 +182,7 @@ async def test_serve_raises_fatal_error_and_closes_connections(): transport = ForeverTransport() fastcs = FastCS([ControllerSpec("A", DroppingController, connection)], [transport]) - with pytest.raises(FatalError, match="changed structure"): + with pytest.raises(IntrospectionMismatchError): await asyncio.wait_for(fastcs.serve(interactive=False), timeout=1) # Once on reconnect, once on shutdown diff --git a/tests/test_controller_runner.py b/tests/test_controller_runner.py index 47aebb3a..8d78dbc7 100644 --- a/tests/test_controller_runner.py +++ b/tests/test_controller_runner.py @@ -6,7 +6,7 @@ from fastcs.connections import Connection, DisconnectedError, disconnect_on from fastcs.connections.connection import _structure from fastcs.controllers import Controller, ControllerSpec -from fastcs.exceptions import FatalError +from fastcs.exceptions import IntrospectionMismatchError, ReconnectFailedError from fastcs.methods import scan from fastcs.runner import ControllerRunner from fastcs.util import ONCE @@ -377,7 +377,7 @@ async def test_introspection_changing_structure_on_reconnect_is_fatal(): await runner.start() connection.state.mark_down() - with pytest.raises(FatalError, match="changed structure"): + with pytest.raises(IntrospectionMismatchError): await asyncio.wait_for(runner.wait_for_fatal_error(), timeout=1) await runner.stop() @@ -499,9 +499,10 @@ async def test_watcher_reconnects_then_marks_controllers_up(): @pytest.mark.asyncio -async def test_watcher_gives_up_and_leaves_controllers_down(): - connection = FlakyConnection(failures=0) - connection.reconnect_attempts = 0 +@pytest.mark.parametrize("attempts", [0, 2]) +async def test_watcher_raises_when_reconnect_attempts_run_out(attempts: int): + connection = FlakyConnection(failures=attempts) + connection.reconnect_attempts = attempts runner = await ControllerRunner.build( [ControllerSpec("A", HoldingController, connection)] ) @@ -509,9 +510,10 @@ async def test_watcher_gives_up_and_leaves_controllers_down(): controller = runner.controllers[0] connection.state.mark_down() - await asyncio.wait_for(runner._watchers[id(connection)], timeout=1) + with pytest.raises(ReconnectFailedError): + await asyncio.wait_for(runner.wait_for_fatal_error(), timeout=1) - assert connection.connects == 1 + assert connection.connects == 1 + attempts assert controller.connected.get() is False assert not runner._scan_gates[id(controller)].is_set() await runner.stop() From 5c6d27c22116e7d673b0fd203f74cb0663df87c7 Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Mon, 16 Nov 2026 11:31:57 +0000 Subject: [PATCH 20/22] set default reconnect settings to None, and improve implementation of reconnect attempts --- docs/explanations/connections.md | 9 +++++--- src/fastcs/connections/connection.py | 20 +++++++++++++++--- src/fastcs/launch.py | 4 ++-- src/fastcs/runner.py | 28 ++++++++++++++++++------- tests/test_connection.py | 7 +++++++ tests/test_control_system.py | 3 +++ tests/test_controller_runner.py | 31 ++++++++++++++++++++++++---- 7 files changed, 83 insertions(+), 19 deletions(-) diff --git a/docs/explanations/connections.md b/docs/explanations/connections.md index 25a3354c..9c78a9fa 100644 --- a/docs/explanations/connections.md +++ b/docs/explanations/connections.md @@ -100,8 +100,8 @@ connections: reconnect_period: 2.0 ``` -`reconnect_attempts: 0` means stop as soon as the connection goes down. A connection class can set its own -defaults: +`reconnect_attempts: 0` or `null` means stop as soon as the connection goes down. That +is the default for a connection class that doesn't set its own, which it does with: ```python class EigerConnection(Connection): @@ -109,9 +109,12 @@ class EigerConnection(Connection): reconnect_period = 3.0 ``` +A class that sets `reconnect_attempts` must set `reconnect_period` too, or defining it +raises a `TypeError`. + | Connection | `reconnect_attempts` | `reconnect_period` | |---|---|---| -| `Connection` | 10 | 5.0 | +| `Connection` | `None` | `None` | | `IPConnection` | 30 | 2.0 | | `HTTPConnection` | 60 | 5.0 | | `SerialConnection` | 10 | 1.0 | diff --git a/src/fastcs/connections/connection.py b/src/fastcs/connections/connection.py index d10d05bd..dfa40dc4 100644 --- a/src/fastcs/connections/connection.py +++ b/src/fastcs/connections/connection.py @@ -77,13 +77,27 @@ async def introspect(self) -> EigerIntrospection: ... # Replaced with a live state by the framework when it opens this connection state: ConnectionState = _InertConnectionState() - # How many times, and how many seconds apart, the framework tries to reconnect - reconnect_attempts: int = 10 - reconnect_period: float = 5.0 + # How many times, and how many seconds apart, the framework tries to reconnect. + # None or 0 means don't reconnect and stop FastCS when the link goes down + reconnect_attempts: int | None = None + reconnect_period: float | None = None _introspection: IntrospectionT _introspection_structure: Any + def __init_subclass__(cls, **kwargs: Any) -> None: + """Check the reconnect settings of a connection class. + + Raises: + TypeError: If ``reconnect_attempts`` is set without ``reconnect_period`` + + """ + super().__init_subclass__(**kwargs) + if cls.reconnect_attempts and cls.reconnect_period is None: + raise TypeError( + f"{cls.__name__} sets reconnect_attempts but not reconnect_period" + ) + @property def introspection(self) -> IntrospectionT: """What ``introspect`` returned when the framework opened this connection. diff --git a/src/fastcs/launch.py b/src/fastcs/launch.py index 47e97a74..77d59a27 100644 --- a/src/fastcs/launch.py +++ b/src/fastcs/launch.py @@ -266,11 +266,11 @@ def _connection_settings_model(connection_type: type[Connection]) -> type[BaseMo fields[name] = (hints[name], default) fields["reconnect_attempts"] = ( - int, + int | None, Field(default=connection_type.reconnect_attempts, ge=0), ) fields["reconnect_period"] = ( - float, + float | None, Field(default=connection_type.reconnect_period, ge=0), ) diff --git a/src/fastcs/runner.py b/src/fastcs/runner.py index 0cb2e489..46b5c0bf 100644 --- a/src/fastcs/runner.py +++ b/src/fastcs/runner.py @@ -184,14 +184,29 @@ async def _reconnect(self, connection: Connection) -> None: connection up if the introspection's structure is unchanged. Raises: - ReconnectFailedError: If it doesn't reconnect within ``reconnect_attempts`` + ReconnectFailedError: If reconnecting is disabled, it has no + ``reconnect_period``, or it doesn't reconnect within + ``reconnect_attempts`` IntrospectionMismatchError: If the new introspection's structure is different """ - for attempt in range(connection.reconnect_attempts): + name = type(connection).__name__ + attempts = connection.reconnect_attempts + period = connection.reconnect_period + if not attempts: + raise ReconnectFailedError( + f"{name} went down, and reconnecting is disabled" + ) + if period is None: + raise ReconnectFailedError( + f"{name} can't reconnect, as it has reconnect_attempts but no " + "reconnect_period" + ) + + for attempt in range(attempts): if attempt: - await asyncio.sleep(connection.reconnect_period) + await asyncio.sleep(period) for dependency in self._dependencies[id(connection)]: await dependency.state.wait_up() @@ -211,16 +226,15 @@ async def _reconnect(self, connection: Connection) -> None: if connection.introspection_changed(introspection): raise IntrospectionMismatchError( - f"{type(connection).__name__} introspection changed structure on " - "reconnect, so the controllers built from it are out of date" + f"{name} introspection changed structure on reconnect, so the " + "controllers built from it are out of date" ) connection.state.mark_up() return raise ReconnectFailedError( - f"{type(connection).__name__} went down and didn't reconnect within " - f"{connection.reconnect_attempts} attempts" + f"{name} went down and didn't reconnect within {attempts} attempts" ) async def _mark_up(self, connection: Connection) -> None: diff --git a/tests/test_connection.py b/tests/test_connection.py index c90d5452..d44fddaf 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -62,3 +62,10 @@ async def test_bare_decorator_and_inert_state(): with pytest.raises(ConnectionResetError): await connection.send() assert connection.state.up + + +def test_reconnect_attempts_without_period_raises(): + with pytest.raises(TypeError, match="sets reconnect_attempts but not"): + + class _NoPeriodConnection(_FlakyConnection): + reconnect_attempts = 3 diff --git a/tests/test_control_system.py b/tests/test_control_system.py index 32fb39f2..d5e9afbf 100644 --- a/tests/test_control_system.py +++ b/tests/test_control_system.py @@ -133,6 +133,9 @@ async def test_connections_closed_if_startup_fails(): class ChangingConnection(Connection[dict[str, int]]): + reconnect_attempts = 10 + reconnect_period = 0 + def __init__(self) -> None: self.introspections = 0 self.closes = 0 diff --git a/tests/test_controller_runner.py b/tests/test_controller_runner.py index 8d78dbc7..72e89f46 100644 --- a/tests/test_controller_runner.py +++ b/tests/test_controller_runner.py @@ -102,6 +102,9 @@ async def test_clearing_a_scan_gate_pauses_only_that_controllers_scans(): class RecordingConnection(Connection): + reconnect_attempts = 10 + reconnect_period = 0 + def __init__(self, events: list[str], name: str = "") -> None: self.events = events self.suffix = f" {name}" if name else "" @@ -318,6 +321,9 @@ class DeviceIntrospection: class IntrospectingConnection(Connection[DeviceIntrospection]): + reconnect_attempts = 10 + reconnect_period = 0 + async def connect(self) -> None: pass @@ -458,6 +464,7 @@ def test_seal_twice_raises(): class FlakyConnection(Connection): + reconnect_attempts = 10 reconnect_period = 0 def __init__(self, failures: int) -> None: @@ -499,9 +506,9 @@ async def test_watcher_reconnects_then_marks_controllers_up(): @pytest.mark.asyncio -@pytest.mark.parametrize("attempts", [0, 2]) -async def test_watcher_raises_when_reconnect_attempts_run_out(attempts: int): - connection = FlakyConnection(failures=attempts) +@pytest.mark.parametrize("attempts", [None, 0, 2]) +async def test_watcher_raises_when_reconnect_attempts_run_out(attempts: int | None): + connection = FlakyConnection(failures=attempts or 0) connection.reconnect_attempts = attempts runner = await ControllerRunner.build( [ControllerSpec("A", HoldingController, connection)] @@ -513,12 +520,27 @@ async def test_watcher_raises_when_reconnect_attempts_run_out(attempts: int): with pytest.raises(ReconnectFailedError): await asyncio.wait_for(runner.wait_for_fatal_error(), timeout=1) - assert connection.connects == 1 + attempts + assert connection.connects == 1 + (attempts or 0) assert controller.connected.get() is False assert not runner._scan_gates[id(controller)].is_set() await runner.stop() +@pytest.mark.asyncio +async def test_watcher_raises_when_instance_has_attempts_but_no_period(): + connection = FlakyConnection(failures=0) + connection.reconnect_period = None + runner = await ControllerRunner.build( + [ControllerSpec("A", HoldingController, connection)] + ) + await runner.start() + + connection.state.mark_down() + with pytest.raises(ReconnectFailedError, match="no reconnect_period"): + await asyncio.wait_for(runner.wait_for_fatal_error(), timeout=1) + await runner.stop() + + class HeldEigerConnection(EigerConnection): def __init__(self, events: list[str], name: str = "") -> None: super().__init__(events, name) @@ -557,6 +579,7 @@ async def test_reconnect_waits_for_dependencies_to_be_up(): class HandshakeConnection(Connection): + reconnect_attempts = 10 reconnect_period = 0 def __init__(self) -> None: From d3fb8bede61efb75030761b24c4e1bfaf48bf54f Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Mon, 16 Nov 2026 11:39:12 +0000 Subject: [PATCH 21/22] check reconnect_period is not None when reconnect_attempts is set, at class init and using a schema for fastcs.yaml --- src/fastcs/demo/schema.json | 19 +++++++--- src/fastcs/launch.py | 70 ++++++++++++++++++++++++++++++++----- tests/test_launch.py | 9 +++++ 3 files changed, 85 insertions(+), 13 deletions(-) diff --git a/src/fastcs/demo/schema.json b/src/fastcs/demo/schema.json index 1b8bc982..f45f03a7 100644 --- a/src/fastcs/demo/schema.json +++ b/src/fastcs/demo/schema.json @@ -178,12 +178,10 @@ "additionalProperties": false, "properties": { "ip": { - "default": "127.0.0.1", "title": "Ip", "type": "string" }, "port": { - "default": 25565, "title": "Port", "type": "integer" }, @@ -208,10 +206,17 @@ "type": "integer" }, "reconnect_attempts": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], "default": 30, - "minimum": 0, - "title": "Reconnect Attempts", - "type": "integer" + "title": "Reconnect Attempts" }, "reconnect_period": { "default": 2.0, @@ -220,6 +225,10 @@ "type": "number" } }, + "required": [ + "ip", + "port" + ], "title": "IPConnectionSettings", "type": "object" }, diff --git a/src/fastcs/launch.py b/src/fastcs/launch.py index 77d59a27..760965c2 100644 --- a/src/fastcs/launch.py +++ b/src/fastcs/launch.py @@ -5,17 +5,26 @@ import json from pathlib import Path from typing import ( + TYPE_CHECKING, Annotated, Any, Literal, Optional, + Self, Union, get_origin, get_type_hints, ) import typer -from pydantic import BaseModel, Field, ValidationError, create_model +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationError, + create_model, + model_validator, +) from ruamel.yaml import YAML from fastcs import __version__ @@ -234,6 +243,48 @@ def _controller_specs( return specs +class _ConnectionSettings(BaseModel): + """Base for each connection's block under `connections:`""" + + model_config = ConfigDict(extra="forbid") + + if TYPE_CHECKING: + # Fields added by _connection_settings_model, declared for type checkers + reconnect_attempts: int | None + reconnect_period: float | None + + @model_validator(mode="after") + def _check_reconnect_period(self) -> Self: + """Check there is a period to reconnect with. + + Raises: + ValueError: If ``reconnect_attempts`` is set without ``reconnect_period`` + + """ + if self.reconnect_attempts and self.reconnect_period is None: + raise ValueError("reconnect_attempts is set, so reconnect_period is needed") + return self + + +class _ConnectionSettingsWithoutPeriod(_ConnectionSettings): + """Base for the block of a connection class with no ``reconnect_period``, whose + schema asks for one in the editor once ``reconnect_attempts`` is set""" + + model_config = ConfigDict( + json_schema_extra={ + "if": { + "properties": {"reconnect_attempts": {"type": "integer", "minimum": 1}}, + "required": ["reconnect_attempts"], + }, + "then": {"required": ["reconnect_period"]}, + } + ) + + +def _drop_default(schema: dict[str, Any]) -> None: + schema.pop("default", None) + + @functools.cache def _connection_settings_model(connection_type: type[Connection]) -> type[BaseModel]: """Build the Pydantic model for one connection's block under `connections:`. @@ -269,14 +320,17 @@ def _connection_settings_model(connection_type: type[Connection]) -> type[BaseMo int | None, Field(default=connection_type.reconnect_attempts, ge=0), ) - fields["reconnect_period"] = ( - float | None, - Field(default=connection_type.reconnect_period, ge=0), - ) - return create_model( - f"{connection_type.__name__}Settings", __config__={"extra": "forbid"}, **fields - ) + # A period can't be set to null in the block, as there's no use for that + base = _ConnectionSettings + if connection_type.reconnect_period is None: + base = _ConnectionSettingsWithoutPeriod + period = Field(default=None, ge=0, json_schema_extra=_drop_default) + else: + period = Field(default=connection_type.reconnect_period, ge=0) + fields["reconnect_period"] = (float, period) + + return create_model(f"{connection_type.__name__}Settings", __base__=base, **fields) def _options_field_definitions(options_type: type) -> dict[str, tuple[Any, Any]]: diff --git a/tests/test_launch.py b/tests/test_launch.py index 40939dfa..780ca8f3 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -383,6 +383,15 @@ def test_reconnect_setting_defaults_come_from_the_connection_class(): assert fields["reconnect_period"].default == IPConnection.reconnect_period +def test_reconnect_attempts_without_period_rejected_by_settings_and_schema(): + model = _connection_settings_model(NoSettingsConnection) + with pytest.raises(ValidationError, match="reconnect_period is needed"): + model.model_validate({"reconnect_attempts": 3}) + + assert model.model_json_schema()["then"] == {"required": ["reconnect_period"]} + assert "then" not in _connection_settings_model(IPConnection).model_json_schema() + + def test_parametrised_and_no_settings_connections(): options_model = _build_options_model([WithGenericConnections]) options_model.model_validate( From 0db84f6c3ff8632da36cebf599a8d22f19075a9c Mon Sep 17 00:00:00 2001 From: Tom Kane Date: Mon, 16 Nov 2026 16:16:19 +0000 Subject: [PATCH 22/22] update how disconnect_on works, explicitly needing anything beyond DiconnectedError, and using new disconnect_after_timeouts when using TimeoutError --- docs/explanations/connections.md | 28 ++++++-- src/fastcs/connections/connection.py | 71 +++++++++++++++++---- src/fastcs/connections/http_connection.py | 33 ++++++---- src/fastcs/connections/ip_connection.py | 15 +++-- src/fastcs/connections/serial_connection.py | 15 +++-- src/fastcs/demo/schema.json | 6 ++ src/fastcs/launch.py | 19 +++++- tests/test_connection.py | 55 ++++++++++++++-- tests/test_launch.py | 1 + 9 files changed, 198 insertions(+), 45 deletions(-) diff --git a/docs/explanations/connections.md b/docs/explanations/connections.md index 9c78a9fa..009dc456 100644 --- a/docs/explanations/connections.md +++ b/docs/explanations/connections.md @@ -25,9 +25,7 @@ reply = await connection.query("R?") # sends "R?\r\n", returns the reply withou For binary protocols, `write_bytes` and `query_bytes` send raw bytes without a terminator. If an exchange fails or takes longer than `timeout`, the link is closed and later calls raise `DisconnectedError`. If the link fails, such as the device closing the -socket or a serial adapter being unplugged, the exchange raises `ConnectionError`, which -`DisconnectedError` subclasses, so -`except (ConnectionError, TimeoutError)` catches any link failure. +socket or a serial adapter being unplugged, the exchange raises `ConnectionError`. `HTTPConnection` talks to a device with a JSON API. `get`, `put`, `post` and `delete` take a path relative to its `url`, send JSON, and return the parsed reply. An error @@ -56,15 +54,23 @@ from fastcs.connections import Connection, disconnect_on class EigerConnection(Connection): + disconnect_after_timeouts = 3 ... - @disconnect_on(TimeoutError) + @disconnect_on(ConnectionError, TimeoutError) async def get(self, key: str) -> Any: return await self.client.get(f"/detector/{key}") ``` -If the method raises a `ConnectionError`, a `DisconnectedError` or one of the listed -exceptions, the connection is marked down and the exception carries on to the caller. +If the method raises a `DisconnectedError` or one of the listed exceptions, the +connection is marked down and the exception carries on to the caller. Bare +`@disconnect_on` only lists `DisconnectedError`, so list `ConnectionError` too if it +means the link has gone. + +If `TimeoutError` is listed, directly or through a base class like `OSError`, timeouts +are counted: the connection is marked down after `disconnect_after_timeouts` in a row, +1 by default, and any call that works starts the count again. + While it's down, decorated methods raise `DisconnectedError` without touching the device, except when called from the connection's own `connect()` or `introspect()` while FastCS reconnects it. Other exceptions, like an error reply from the device, leave the @@ -89,7 +95,8 @@ controllers again. ### Reconnect settings -Each connection's block in `fastcs.yaml` can set how it reconnects: +Each connection's block in `fastcs.yaml` can set how it reconnects, and how many +timeouts mean it has gone: ```yaml connections: @@ -98,6 +105,7 @@ connections: port: 4000 reconnect_attempts: 5 reconnect_period: 2.0 + disconnect_after_timeouts: 1 ``` `reconnect_attempts: 0` or `null` means stop as soon as the connection goes down. That @@ -119,6 +127,12 @@ raises a `TypeError`. | `HTTPConnection` | 60 | 5.0 | | `SerialConnection` | 10 | 1.0 | +`disconnect_after_timeouts` is 1 unless a class sets it, and must be at least 1. It only +applies to methods whose `@disconnect_on` lists `TimeoutError`, so `HTTPConnection`'s +timeouts never mark it down. `IPConnection` and `SerialConnection` close the link on a +timeout, so a late reply isn't read as the reply to the next exchange, and a higher +value only delays marking them down until the next call. + ### Which controllers a connection belongs to FastCS finds the controllers to mark down by looking for the connection on them, so: diff --git a/src/fastcs/connections/connection.py b/src/fastcs/connections/connection.py index dfa40dc4..e1f2a9d9 100644 --- a/src/fastcs/connections/connection.py +++ b/src/fastcs/connections/connection.py @@ -16,8 +16,8 @@ """What a `Connection`'s ``introspect`` returns, or ``None`` if it doesn't introspect""" -class DisconnectedError(ConnectionError): - """Raised when using a connection whose link isn't open""" +class DisconnectedError(Exception): + """Raised when using a connection whose link is down or closed""" class ConnectionState: @@ -30,6 +30,7 @@ def __init__(self) -> None: self._up = asyncio.Event() self._up.set() self._down = asyncio.Event() + self._timeouts = 0 @property def up(self) -> bool: @@ -43,9 +44,24 @@ def mark_down(self) -> None: def mark_up(self) -> None: """Mark the link up, waking anything waiting for that""" + self._timeouts = 0 self._down.clear() self._up.set() + def count_timeout(self) -> int: + """Count a timeout from a ``@disconnect_on`` method. + + Returns: + How many timeouts there have been in a row + + """ + self._timeouts += 1 + return self._timeouts + + def reset_timeouts(self) -> None: + """Start counting timeouts again, after a ``@disconnect_on`` method works""" + self._timeouts = 0 + async def wait_down(self) -> None: """Wait until the link is marked down""" await self._down.wait() @@ -61,6 +77,10 @@ class _InertConnectionState(ConnectionState): def mark_down(self) -> None: pass + def count_timeout(self) -> int: + # Shared by every unopened connection, so it doesn't count + return 0 + class Connection(ABC, Generic[IntrospectionT]): """A link to a device, which can optionally introspect it. @@ -82,6 +102,10 @@ async def introspect(self) -> EigerIntrospection: ... reconnect_attempts: int | None = None reconnect_period: float | None = None + # How many timeouts in a row mean the link has gone, from @disconnect_on methods + # that list TimeoutError. At least 1 + disconnect_after_timeouts: int = 1 + _introspection: IntrospectionT _introspection_structure: Any @@ -89,7 +113,8 @@ def __init_subclass__(cls, **kwargs: Any) -> None: """Check the reconnect settings of a connection class. Raises: - TypeError: If ``reconnect_attempts`` is set without ``reconnect_period`` + TypeError: If ``reconnect_attempts`` is set without ``reconnect_period``, + or ``disconnect_after_timeouts`` isn't an integer of at least 1 """ super().__init_subclass__(**kwargs) @@ -97,6 +122,12 @@ def __init_subclass__(cls, **kwargs: Any) -> None: raise TypeError( f"{cls.__name__} sets reconnect_attempts but not reconnect_period" ) + timeouts = cls.disconnect_after_timeouts + if not isinstance(timeouts, int) or timeouts < 1: + raise TypeError( + f"{cls.__name__} sets disconnect_after_timeouts to {timeouts!r}, but " + "it must be an integer of at least 1" + ) @property def introspection(self) -> IntrospectionT: @@ -164,19 +195,26 @@ def disconnect_on( def disconnect_on(*errors: Any) -> Any: """Mark a connection IO method whose failure means the link has gone. - If the method raises one of ``errors``, a `ConnectionError` or a - `DisconnectedError`, the connection's ``state`` is marked down and the exception - is re-raised. If it is called while the state is down, it raises - `DisconnectedError` without calling the method. + If the method raises a `DisconnectedError` or one of ``errors``, the + connection's ``state`` is marked down and the exception is re-raised. If it is + called while the state is down, it raises `DisconnectedError` without calling the + method. Used bare, as ``@disconnect_on``, only `DisconnectedError` marks it down. + + If ``errors`` includes `TimeoutError`, directly or through a base class like + `OSError`, a timeout only marks it down once there have been the connection's + ``disconnect_after_timeouts`` in a row. A call that returns starts the count again. Example:: class EigerConnection(Connection): - @disconnect_on(TimeoutError) + disconnect_after_timeouts = 3 + + @disconnect_on(ConnectionError, TimeoutError) async def get(self, key: str) -> Any: ... Args: - errors: Other exceptions that mean the link has gone + errors: The exceptions, besides `DisconnectedError`, that mean the link has + gone Raises: TypeError: If the method isn't async, or an error isn't an exception class @@ -186,7 +224,8 @@ async def get(self, key: str) -> Any: ... # Used bare, as @disconnect_on, so the only argument is the method itself return disconnect_on()(errors[0]) - link_errors = (ConnectionError, *errors) + link_errors = (DisconnectedError, *errors) + counts_timeouts = issubclass(TimeoutError, link_errors) def decorator(method: _MethodT) -> _MethodT: name = getattr(method, "__qualname__", repr(method)) @@ -204,11 +243,21 @@ async def wrapper(self: Connection, *args: Any, **kwargs: Any) -> Any: raise DisconnectedError(f"{type(self).__name__} is disconnected") try: - return await method(self, *args, **kwargs) + result = await method(self, *args, **kwargs) + except TimeoutError: + # Before link_errors, which may include it through a base class + if counts_timeouts: + limit = self.disconnect_after_timeouts + if self.state.count_timeout() >= limit: + self.state.mark_down() + raise except link_errors: self.state.mark_down() raise + self.state.reset_timeouts() + return result + return cast(_MethodT, wrapper) return decorator diff --git a/src/fastcs/connections/http_connection.py b/src/fastcs/connections/http_connection.py index 97cfc2e1..66e47d7c 100644 --- a/src/fastcs/connections/http_connection.py +++ b/src/fastcs/connections/http_connection.py @@ -18,6 +18,11 @@ class HTTPConnection(Connection[IntrospectionT], Tracer): """An HTTP link to a device with a JSON API. + If a request raises a `ConnectionError`, the link is closed and the connection + marked down, so later calls raise `DisconnectedError`. A `TimeoutError` leaves + the link open and doesn't mark it down, as one slow request doesn't mean the + server has gone. + Args: url: The base URL that request paths are relative to timeout: Seconds to wait to connect, or for each response @@ -89,8 +94,9 @@ async def get(self, path: str, params: dict[str, Any] | None = None) -> Any: The parsed JSON reply, or ``None`` if the reply is empty Raises: - DisconnectedError: If the link isn't open - ConnectionError: If the server can't be reached + DisconnectedError: If the link is down or closed + ConnectionError: If the request fails on the link, such as the server + being unreachable TimeoutError: If the reply takes longer than ``timeout`` httpx.HTTPStatusError: If the reply has an error status ValueError: If the reply isn't valid JSON @@ -105,8 +111,9 @@ async def put(self, path: str, json: Any = None) -> Any: The parsed JSON reply, or ``None`` if the reply is empty Raises: - DisconnectedError: If the link isn't open - ConnectionError: If the server can't be reached + DisconnectedError: If the link is down or closed + ConnectionError: If the request fails on the link, such as the server + being unreachable TimeoutError: If the reply takes longer than ``timeout`` httpx.HTTPStatusError: If the reply has an error status ValueError: If the reply isn't valid JSON @@ -121,8 +128,9 @@ async def post(self, path: str, json: Any = None) -> Any: The parsed JSON reply, or ``None`` if the reply is empty Raises: - DisconnectedError: If the link isn't open - ConnectionError: If the server can't be reached + DisconnectedError: If the link is down or closed + ConnectionError: If the request fails on the link, such as the server + being unreachable TimeoutError: If the reply takes longer than ``timeout`` httpx.HTTPStatusError: If the reply has an error status ValueError: If the reply isn't valid JSON @@ -137,8 +145,9 @@ async def delete(self, path: str) -> Any: The parsed JSON reply, or ``None`` if the reply is empty Raises: - DisconnectedError: If the link isn't open - ConnectionError: If the server can't be reached + DisconnectedError: If the link is down or closed + ConnectionError: If the request fails on the link, such as the server + being unreachable TimeoutError: If the reply takes longer than ``timeout`` httpx.HTTPStatusError: If the reply has an error status ValueError: If the reply isn't valid JSON @@ -158,14 +167,16 @@ async def request(self, method: str, path: str, **kwargs: Any) -> httpx.Response The response, whatever its status Raises: - DisconnectedError: If the link isn't open - ConnectionError: If the server can't be reached + DisconnectedError: If the link is down or closed + ConnectionError: If the request fails on the link, such as the server + being unreachable TimeoutError: If the reply takes longer than ``timeout`` """ return await self._send(method, path, **kwargs) async def close(self) -> None: + """Close the link, if it's open""" async with self._lock: if self._client is None: return @@ -174,7 +185,7 @@ async def close(self) -> None: self._client = None await client.aclose() - @disconnect_on + @disconnect_on(ConnectionError) async def _send(self, method: str, path: str, **kwargs: Any) -> httpx.Response: client = self._client if client is None: diff --git a/src/fastcs/connections/ip_connection.py b/src/fastcs/connections/ip_connection.py index 276abeab..3c10cf77 100644 --- a/src/fastcs/connections/ip_connection.py +++ b/src/fastcs/connections/ip_connection.py @@ -14,6 +14,12 @@ class IPConnection(Connection[IntrospectionT], Tracer): """A TCP link to a device that exchanges terminated messages. + If an exchange raises an `OSError`, such as a `ConnectionError`, the link is + closed and the connection marked down, so later calls raise `DisconnectedError`. + A `TimeoutError` closes the link too, as a late reply would be read as the reply + to the next exchange, so a ``disconnect_after_timeouts`` above 1 only delays + marking the connection down until the next call. + Args: ip: The address to connect to port: The port to connect to @@ -76,7 +82,7 @@ async def write(self, message: str) -> None: """Send a message, adding the terminator. Raises: - DisconnectedError: If the link isn't open + DisconnectedError: If the link is down or closed ConnectionError: If the device closes the link TimeoutError: If sending takes longer than ``timeout`` @@ -96,7 +102,7 @@ async def query(self, message: str) -> str: The reply, without its terminator Raises: - DisconnectedError: If the link isn't open + DisconnectedError: If the link is down or closed ConnectionError: If the device closes the link TimeoutError: If the reply takes longer than ``timeout`` ValueError: If the reply is longer than ``limit`` or can't be decoded @@ -123,7 +129,7 @@ async def write_bytes(self, data: bytes) -> None: """Send raw bytes, without a terminator. Raises: - DisconnectedError: If the link isn't open + DisconnectedError: If the link is down or closed ConnectionError: If the device closes the link TimeoutError: If sending takes longer than ``timeout`` @@ -142,7 +148,7 @@ async def query_bytes(self, data: bytes, size: int) -> bytes: The reply, exactly ``size`` bytes long Raises: - DisconnectedError: If the link isn't open + DisconnectedError: If the link is down or closed ConnectionError: If the device closes the link TimeoutError: If the reply takes longer than ``timeout`` @@ -156,6 +162,7 @@ async def query_bytes(self, data: bytes, size: int) -> bytes: return reply async def close(self) -> None: + """Close the link, if it's open""" async with self._lock: if self._streams is None: return diff --git a/src/fastcs/connections/serial_connection.py b/src/fastcs/connections/serial_connection.py index eb28e992..d1e587f9 100644 --- a/src/fastcs/connections/serial_connection.py +++ b/src/fastcs/connections/serial_connection.py @@ -18,6 +18,12 @@ class SerialConnection(Connection[IntrospectionT], Tracer): """A serial link to a device that exchanges terminated messages. + If an exchange raises an `OSError`, such as a `ConnectionError`, the link is + closed and the connection marked down, so later calls raise `DisconnectedError`. + A `TimeoutError` closes the link too, as a late reply would be read as the reply + to the next exchange, so a ``disconnect_after_timeouts`` above 1 only delays + marking the connection down until the next call. + Args: port: The serial port to open baud: The baud rate @@ -101,7 +107,7 @@ async def write(self, message: str) -> None: """Send a message, adding the terminator. Raises: - DisconnectedError: If the link isn't open + DisconnectedError: If the link is down or closed ConnectionError: If the port fails, such as an unplugged adapter TimeoutError: If sending takes longer than ``timeout`` @@ -120,7 +126,7 @@ async def query(self, message: str) -> str: The reply, without its terminator Raises: - DisconnectedError: If the link isn't open + DisconnectedError: If the link is down or closed ConnectionError: If the port fails, such as an unplugged adapter TimeoutError: If the reply takes longer than ``timeout`` ValueError: If the reply is longer than ``limit`` or can't be decoded @@ -145,7 +151,7 @@ async def write_bytes(self, data: bytes) -> None: """Send raw bytes, without a terminator. Raises: - DisconnectedError: If the link isn't open + DisconnectedError: If the link is down or closed ConnectionError: If the port fails, such as an unplugged adapter TimeoutError: If sending takes longer than ``timeout`` @@ -163,7 +169,7 @@ async def query_bytes(self, data: bytes, size: int) -> bytes: The reply, exactly ``size`` bytes long Raises: - DisconnectedError: If the link isn't open + DisconnectedError: If the link is down or closed ConnectionError: If the port fails, such as an unplugged adapter TimeoutError: If the reply takes longer than ``timeout`` @@ -176,6 +182,7 @@ async def query_bytes(self, data: bytes, size: int) -> bytes: return reply async def close(self) -> None: + """Close the link, if it's open""" async with self._lock: if self._stream is None: return diff --git a/src/fastcs/demo/schema.json b/src/fastcs/demo/schema.json index f45f03a7..4ad4b689 100644 --- a/src/fastcs/demo/schema.json +++ b/src/fastcs/demo/schema.json @@ -223,6 +223,12 @@ "minimum": 0, "title": "Reconnect Period", "type": "number" + }, + "disconnect_after_timeouts": { + "default": 1, + "minimum": 1, + "title": "Disconnect After Timeouts", + "type": "integer" } }, "required": [ diff --git a/src/fastcs/launch.py b/src/fastcs/launch.py index 760965c2..71470f21 100644 --- a/src/fastcs/launch.py +++ b/src/fastcs/launch.py @@ -226,9 +226,11 @@ def _controller_specs( values = {k: getattr(settings, k) for k in type(settings).model_fields} attempts = values.pop("reconnect_attempts") period = values.pop("reconnect_period") + timeouts = values.pop("disconnect_after_timeouts") connection = connection_type(**values) connection.reconnect_attempts = attempts connection.reconnect_period = period + connection.disconnect_after_timeouts = timeouts kwargs[name] = connection if registered.options_name is not None: @@ -252,6 +254,7 @@ class _ConnectionSettings(BaseModel): # Fields added by _connection_settings_model, declared for type checkers reconnect_attempts: int | None reconnect_period: float | None + disconnect_after_timeouts: int @model_validator(mode="after") def _check_reconnect_period(self) -> Self: @@ -285,12 +288,20 @@ def _drop_default(schema: dict[str, Any]) -> None: schema.pop("default", None) +_FRAMEWORK_SETTINGS = ( + "reconnect_attempts", + "reconnect_period", + "disconnect_after_timeouts", +) +"""Settings in each connection's block that are for the framework, not ``__init__``""" + + @functools.cache def _connection_settings_model(connection_type: type[Connection]) -> type[BaseModel]: """Build the Pydantic model for one connection's block under `connections:`. The block has the constructor's arguments, and the framework's - ``reconnect_attempts`` and ``reconnect_period``. + ``reconnect_attempts``, ``reconnect_period`` and ``disconnect_after_timeouts``. Raises: LaunchError: If a constructor argument has no type hint, or has the name of a @@ -308,7 +319,7 @@ def _connection_settings_model(connection_type: type[Connection]) -> type[BaseMo f"Expected typehinting in '{connection_type.__name__}.__init__' " f"but received {sig}. Add a typehint for `{name}`." ) - if name in ("reconnect_attempts", "reconnect_period"): + if name in _FRAMEWORK_SETTINGS: raise LaunchError( f"'{connection_type.__name__}.__init__' can't take `{name}`, which is " "a framework setting." @@ -329,6 +340,10 @@ def _connection_settings_model(connection_type: type[Connection]) -> type[BaseMo else: period = Field(default=connection_type.reconnect_period, ge=0) fields["reconnect_period"] = (float, period) + fields["disconnect_after_timeouts"] = ( + int, + Field(default=connection_type.disconnect_after_timeouts, ge=1), + ) return create_model(f"{connection_type.__name__}Settings", __base__=base, **fields) diff --git a/tests/test_connection.py b/tests/test_connection.py index d44fddaf..fee75ba1 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -26,8 +26,8 @@ async def get(self, error: Exception | None = None) -> int: return self.calls @disconnect_on - async def send(self) -> None: - raise ConnectionResetError() + async def send(self, error: Exception) -> None: + raise error @pytest.mark.asyncio @@ -50,22 +50,65 @@ async def test_listed_error_marks_down_and_gates_later_calls(): connection.state.mark_up() assert await connection.get() == 3 + # Not listed, so it isn't a link error, though it is a ConnectionError with pytest.raises(ConnectionResetError): await connection.get(ConnectionResetError()) - assert not connection.state.up + assert connection.state.up @pytest.mark.asyncio -async def test_bare_decorator_and_inert_state(): +async def test_bare_decorator_only_marks_down_on_disconnected_error(): connection = _FlakyConnection() + with pytest.raises(DisconnectedError): + await connection.send(DisconnectedError()) + assert connection.state.up # The inert state of an unopened connection - with pytest.raises(ConnectionResetError): - await connection.send() + connection.state = ConnectionState() + for error in (ConnectionResetError(), TimeoutError()): + with pytest.raises(type(error)): + await connection.send(error) assert connection.state.up + with pytest.raises(DisconnectedError): + await connection.send(DisconnectedError()) + assert not connection.state.up + def test_reconnect_attempts_without_period_raises(): with pytest.raises(TypeError, match="sets reconnect_attempts but not"): class _NoPeriodConnection(_FlakyConnection): reconnect_attempts = 3 + + +class _SlowConnection(_FlakyConnection): + disconnect_after_timeouts = 2 + + @disconnect_on(OSError) + async def read(self, error: Exception | None = None) -> None: + if error is not None: + raise error + + +@pytest.mark.asyncio +async def test_timeouts_mark_down_after_disconnect_after_timeouts_in_a_row(): + connection = _SlowConnection() + connection.state = ConnectionState() + + # OSError includes TimeoutError, but timeouts are counted instead + with pytest.raises(TimeoutError): + await connection.read(TimeoutError()) + assert connection.state.up + + await connection.read() # Starts the count again + for _ in range(2): + with pytest.raises(TimeoutError): + await connection.read(TimeoutError()) + assert not connection.state.up + + +def test_disconnect_after_timeouts_must_be_at_least_one(): + with pytest.raises(TypeError, match="must be an integer of at least 1"): + + class _NeverConnection(_FlakyConnection): + disconnect_after_timeouts = 0 diff --git a/tests/test_launch.py b/tests/test_launch.py index 780ca8f3..a1794fd7 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -381,6 +381,7 @@ def test_reconnect_setting_defaults_come_from_the_connection_class(): fields = _connection_settings_model(IPConnection).model_fields assert fields["reconnect_attempts"].default == IPConnection.reconnect_attempts assert fields["reconnect_period"].default == IPConnection.reconnect_period + assert fields["disconnect_after_timeouts"].default == 1 def test_reconnect_attempts_without_period_rejected_by_settings_and_schema():