diff --git a/docs/examples/create_device_module/create_device_module.py b/docs/examples/create_device_module/create_device_module.py index 0896da8..9ec3609 100644 --- a/docs/examples/create_device_module/create_device_module.py +++ b/docs/examples/create_device_module/create_device_module.py @@ -35,7 +35,7 @@ # # behavior = schema.create_device_module(yml_text, converters={"DataConverter": DataConverter()}) # -# An unresolved custom type raises `UnknownConverterError`. Pass `strict=False` to +# An unresolved custom type raises `UnknownConverterError`. Pass `require_converters=False` to # decode it natively instead. A register marked `private` in the schema is emitted # with an underscore-prefixed name. For the parsed schema model rather than a module, # `parse_device_schema(yml_text)` returns that directly. diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index accb8b4..6dd4c62 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -25,24 +25,27 @@ Reading is based on a [device module](../harp-device) that describes how to deco from harp import data reader = data.create_dataset_reader("session.harp") -df = reader.read(AnalogData) # by register class -df = reader.read(44) # by address -everything = reader.read_all() # {register_name: DataFrame} +behavior = reader.device_module +df = reader.read(behavior.AnalogData) # by register class +df = reader.read(44) # by address +everything = reader.read_all() # {register_name: DataFrame} ``` Given a device module already in hand, either a pre-generated package or one built with `create_device_module`, pass it to `DatasetReader` directly: ```python -from pathlib import Path - from harp import data -from harp.device import schema +from harp.device import behavior -behavior = schema.create_device_module((Path("session.harp") / "device.yml").read_bytes()) reader = data.DatasetReader(behavior, "session.harp") +df = reader.read(behavior.AnalogData) ``` -Timestamps are auto-detected per register and placed on the DataFrame index named `"Time"`: float seconds by default, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout, or `name=` to override the file prefix. +Timestamps are auto-detected per register and placed on the DataFrame index named `"Time"`: float seconds by default, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. + +The `` prefix comes from the `DEVICE_NAME` declared by the device module. Pass `name=` to override it, or to supply one when the module declares an empty name. + +When the folder carries a `device.yml` and the module declares an identity, their `whoAmI` values are checked against each other. Reusing a module across sessions and reaching the wrong folder then fails on construction rather than decoding the files against the wrong register map. Pass `validate=False` to turn off every check the reader performs, so a folder whose `device.yml` is damaged can be read with a module obtained elsewhere. ## Read a single register file diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index bb8a49b..06f013c 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -3,15 +3,22 @@ from datetime import datetime from os import PathLike from pathlib import Path -from typing import Any +from typing import Any, Generic, TypeVar import pandas as pd -from harp.device.schema import DeviceModuleLike, create_device_module +from harp.device.schema import ( + DeviceModule, + DeviceModuleLike, + create_device_module, + parse_device_schema, +) from harp.protocol import RegisterBase from harp.protocol._constants import _TIMESTAMP_FLAG from ._reader import parse_to_dataframe +M = TypeVar("M", bound=DeviceModuleLike) + RegisterKey = type[RegisterBase[Any]] | int FileNameResolver = Callable[[Path, str], Mapping[int, list[Path]]] @@ -31,7 +38,7 @@ def default_file_resolver(root: Path, name: str) -> dict[int, list[Path]]: return files -class DatasetReader: +class DatasetReader(Generic[M]): """Reader over a de-multiplexed Harp dataset folder. Construct from a device module and a dataset folder, then read the @@ -43,9 +50,21 @@ class DatasetReader: everything = reader.read_all() # {register_name: DataFrame} ``device_module`` is a device module -- a generated device package, or one built from a - schema with :func:`~harp.device.schema.create_device_module`. Its ``REGISTER_MAP`` and - ``__name__`` are read on demand. ``name`` overrides the ```` file - prefix, which defaults to the module name. + schema with :func:`~harp.device.schema.create_device_module`. Its ``REGISTER_MAP`` is + read on demand. + + The files are matched by a ```` prefix, taken from the ``DEVICE_NAME`` + declared by the module. Pass ``name`` to override it, or to supply one when the module + declares an empty name. + + When the folder carries a ``device.yml`` and the module declares an identity, their + ``whoAmI`` values are checked against each other. A module paired with the wrong + folder then fails here rather than decoding the files against the wrong register + map. ``validate`` turns off every check the reader performs, so a folder whose + ``device.yml`` is damaged can be read with a module obtained elsewhere. + + The reader is typed on the module it was given, so registers stay reachable through + :attr:`device_module` at whatever precision that module offers. File resolution defaults to the Harp file format: ``_
.bin`` and, when a register was logged as several ``_
_.bin`` chunks, @@ -55,17 +74,21 @@ class DatasetReader: def __init__( self, - device_module: DeviceModuleLike, + device_module: M, root: str | PathLike[str], *, name: str | None = None, resolver: FileNameResolver = default_file_resolver, + validate: bool = True, ) -> None: self._device_module = device_module self._root = Path(root) self._name_override = name self._resolver = resolver - self._files = dict(self._resolver(self._root, self.name)) + self._name = self._resolve_name() + self._files = dict(self._resolver(self._root, self._name)) + if validate: + self._validate_whoami() @property def root(self) -> Path: @@ -73,14 +96,48 @@ def root(self) -> Path: return self._root @property - def device_module(self) -> DeviceModuleLike: - """The device module this reader parses against.""" + def device_module(self) -> M: + """The device module this reader parses against, as the type it was given. + + A generated package resolves each register to its own class; one built by + :func:`~harp.device.schema.create_device_module` resolves them collectively, + the same ceiling as reaching it directly. + """ return self._device_module @property def name(self) -> str: """The ```` prefix used to match binary files.""" - return self._name_override or self._device_module.__name__ + return self._name + + def _validate_whoami(self) -> None: + expected = self._device_module.WHO_AM_I + if expected == 0: + return + path = self._root / DEVICE_SCHEMA_FILENAME + if not path.is_file(): + return + try: + actual = parse_device_schema(path.read_bytes()).whoAmI + except ValueError: + return + if actual is None or int(actual) == expected: + return + raise ValueError( + f"WhoAmI mismatch: {self._name} expects 0x{expected:04x} but the schema in " + f"{self._root} declares 0x{int(actual):04x}." + ) + + def _resolve_name(self) -> str: + if self._name_override is not None: + return self._name_override + declared = self._device_module.DEVICE_NAME + if declared: + return declared + raise ValueError( + f"The device module declares an empty DEVICE_NAME, so it cannot name the " + f"files under {self._root}. Pass name= with the file prefix to read." + ) @property def registers(self) -> Mapping[int, type[RegisterBase[Any]]]: @@ -192,8 +249,9 @@ def create_dataset_reader( name: str | None = None, resolver: FileNameResolver = default_file_resolver, converters: Mapping[str, Any] | None = None, - strict: bool = True, -) -> DatasetReader: + require_converters: bool = True, + validate: bool = True, +) -> DatasetReader[DeviceModule]: """Build a :class:`DatasetReader` for a dataset folder, device and all. Convenience wrapper that finds the device schema inside ``root`` (``device.yml`` @@ -204,10 +262,15 @@ def create_dataset_reader( df = reader.read(44) ``schema`` points at the schema file explicitly when it isn't ``root/device.yml``. - ``converters`` and ``strict`` are forwarded to :func:`~harp.device.schema.create_device_module` - for custom ``interfaceType`` decoding; ``name`` and ``resolver`` are forwarded to + ``converters`` and ``require_converters`` are forwarded to + :func:`~harp.device.schema.create_device_module` for custom ``interfaceType`` + decoding; ``name``, ``resolver`` and ``validate`` are forwarded to :class:`DatasetReader`. Use ``DatasetReader(device_module, root)`` directly given a device module already in hand, for example a pre-generated one. + + Note ``validate`` cannot rescue a damaged ``device.yml`` here, since the module is + built from that same file and fails before the reader exists. Reading such a folder + means supplying a module obtained elsewhere. """ root_path = Path(root) schema_path = Path(schema) if schema is not None else root_path / DEVICE_SCHEMA_FILENAME @@ -217,6 +280,6 @@ def create_dataset_reader( f"or build the device module yourself and use DatasetReader(device_module, root)." ) device_module = create_device_module( - schema_path.read_text(), converters=converters, strict=strict + schema_path.read_text(), converters=converters, require_converters=require_converters ) - return DatasetReader(device_module, root_path, name=name, resolver=resolver) + return DatasetReader(device_module, root_path, name=name, resolver=resolver, validate=validate) diff --git a/src/packages/harp-device/README.md b/src/packages/harp-device/README.md index 5ed1692..3afe707 100644 --- a/src/packages/harp-device/README.md +++ b/src/packages/harp-device/README.md @@ -59,7 +59,7 @@ reg = behavior.REGISTER_MAP[44] # or by address The module is not registered in `sys.modules`, so it has to be bound rather than imported. Names come from the schema at runtime, so they don't autocomplete and aren't statically checked. A generated package on disk gives both. -For a custom `interfaceType`, pass its converter via `converters=`, keyed by `{InterfaceType}Converter` or `{MemberName}Converter`. An unresolved custom type raises `UnknownConverterError`, or pass `strict=False` to decode it natively: +For a custom `interfaceType`, pass its converter via `converters=`, keyed by `{InterfaceType}Converter` or `{MemberName}Converter`. An unresolved custom type raises `UnknownConverterError`, or pass `require_converters=False` to decode it natively: ```python schema.create_device_module(yml_text, converters={"DataConverter": DataConverter()}) diff --git a/src/packages/harp-device/src/harp/device/client/_device.py b/src/packages/harp-device/src/harp/device/client/_device.py index 6734ddc..0bcbb9c 100644 --- a/src/packages/harp-device/src/harp/device/client/_device.py +++ b/src/packages/harp-device/src/harp/device/client/_device.py @@ -167,7 +167,7 @@ def _validate_whoami(self) -> None: actual = int(self.read(WhoAmI).parsed) if actual != expected: raise RuntimeError( - f"WhoAmI mismatch: {module.__name__} expects 0x{expected:04x} " + f"WhoAmI mismatch: {module.DEVICE_NAME} expects 0x{expected:04x} " f"but the device reported 0x{actual:04x}." ) diff --git a/src/packages/harp-device/src/harp/device/schema/_emit.py b/src/packages/harp-device/src/harp/device/schema/_emit.py index 9f79e4f..1dfb3eb 100644 --- a/src/packages/harp-device/src/harp/device/schema/_emit.py +++ b/src/packages/harp-device/src/harp/device/schema/_emit.py @@ -242,11 +242,11 @@ def __init__( self, device: Union[DeviceModel, Registers], converters: Optional[Mapping[str, ConverterValue]], - strict: bool, + require_converters: bool, ) -> None: self.device = device self.converters = dict(converters or {}) - self.strict = strict + self.require_converters = require_converters self.group_masks = device.groupMasks or {} self.bit_masks = device.bitMasks or {} self.enums = self._build_enums() @@ -328,12 +328,12 @@ def _extension(self, symbol: str, ctx: ConverterContext) -> Converter[Any]: value = self.converters.get(symbol) if value is not None: return _materialize(value, ctx) - if not self.strict: + if not self.require_converters: return IdentityConverter(ctx.element) raise UnknownConverterError( f"no converter {symbol!r} in converters=; pass " f"converters={{{symbol!r}: Converter>}} " - f"or strict=False to decode as the native type" + f"or require_converters=False to decode as the native type" ) # -- defaults --------------------------------------------------------- @@ -517,7 +517,7 @@ def create_registers( source: str | bytes | DeviceModel | Registers, *, converters: Optional[Mapping[str, ConverterValue]] = None, - strict: bool = True, + require_converters: bool = True, ) -> dict[str, type[RegisterBase[Any]]]: """Emit runtime register classes from a device schema. @@ -530,11 +530,11 @@ def create_registers( :class:`~harp.protocol.Converter` instance or a factory ``(ctx: ConverterContext) -> Converter`` that builds one from the DSL context. A custom type with no matching converter raises - ``UnknownConverterError`` when ``strict`` (the default); ``strict=False`` - decodes it as its native element type instead. A register whose DSL ``visibility`` - is ``private`` is emitted with an underscore-prefixed class (``_Reserved0``), as the - generator emits it. Note that the converter symbol for a payload field derives from - its *verbatim* yml key, not the renamed field. + ``UnknownConverterError``. Pass ``require_converters=False`` to decode it as its + native element type instead. A register whose DSL ``visibility`` is ``private`` is + emitted with an underscore-prefixed class (``_Reserved0``), as the generator emits + it. Note that the converter symbol for a payload field derives from its *verbatim* + yml key, not the renamed field. """ device = source if isinstance(source, Registers) else parse_device_schema(source) - return _Emitter(device, converters, strict).emit() + return _Emitter(device, converters, require_converters).emit() diff --git a/src/packages/harp-device/src/harp/device/schema/_module.py b/src/packages/harp-device/src/harp/device/schema/_module.py index 4c1f422..259136f 100644 --- a/src/packages/harp-device/src/harp/device/schema/_module.py +++ b/src/packages/harp-device/src/harp/device/schema/_module.py @@ -27,27 +27,34 @@ class DeviceModuleLike(Protocol): what identifies it is describing a device. Matching structurally accepts both it and :class:`DeviceModule`, and rejects the common register set, which carries registers but is not a device. + + ``DEVICE_NAME`` is required rather than optional, so a generated package always + states the name used for its recordings. A schema declaring none still + builds, since :class:`DeviceModule` declares the member and leaves it empty. """ - __name__: str - REGISTER_MAP: dict[int, type[RegisterBase[Any]]] + DEVICE_NAME: str WHO_AM_I: int + REGISTER_MAP: dict[int, type[RegisterBase[Any]]] class DeviceModule(types.ModuleType): """The type of the module returned by :func:`create_device_module`. The declarations of the schema are reached by name and typed ``Any``, since they - exist only at runtime. ``REGISTER_MAP``, ``WHO_AM_I`` and ``__all__`` are declared - here and carry their own types. + exist only at runtime. ``DEVICE_NAME``, ``REGISTER_MAP``, ``WHO_AM_I`` and + ``__all__`` are declared here and carry their own types. """ - REGISTER_MAP: dict[int, type[RegisterBase[Any]]] - """Address -> register class, the common Harp registers merged with those of the schema.""" + DEVICE_NAME: str + """The device name declared by the schema. Empty when absent.""" WHO_AM_I: int """The device identity declared by the schema. ``0`` when absent.""" + REGISTER_MAP: dict[int, type[RegisterBase[Any]]] + """Address -> register class, the common Harp registers merged with those of the schema.""" + __all__: list[str] """The declarations of the schema, beside ``REGISTER_MAP`` and ``WHO_AM_I``.""" @@ -57,7 +64,7 @@ def create_device_module( *, name: Optional[str] = None, converters: Optional[Mapping[str, ConverterValue]] = None, - strict: bool = True, + require_converters: bool = True, ) -> DeviceModule: """Emit a module of register classes from ``device.yml`` text. @@ -72,8 +79,12 @@ def create_device_module( * ``REGISTER_MAP``, the device address space, so the common registers are present here even though the module does not name them; * ``WHO_AM_I``, the identity declared by the schema (``0`` for an unregistered device); - * ``__name__``, the ``device`` name of the schema, or ``name`` when given - (``"Device"`` for a header-less register fragment). + * ``DEVICE_NAME``, the ``device`` name of the schema, or ``name`` when given, and + empty for a header-less register fragment. Recordings are written under this + name, so :class:`~harp.data.DatasetReader` matches files by it; + * ``__name__``, the same name, falling back to ``"Device"`` so the module is never + anonymous. This names the module rather than the device, and is not part of what + a device module promises. Because the names come from the schema at runtime they don't autocomplete, and each resolves as ``Any`` rather than its own type. A generated device package is @@ -89,9 +100,10 @@ def create_device_module( behavior.AnalogData """ device = parse_device_schema(text) - emitter = _Emitter(device, converters, strict) + emitter = _Emitter(device, converters, require_converters) registers = emitter.emit() - module_name = name or device.device or _DEFAULT_NAME + device_name = name or device.device or "" + module_name = device_name or _DEFAULT_NAME contents: dict[str, Any] = {**emitter.enums, **emitter.payloads, **registers} register_map = {cls.address: cls for cls in CORE_REGISTER_MAP.values()} @@ -103,8 +115,9 @@ def create_device_module( module = DeviceModule(module_name, f"Harp registers for {module_name}, from a schema.") vars(module).update( contents, + DEVICE_NAME=device_name, REGISTER_MAP=register_map, WHO_AM_I=int(device.whoAmI or 0), - __all__=[*sorted(contents), "REGISTER_MAP", "WHO_AM_I"], + __all__=[*sorted(contents), "DEVICE_NAME", "REGISTER_MAP", "WHO_AM_I"], ) return module diff --git a/tests/conformance.py b/tests/conformance.py index bea71e7..4fe587c 100644 --- a/tests/conformance.py +++ b/tests/conformance.py @@ -8,7 +8,7 @@ from typing import Any, ClassVar, assert_type import numpy as np -from harp.data import DatasetReader +from harp.data import DatasetReader, create_dataset_reader from harp.device.client import Device, ITransport from harp.device.core import OperationControl, OperationControlPayload, WhoAmI from harp.device.schema import DeviceModule, DeviceModuleLike, create_device_module @@ -85,17 +85,44 @@ def dataset_reader_accepts_either_module( reader.read(44) +def dataset_reader_keeps_module_type( + schema_built: DeviceModule, generated: DeviceModuleLike +) -> None: + """The reader is typed on the module it was given, not on the contract. + + Reading it back as ``DeviceModuleLike`` would leave only the three declarations of + the contract, so every register reached through the reader would fail to resolve. + """ + assert_type(DatasetReader(schema_built, "session.harp").device_module, DeviceModule) + assert_type(DatasetReader(generated, "session.harp").device_module, DeviceModuleLike) + + +def dataset_reader_registers_resolve_through_the_module() -> None: + """A register stays reachable through the reader, at the precision of its module. + + A schema-built module resolves collectively, as it does when reached directly, so + the ceiling here is the one :func:`create_device_module` documents. A generated + package carries its own declarations and resolves each to its own class. + """ + reader = create_dataset_reader("session.harp") + assert_type(reader, DatasetReader[DeviceModule]) + assert_type(reader.device_module.AnalogData, Any) + reader.read(reader.device_module.AnalogData) + + def open_serial_device_prefers_the_subclass_overload() -> None: """A Device subclass is matched as a subclass even when it looks like a module. type[D] is narrower than the structural module overload, so it has to come first: - a class carrying REGISTER_MAP and WHO_AM_I satisfies DeviceModuleLike too, and the - module overload would otherwise win and return Device[type[Hybrid]]. + a class carrying the members of DeviceModuleLike satisfies it too, and the module + overload would otherwise win and return Device[type[Hybrid]]. The class has to + carry every member for this to test the ordering rather than the match. """ class Hybrid(Device[None]): - REGISTER_MAP: ClassVar[dict[int, type[RegisterBase[Any]]]] = {} + DEVICE_NAME: ClassVar[str] = "Hybrid" WHO_AM_I: ClassVar[int] = 1216 + REGISTER_MAP: ClassVar[dict[int, type[RegisterBase[Any]]]] = {} device = open_serial_device(Hybrid, port="COM3") assert_type(device, Hybrid) diff --git a/tests/data/test_dataset.py b/tests/data/test_dataset.py index fe1d733..8b042b1 100644 --- a/tests/data/test_dataset.py +++ b/tests/data/test_dataset.py @@ -22,16 +22,16 @@ def _records(cls, n, seed): @pytest.fixture def emitted_module(device_yml): - # strict=False: the test device.yml uses a custom DataConverter we don't inject + # require_converters=False: the test device.yml uses a custom DataConverter we don't inject # here; native decoding is enough to exercise file resolution and parsing. - return create_device_module(device_yml, strict=False) + return create_device_module(device_yml, require_converters=False) @pytest.fixture def dataset(emitted_module, tmp_path): """A dataset folder with three app registers; the first is timestamped.""" mod = emitted_module - name = mod.__name__ + name = mod.DEVICE_NAME addresses = [a for a in sorted(mod.REGISTER_MAP) if a >= 32][:3] specs = {} for i, address in enumerate(addresses): @@ -71,7 +71,7 @@ def test_reads_common_registers_not_named_by_module(emitted_module, tmp_path): for cls in (WhoAmI, TimestampSeconds): records = _records(cls, 4, seed=cls.address) buf = bytes(cls.format_bulk(records)) - (tmp_path / f"{mod.__name__}_{cls.address}.bin").write_bytes(buf) + (tmp_path / f"{mod.DEVICE_NAME}_{cls.address}.bin").write_bytes(buf) reader = DatasetReader(mod, tmp_path) # By address, and by the class imported from harp.device, and in read_all. @@ -120,27 +120,32 @@ def test_read_all_keyed_by_register_name(dataset): def test_suffix_chunks_are_concatenated(emitted_module, tmp_path): + # Chunk suffixes in this test are ISO 8601 UTC timestamps in basic format, so + # filename order is chronological order. Written newest first to test the sorting. mod = emitted_module - name = mod.__name__ + name = mod.DEVICE_NAME address = next(a for a in sorted(mod.REGISTER_MAP) if a >= 32) cls = mod.REGISTER_MAP[address] - chunk0 = bytes(cls.format_bulk(_records(cls, 3, seed=1))) - chunk1 = bytes(cls.format_bulk(_records(cls, 2, seed=2))) - (tmp_path / f"{name}_{address}_0.bin").write_bytes(chunk0) - (tmp_path / f"{name}_{address}_1.bin").write_bytes(chunk1) + chunks = { + "20260816T090000Z": bytes(cls.format_bulk(_records(cls, 3, seed=1))), + "20260816T100000Z": bytes(cls.format_bulk(_records(cls, 2, seed=2))), + } + for suffix in reversed(list(chunks)): + (tmp_path / f"{name}_{address}_{suffix}.bin").write_bytes(chunks[suffix]) reader = DatasetReader(mod, tmp_path) - combined = parse_to_dataframe(cls, chunk0 + chunk1, timestamp=False) + combined = parse_to_dataframe(cls, b"".join(chunks.values()), timestamp=False) assert reader.read(cls).reset_index(drop=True).equals(combined) # A specific chunk can still be selected by suffix. - only0 = parse_to_dataframe(cls, chunk0, timestamp=False) - assert reader.read(cls, suffix="0").equals(only0) + earliest = parse_to_dataframe(cls, chunks["20260816T090000Z"], timestamp=False) + assert reader.read(cls, suffix="20260816T090000Z").equals(earliest) def test_non_module_raises_on_register_access(dataset): - _mod, _name, root, _specs = dataset + _mod, name, root, _specs = dataset # Registers are derived lazily; anything without a REGISTER_MAP fails on access. - reader = DatasetReader(object, root) + # name and validate keep construction from reading the module at all. + reader = DatasetReader(object, root, name=name, validate=False) with pytest.raises(AttributeError, match="REGISTER_MAP"): _ = reader.registers @@ -152,6 +157,33 @@ def test_explicit_name_overrides(dataset): assert reader.name == name +def test_name_from_device_name_not_module_name(dataset): + # The prefix follows DEVICE_NAME rather than __name__, so rebinding the module + # does not change which files are read. + mod, name, root, _specs = dataset + mod.__name__ = "not_the_device_name" + assert DatasetReader(mod, root).name == name + + +def test_empty_dataset_reads_empty(emitted_module, tmp_path): + # A session that logged nothing is a dataset with no data, not a failure. + reader = DatasetReader(emitted_module, tmp_path) + assert reader.name == emitted_module.DEVICE_NAME + assert reader.files == {} + assert reader.read_all() == {} + + +def test_nameless_module_raises_on_construction(dataset): + # A header-less schema declares no device, so its module names nothing and file + # names are not consulted. This is the reader failing to be set up, not empty data. + _mod, name, root, _specs = dataset + nameless = create_device_module("registers:\n Foo: {address: 40, type: U16, access: Read}\n") + assert nameless.DEVICE_NAME == "" + with pytest.raises(ValueError, match="Pass name="): + DatasetReader(nameless, root) + assert DatasetReader(nameless, root, name=name).name == name + + def test_missing_register_file_raises(dataset): mod, _name, root, _specs = dataset reader = DatasetReader(mod, root) @@ -202,7 +234,7 @@ def test_files_property_lists_discovered_bins(dataset): def test_read_all_registers_of_mock_device(emitted_module, tmp_path): """Write one .bin per register of the device.yml device, then read them all back.""" mod = emitted_module - name = mod.__name__ + name = mod.DEVICE_NAME expected = {} for address, cls in mod.REGISTER_MAP.items(): records = _records(cls, 4, seed=address) @@ -235,8 +267,9 @@ def test_reader_derives_name_and_registers_from_module(dataset): def test_create_dataset_reader_builds_module_from_device_yml(dataset, device_yml): mod, _name, root, specs = dataset (root / "device.yml").write_text(device_yml) - # strict=False mirrors the emitted_module fixture (custom DataConverter not injected). - reader = create_dataset_reader(root, strict=False) + # require_converters=False mirrors the emitted_module fixture, which does not + # inject the custom DataConverter either. + reader = create_dataset_reader(root, require_converters=False) assert isinstance(reader, DatasetReader) # Reads match a reader built from an explicitly-generated module. reference = DatasetReader(mod, root) @@ -248,11 +281,98 @@ def test_create_dataset_reader_accepts_explicit_schema_path(dataset, device_yml, _mod, _name, root, specs = dataset schema_path = tmp_path / "elsewhere.yml" # not inside the dataset folder schema_path.write_text(device_yml) - reader = create_dataset_reader(root, schema=schema_path, strict=False) + reader = create_dataset_reader(root, schema=schema_path, require_converters=False) address = next(iter(specs)) assert not reader.read(address).empty +def _with_whoami(device_yml: str, who_am_i: int) -> str: + return f"whoAmI: {who_am_i}\n{device_yml}" + + +def test_whoami_mismatch_raises_on_construction(dataset, device_yml, tmp_path): + # A folder whose schema declares a different device is rejected. + _mod, name, root, specs = dataset + (root / "device.yml").write_text(_with_whoami(device_yml, 1216)) + other = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + + elsewhere = tmp_path / "other_session" + elsewhere.mkdir() + for address in specs: + (elsewhere / f"{name}_{address}.bin").write_bytes(b"") + (elsewhere / "device.yml").write_text(_with_whoami(device_yml, 1234)) + + with pytest.raises(ValueError, match="WhoAmI mismatch"): + DatasetReader(other, elsewhere) + + +def test_matching_whoami_does_not_block_read(dataset, device_yml): + _mod, _name, root, specs = dataset + (root / "device.yml").write_text(_with_whoami(device_yml, 1216)) + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + assert not DatasetReader(mod, root).read(next(iter(specs))).empty + + +def test_unregistered_module_skips_whoami_check(dataset, device_yml): + # WHO_AM_I of 0 marks an unregistered device, so there is nothing to check against. + mod, _name, root, _specs = dataset + (root / "device.yml").write_text(_with_whoami(device_yml, 1234)) + assert mod.WHO_AM_I == 0 + assert isinstance(DatasetReader(mod, root), DatasetReader) + + +def test_folder_without_schema_skips_whoami_check(dataset, device_yml): + _mod, _name, root, _specs = dataset # no device.yml written into the folder + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + assert isinstance(DatasetReader(mod, root), DatasetReader) + + +def test_schema_without_whoami_skips_check(dataset, device_yml): + _mod, _name, root, _specs = dataset + (root / "device.yml").write_text(device_yml) # the fixture declares no whoAmI + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + assert isinstance(DatasetReader(mod, root), DatasetReader) + + +def test_unmodellable_schema_skips_check(dataset, device_yml): + # Well-formed YAML that pyharp cannot describe, such as a newer or older revision, + # must not stop a module that works from decoding the binaries beside it. + _mod, _name, root, specs = dataset + (root / "device.yml").write_text("registers: [this is not a register map]\n") + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + assert not DatasetReader(mod, root).read(next(iter(specs))).empty + + +def test_validate_false_reads_corrupt_schema(dataset, device_yml): + # The escape hatch: a damaged sidecar must not make a folder unreadable when the + # module decoding it came from elsewhere. + _mod, _name, root, specs = dataset + (root / "device.yml").write_text("registers: {unbalanced\n") + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + reader = DatasetReader(mod, root, validate=False) + assert not reader.read(next(iter(specs))).empty + + +def test_validate_false_skips_mismatch(dataset, device_yml, tmp_path): + _mod, name, root, specs = dataset + (root / "device.yml").write_text(_with_whoami(device_yml, 1234)) + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + assert DatasetReader(mod, root, validate=False).name == name + + +def test_corrupt_schema_is_not_skipped(dataset, device_yml): + # A schema that is not well-formed is a broken dataset rather than one this + # version cannot describe, so it surfaces instead of being skipped. + _mod, _name, root, _specs = dataset + (root / "device.yml").write_text("registers: {unbalanced\n") + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + with pytest.raises(Exception) as excinfo: + DatasetReader(mod, root) + # Pinning the property rather than the parser: anything deriving from ValueError + # would have been swallowed by the skip, so this must not. + assert not isinstance(excinfo.value, ValueError) + + def test_create_dataset_reader_missing_schema_raises(dataset): _mod, _name, root, _specs = dataset # no device.yml written into the folder with pytest.raises(FileNotFoundError, match="device.yml"): diff --git a/tests/device/expected_device.py b/tests/device/expected_device.py index db3371c..0f027f3 100644 --- a/tests/device/expected_device.py +++ b/tests/device/expected_device.py @@ -31,6 +31,7 @@ __all__ = [ + "DEVICE_NAME", "WHO_AM_I", "PortDigitalIOS", "PwmPort", @@ -64,6 +65,7 @@ "REGISTER_MAP", ] +DEVICE_NAME: str = "Tests" WHO_AM_I: int = 0 diff --git a/tests/device/test_create_device_module.py b/tests/device/test_create_device_module.py index 5fa8653..ae8c39f 100644 --- a/tests/device/test_create_device_module.py +++ b/tests/device/test_create_device_module.py @@ -12,7 +12,7 @@ from .converters import DataConverter CONVERTERS = {"DataConverter": DataConverter()} -MODULE_CONSTANTS = {"REGISTER_MAP", "WHO_AM_I"} +MODULE_CONSTANTS = {"DEVICE_NAME", "REGISTER_MAP", "WHO_AM_I"} @pytest.fixture @@ -31,6 +31,11 @@ def test_returns_device_module(test_module): assert isinstance(test_module, DeviceModule) +def test_device_name_from_schema(test_module): + # Recordings are written under this name, so a reader can match files by it. + assert test_module.DEVICE_NAME == "Tests" + + def test_whoami_defaults_to_zero_when_absent(test_module): # device.yml (application-device metadata) omits whoAmI. assert test_module.WHO_AM_I == 0 @@ -121,13 +126,16 @@ def test_device_register_overrides_core_on_clash(): assert mod.Shadow.address == 0 -def test_headerless_fragment_builds_default_module(): - # A register-only fragment is a valid (nameless) device; name falls back to "Device". +def test_headerless_fragment_builds_nameless_device(): + # A register-only fragment declares no device, so it is nameless and unregistered. + # The empty DEVICE_NAME is what sends a reader to the folder for a prefix, while + # "Device" only keeps the module itself from being anonymous. mod = create_device_module("registers:\n Foo: {address: 40, type: U16, access: Read}\n") - assert mod.__name__ == "Device" + assert mod.DEVICE_NAME == "" assert mod.WHO_AM_I == 0 assert mod.REGISTER_MAP[40].__name__ == "Foo" assert mod.Foo.address == 40 + assert mod.__name__ == "Device" def test_all_covers_declarations_and_module_constants(test_module): diff --git a/tests/device/test_device.py b/tests/device/test_device.py index 570be9c..7d782de 100644 --- a/tests/device/test_device.py +++ b/tests/device/test_device.py @@ -16,6 +16,7 @@ def read(self) -> bytes: def _module(name: str, **attrs: object) -> types.ModuleType: mod = types.ModuleType(name) + mod.DEVICE_NAME = name for key, value in attrs.items(): setattr(mod, key, value) return mod diff --git a/tests/device/test_emit.py b/tests/device/test_emit.py index 1a677fb..eebab9a 100644 --- a/tests/device/test_emit.py +++ b/tests/device/test_emit.py @@ -302,7 +302,7 @@ def test_unknown_converter_raises(device_yml): def test_non_strict_falls_back_to_native(device_yml): - regs = create_registers(device_yml, strict=False) + regs = create_registers(device_yml, require_converters=False) # Data decodes as the raw native element (u8[2]) rather than the custom int. reg = regs["CustomMemberConverter"] assert reg.payload_class.payload_dtype.itemsize == 3