Align runtime modules with generated device packages

docs/examples/create_device_module/create_device_module.py

@@ -36,6 +36,6 @@
 #   behavior = schema.create_device_module(yml_text, converters={"DataConverter": DataConverter()})
 #
 # An unresolved custom type raises `UnknownConverterError`. Pass `strict=False` to
-# decode it natively instead. `exclude_private=True`, the default, drops registers
-# marked `private` in the schema. For the parsed schema model rather than a module,
+# decode it natively instead. A register marked `private` in the schema is emitted
+# with an underscore-prefixed name. For the parsed schema model rather than a module,
 # `parse_device_schema(yml_text)` returns that directly.

src/packages/harp-device/README.md

@@ -27,7 +27,7 @@ REGISTER_MAP = {**_CORE_REGISTER_MAP, 32: DigitalInputState, ...}
 
 This is the same structure `create_device_module` builds from a schema, so a device reads the same way whether it was generated ahead of time or compiled at runtime. A `WHO_AM_I` of `0` marks an unregistered device, used while a device is in development or outside the official registry, and identity checks are skipped for it.
 
-A device module names only the registers its schema declares, so `REGISTER_MAP` is the device address space while the module namespace is what the device adds to it. The common registers have a single definition, in `harp.device.core`, and are not a device, so the core register set carries no `WHO_AM_I`.
+A device module names only what its schema declares, the registers beside the enums and payload classes they are built from, so `REGISTER_MAP` is the device address space while the module namespace is what the device adds to it. The common registers and any core mask the schema reuses have a single definition, in `harp.device.core`, and are reached from there rather than through the device module. The core register set is not a device, so it carries no `WHO_AM_I`.
 
 Pass the module to `Device`, or to `open_serial_device`, to validate identity on open:
 
@@ -45,7 +45,7 @@ A new transport is just an object implementing the `ITransport` protocol, with `
 
 ## Generating registers from a `device.yml`
 
-Without a pre-generated device package, `create_device_module` builds the same structure at runtime from Harp `device.yml` text: register classes at module level, a `REGISTER_MAP` beside them, and the identity declared by the schema as `WHO_AM_I`. Identifiers match a generated package name for name: register, enum, and payload class names come from the yml verbatim, payload fields are `snake_case`, and enum members are `SCREAMING_SNAKE_CASE`.
+Without a pre-generated device package, `create_device_module` builds the same structure at runtime from Harp `device.yml` text: register, enum and payload classes at module level, a `REGISTER_MAP` beside them, and the identity declared by the schema as `WHO_AM_I`. Identifiers match a generated package name for name: register, enum, and payload class names come from the yml verbatim, payload fields are `snake_case`, and enum members are `SCREAMING_SNAKE_CASE`. A `maskType` the schema does not declare resolves against the core masks, and a register marked `private` is emitted with an underscore-prefixed name.
 
 ```python
 from pathlib import Path

src/packages/harp-device/src/harp/device/schema/_emit.py

@@ -41,9 +41,18 @@
 from harp.protocol._payload import _reserved_field_reason
 from harp.protocol import PayloadType as ProtoPayloadType
 
+from harp.device import core
+
 from ._model import DeviceModel, PayloadMember, PayloadType, Register, Registers, Visibility
 from ._naming import enum_member_name, field_name
 
+
+_CORE_MASKS: dict[str, Any] = {
+    name: declaration
+    for name, declaration in vars(core).items()
+    if name in core.__all__ and isinstance(declaration, type) and issubclass(declaration, enum.Enum)
+}
+
 _ELEMENT: dict[PayloadType, type[np.generic]] = {
     PayloadType.U8: np.uint8,
     PayloadType.S8: np.int8,
@@ -201,6 +210,10 @@ class UnknownConverterError(ValueError):
     """A custom ``interfaceType`` needs a converter not found in ``converters=``."""
 
 
+class UnknownMaskError(ValueError):
+    """A ``maskType`` names neither a mask the schema declares nor a core mask."""
+
+
 class NameCollisionError(ValueError):
     """Two schema identifiers collapse to one Python name, or one shadows a reserved name.
 
@@ -230,19 +243,20 @@ def __init__(
         device: Union[DeviceModel, Registers],
         converters: Optional[Mapping[str, ConverterValue]],
         strict: bool,
-        exclude_private: bool,
     ) -> None:
         self.device = device
         self.converters = dict(converters or {})
         self.strict = strict
-        self.exclude_private = exclude_private
         self.group_masks = device.groupMasks or {}
         self.bit_masks = device.bitMasks or {}
         self.enums = self._build_enums()
         # Payload classes are cached by name so registers sharing an ``interfaceType``
         # share one class, as the module-level payload list of the generator does.
         self.payloads: dict[str, type] = {}
 
+    def _find_mask(self, name: str) -> Any:
+        return self.enums.get(name) or _CORE_MASKS.get(name)
+
     # -- naming -----------------------------------------------------------
     def _rename(
         self,
@@ -329,12 +343,12 @@ def _default(self, member: PayloadMember, type_name: str, ctx: ConverterContext)
         if _default_value is None or (member.length or 0) > 1:
             return _NO_DEFAULT
         value = float(_default_value.root)
-        if type_name in self.group_masks:
-            e = self.enums[type_name]
-            for mv in self.group_masks[type_name].values.values():
-                if int(mv) == int(value):
-                    return e(int(value))
-            return int(value)
+        group_mask = self._find_mask(type_name)
+        if group_mask is not None and issubclass(group_mask, enum.IntEnum):
+            try:
+                return group_mask(int(value))
+            except ValueError:
+                return int(value)
         if member.converter is not None:
             return _NO_DEFAULT  # a custom converter owns its own decoding; no numeric default
         it = ctx.interface_type
@@ -367,10 +381,11 @@ def _build_field(self, key: str, member: PayloadMember, reg: Register) -> Any:
         default_kwarg = {} if default is _NO_DEFAULT else {"default": default}
 
         # A group mask is an enum sub-field descriptor, not a Field(converter).
-        if type_name in self.group_masks:
+        group_mask = self._find_mask(type_name)
+        if group_mask is not None and issubclass(group_mask, enum.IntEnum):
             full = (1 << (elem_size * 8)) - 1
             mask = member.mask if member.mask is not None else full
-            return GroupMask(enum=self.enums[type_name], mask=mask, offset=offset, **default_kwarg)
+            return GroupMask(enum=group_mask, mask=mask, offset=offset, **default_kwarg)
 
         field_kwargs: dict[str, Any] = {"offset": offset, **default_kwarg}
         if member.mask is not None:
@@ -413,15 +428,22 @@ def _new_payload(self, class_name: str, owner: str, reg: Register) -> type:
         # anonymous single-value payload
         mt = reg.maskType.root if reg.maskType else None
         it = reg.interfaceType.root if reg.interfaceType else None
-        if mt in self.group_masks:
+        mask = self._find_mask(mt) if mt else None
+        if mask is not None and issubclass(mask, enum.IntFlag):
+            descriptor: Any = BitMask(enum=mask)
+        elif mask is not None:
             full = (1 << (elem_size * 8)) - 1
-            descriptor: Any = GroupMask(enum=self.enums[mt], mask=full)
-        elif mt in self.bit_masks:
-            descriptor = BitMask(enum=self.enums[mt])
-        else:
-            assert it is not None, (
-                f"{owner}: register needs a payloadSpec, maskType, or interfaceType"
+            descriptor = GroupMask(enum=mask, mask=full)
+        elif mt is not None:
+            raise UnknownMaskError(
+                f"{owner}: maskType {mt!r} is neither declared by the schema nor a core "
+                f"mask; declare it or use one of {sorted(_CORE_MASKS)}"
             )
+        elif it is None:
+            raise ValueError(
+                f"{owner}: register declares no payloadSpec, maskType, or interfaceType"
+            )
+        else:
             ctx = ConverterContext(
                 name="__value__",
                 interface_type=it,
@@ -470,8 +492,6 @@ def _build_register(self, name: str, class_name: str, reg: Register) -> type[Reg
     def emit(self) -> dict[str, type[RegisterBase[Any]]]:
         emitted: dict[str, type[RegisterBase[Any]]] = {}
         for name, reg in self.device.registers.items():
-            if self.exclude_private and reg.visibility is Visibility.private:
-                continue
             class_name = self._class_name(name, reg)
             emitted[class_name] = self._build_register(name, class_name, reg)
         return emitted
@@ -498,7 +518,6 @@ def create_registers(
     *,
     converters: Optional[Mapping[str, ConverterValue]] = None,
     strict: bool = True,
-    exclude_private: bool = False,
 ) -> dict[str, type[RegisterBase[Any]]]:
     """Emit runtime register classes from a device schema.
 
@@ -512,10 +531,10 @@ def create_registers(
     ``(ctx: ConverterContext) -> Converter`` that builds one from the DSL
     context. A custom type with no matching converter raises
     ``UnknownConverterError`` when ``strict`` (the default); ``strict=False``
-    decodes it as its native element type instead. ``exclude_private=True`` drops
-    registers whose DSL ``visibility`` is ``private``; when kept, a private register
-    class is underscore-prefixed (``_Reserved0``). Note that the converter symbol for a
-    payload field derives from its *verbatim* yml key, not the renamed field.
+    decodes it as its native element type instead. A register whose DSL ``visibility``
+    is ``private`` is emitted with an underscore-prefixed class (``_Reserved0``), as the
+    generator emits it. Note that the converter symbol for a payload field derives from
+    its *verbatim* yml key, not the renamed field.
     """
     device = source if isinstance(source, Registers) else parse_device_schema(source)
-    return _Emitter(device, converters, strict, exclude_private).emit()
+    return _Emitter(device, converters, strict).emit()

src/packages/harp-device/src/harp/device/schema/_model.py

@@ -7,7 +7,7 @@
 from enum import Enum
 from typing import Annotated, Dict, List, Optional, Union
 
-from pydantic import BaseModel, ConfigDict, Field, RootModel
+from pydantic import BaseModel, ConfigDict, Field, RootModel, model_validator
 
 
 class PayloadType(str, Enum):
@@ -225,6 +225,25 @@ class Registers(BaseModel):
         ),
     )
 
+    @model_validator(mode="after")
+    def _names_are_distinct(self) -> "Registers":
+        """Every generator target renders registers and masks into one namespace."""
+        declared = (
+            ("register", self.registers),
+            ("bit mask", self.bitMasks or {}),
+            ("group mask", self.groupMasks or {}),
+        )
+        seen: Dict[str, str] = {}
+        for kind, names in declared:
+            for name in names:
+                if name in seen:
+                    raise ValueError(
+                        f"{name!r} is declared as both a {seen[name]} and a {kind}; "
+                        f"rename one of them"
+                    )
+                seen[name] = kind
+        return self
+
 
 class DeviceModel(Registers):
     """A device schema: a `Registers` collection plus optional device identity.

src/packages/harp-device/src/harp/device/schema/_module.py

@@ -13,7 +13,7 @@
 from harp.protocol import RegisterBase
 
 from harp.device.core import REGISTER_MAP as CORE_REGISTER_MAP
-from ._emit import ConverterValue, create_registers, parse_device_schema
+from ._emit import ConverterValue, _Emitter, parse_device_schema
 
 _DEFAULT_NAME = "Device"
 """Module name used when the schema carries no ``device`` header."""
@@ -35,13 +35,11 @@ class DeviceModuleLike(Protocol):
 
 
 class DeviceModule(types.ModuleType):
-    """The module :func:`create_device_module` returns, describing what a device module holds.
+    """The type of the module returned by :func:`create_device_module`.
 
-    Declaring the members is what lets a linter resolve them. Register names come
-    from the schema, so they can only be described collectively, through
-    :meth:`__getattr__`; ``REGISTER_MAP`` and ``WHO_AM_I`` are named and keep their
-    own types. A statically generated device package is a plain module and needs
-    none of this, since its registers are written out.
+    The declarations of the schema are reached by name and typed ``Any``, since they
+    exist only at runtime. ``REGISTER_MAP``, ``WHO_AM_I`` and ``__all__`` are declared
+    here and carry their own types.
     """
 
     REGISTER_MAP: dict[int, type[RegisterBase[Any]]]
@@ -50,8 +48,8 @@ class DeviceModule(types.ModuleType):
     WHO_AM_I: int
     """The device identity declared by the schema. ``0`` when absent."""
 
-    def __getattr__(self, name: str) -> type[RegisterBase[Any]]:
-        raise AttributeError(f"module {self.__name__!r} has no register named {name!r}")
+    __all__: list[str]
+    """The declarations of the schema, beside ``REGISTER_MAP`` and ``WHO_AM_I``."""
 
 
 def create_device_module(
@@ -60,13 +58,16 @@ def create_device_module(
     name: Optional[str] = None,
     converters: Optional[Mapping[str, ConverterValue]] = None,
     strict: bool = True,
-    exclude_private: bool = True,
 ) -> DeviceModule:
     """Emit a module of register classes from ``device.yml`` text.
 
-    The module names the registers the schema declares, so ``behavior.AnalogData``
-    resolves while a common register such as ``WhoAmI`` is imported from
-    :mod:`harp.device`, keeping one definition of each. Beside them it holds:
+    The module names what the schema declares, its registers beside the enums and
+    payload classes they are built from, so ``behavior.AnalogData``,
+    ``behavior.AnalogDataPayload`` and ``behavior.EncoderModeMask`` all resolve while a
+    common register such as ``WhoAmI`` is imported from :mod:`harp.device.core`, keeping
+    one definition of each. This is the same set a generated device package holds. A
+    name describing two declarations is rejected rather than shadowed. Beside them
+    it holds:
 
     * ``REGISTER_MAP``, the device address space, so the common registers are
       present here even though the module does not name them;
@@ -75,10 +76,9 @@ def create_device_module(
       (``"Device"`` for a header-less register fragment).
 
     Because the names come from the schema at runtime they don't autocomplete, and
-    each resolves as ``type[RegisterBase[Any]]`` rather than its own register type;
-    a generated device package is a real module on disk and gives both. On an
-    address clash the device register replaces the common one in ``REGISTER_MAP``.
-    ``exclude_private=True`` drops registers whose DSL ``visibility`` is ``private``.
+    each resolves as ``Any`` rather than its own type. A generated device package is
+    a real module on disk and gives both. On an address clash the device register
+    replaces the common one in ``REGISTER_MAP``.
 
     ``text`` is the schema itself rather than a path to it, matching
     :func:`parse_device_schema`, so read the file first. The module is **not**
@@ -89,17 +89,16 @@ def create_device_module(
         behavior.AnalogData
     """
     device = parse_device_schema(text)
-    registers = create_registers(
-        device, converters=converters, strict=strict, exclude_private=exclude_private
-    )
+    emitter = _Emitter(device, converters, strict)
+    registers = emitter.emit()
     module_name = name or device.device or _DEFAULT_NAME
 
-    contents: dict[str, type[RegisterBase[Any]]] = dict(registers)
+    contents: dict[str, Any] = {**emitter.enums, **emitter.payloads, **registers}
     register_map = {cls.address: cls for cls in CORE_REGISTER_MAP.values()}
     register_map.update({cls.address: cls for cls in registers.values()})
 
-    for register in registers.values():
-        register.__module__ = module_name
+    for declaration in contents.values():
+        declaration.__module__ = module_name
 
     module = DeviceModule(module_name, f"Harp registers for {module_name}, from a schema.")
     vars(module).update(

tests/conformance.py

@@ -20,7 +20,7 @@ def schema_built_registers(yml: str) -> None:
     """A module built from a schema types its registers collectively."""
     behavior = create_device_module(yml)
     assert_type(behavior, DeviceModule)
-    assert_type(behavior.AnalogData, type[RegisterBase[Any]])
+    assert_type(behavior.AnalogData, Any)
     assert_type(behavior.REGISTER_MAP, dict[int, type[RegisterBase[Any]]])
     assert_type(behavior.WHO_AM_I, int)