From f945bedbdf7dbf9a0bff2a3d93a0adfd9cfdda53 Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:22:41 -0500 Subject: [PATCH 1/2] Ship a microcontroller only what can run on one mip.install("pydevices") wrote 51 files to a board, about twenty of them desktop, browser or Android backends: windisplay, wasmdisplay, pgdisplay, sdldisplay, jndisplay, psdisplay, androidsdl, win_audio, pygame_audio, sdl2_audio, wasm_audio, web_audio, android_audio, win32, sdl2, threading, wasm, librt and two desktop helpers (pydevices#30). Minutes over WiFi and flash spent on modules that cannot execute. synchronize_mip_package.py now reads mip-split.toml from the source repository, which names those modules per package, and routes them into pydevices-desktop, which already require()s pydevices. The device package names its files one by one -- package("displaydev", files=(...)) -- so the install list is the manifest and nothing else decides it. Measured through mip's own tools/build.py, which is what the index serves: pydevices 51 -> 31 files pydevices-desktop 58 -> 58, the identical set Two guards, both shown failing: a name in mip-split.toml that no longer exists fails the sync ("names modules that no longer exist: gonedisplay"), and so does an MCU-side module that imports a host-only one at module scope ("busdisplay.py:21 imports the host-only module 'sdldisplay' at module scope"). Exit 1 in both cases, 0 on the real tree. A new backend nobody classifies ships to the device, so forgetting costs a board one file, never a host an import. --- docs/publishing-automation.md | 13 ++++ scripts/synchronize_mip_package.py | 108 ++++++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/docs/publishing-automation.md b/docs/publishing-automation.md index 47250f7..a26ecb4 100644 --- a/docs/publishing-automation.md +++ b/docs/publishing-automation.md @@ -95,6 +95,19 @@ differently, because their constraints differ: - The trade is real but bounded: a display-only board installs about 232 KiB more source than `displaydev` alone used to pull. Revisit if that starts to matter on a target. +- **A microcontroller gets only what can run on one.** `displaydev`, `audiodev` + and `multimer` each carry a full set of platform backends, and about twenty of + them are desktop, browser or Android code. The source repository's + `mip-split.toml` names those per package; they ship in `pydevices-desktop` + instead, and `pydevices` names its files one by one so the install list is the + manifest. An MCU install went from 51 files to 31, and `pydevices-desktop` + stayed at 58 — byte for byte the same set, because it already requires + `pydevices` (pydevices#30). + Two rules keep that file from rotting: a name in it that no longer exists + fails the sync, and so does an MCU-side module that imports a host-only one at + module scope. A *new* backend nobody classifies ships to the device, so the + cost of forgetting is one file too many on a board, never a missing import on + a host. - Every publishable entry under `utils/`, plus everything publishable in `board_configs/desktop/`, is bundled into `pydevices-desktop`. - `pydevices-desktop` depends on `pydevices`, so one install gets the complete diff --git a/scripts/synchronize_mip_package.py b/scripts/synchronize_mip_package.py index 81f3dbc..4c8d639 100755 --- a/scripts/synchronize_mip_package.py +++ b/scripts/synchronize_mip_package.py @@ -5,7 +5,9 @@ import argparse import json +import re import shutil +import tomllib from dataclasses import dataclass from pathlib import Path @@ -194,6 +196,77 @@ def render_pydevices_manifest(name: str, version: str, requirements: tuple[str, return "\n".join(lines) +#: Declares which modules of lib/ cannot run on a microcontroller. Its own +#: header says why and what the rules are; this reads it. +MIP_SPLIT_FILE = "mip-split.toml" + + +def read_host_only(source_root: Path) -> dict[str, frozenset[str]]: + """{package: host-only module stems}, checked against what is on disk.""" + split_path = source_root / MIP_SPLIT_FILE + if not split_path.exists(): + return {} + with split_path.open("rb") as handle: + declared = tomllib.load(handle) + + host_only: dict[str, frozenset[str]] = {} + for package, section in declared.items(): + package_dir = source_root / "lib" / package + if not package_dir.is_dir(): + raise SystemExit( + f"{MIP_SPLIT_FILE} names package {package!r}, which is not in lib/" + ) + present = {path.stem for path in package_dir.glob("*.py")} + names = frozenset(section.get("host-only", ())) + # A stale name is the way this file rots: the module gets renamed, the + # entry stops matching anything, and it silently ships to the MCU again. + missing = sorted(names - present) + if missing: + raise SystemExit( + f"{MIP_SPLIT_FILE} [{package}] names modules that no longer " + f"exist: {', '.join(missing)}" + ) + if "auto" in names or "__init__" in names: + raise SystemExit( + f"{MIP_SPLIT_FILE} [{package}] would move __init__ or auto to the " + f"host package, which would leave the MCU unable to import {package}" + ) + host_only[package] = names + return host_only + + +def check_no_host_imports(package_dir: Path, host_only: frozenset[str]) -> None: + """Refuse to ship an MCU module that imports a host-only one at module scope. + + Inside a function is fine and is how every ``auto`` module works; at module + scope it would make the package unimportable on a board, which is the one + way this split can break something. + """ + package = package_dir.name + pattern = re.compile( + r"^(?:from\s+(?:%s|\.)\s+import\s+(\w+)" + r"|from\s+%s\.(\w+)\s+import" + r"|import\s+%s\.(\w+))" % (package, package, package) + ) + for path in sorted(package_dir.glob("*.py")): + if path.stem in host_only: + continue + for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not line or line[:1].isspace(): + continue + match = pattern.match(line.strip()) + if match is None: + continue + name = next((group for group in match.groups() if group), None) + if name in host_only: + raise SystemExit( + f"{package}/{path.name}:{lineno} imports the host-only module " + f"{name!r} at module scope, so it would not import on a " + f"microcontroller. Move the import inside the function, or " + f"drop {name!r} from {MIP_SPLIT_FILE}." + ) + + def synchronize_pydevices(source_root: Path, mip_root: Path, version: str) -> None: destination_root = mip_root / "micropython" / "pydevices" if destination_root.parent != mip_root / "micropython": @@ -211,10 +284,32 @@ def synchronize_pydevices(source_root: Path, mip_root: Path, version: str) -> No package.mkdir() payloads: list[str] = [] names: list[str] = [] + # What a microcontroller cannot run does not go on one. Declared in the + # source repository's mip-split.toml and shipped by pydevices-desktop + # instead, which already require()s this package -- so a host installs one + # thing and gets what it always got (pydevices#30). + host_only = read_host_only(source_root) + host_components: list[tuple[Path, frozenset[str]]] = [] for source in sorted(filter(publishable, (source_root / "lib").iterdir()), key=lambda path: path.name): names.append(source.stem if source.is_file() else source.name) - copy_component(source, package / source.name) - payloads.append(f'module("{source.name}")' if source.is_file() else f'package("{source.name}")') + split = host_only.get(source.name, frozenset()) if source.is_dir() else frozenset() + if not split: + copy_component(source, package / source.name) + payloads.append(f'module("{source.name}")' if source.is_file() else f'package("{source.name}")') + continue + check_no_host_imports(source, split) + device_files = sorted( + path.name for path in source.glob("*.py") if path.stem not in split + ) + destination = package / source.name + destination.mkdir(parents=True) + for name in device_files: + shutil.copy2(source / name, destination / name) + # Named file by file rather than as a whole package, so the install + # list is the manifest and nothing else decides it. + listed = ", ".join(f'"{name}"' for name in device_files) + payloads.append(f'package("{source.name}", files=({listed},))') + host_components.append((source, split)) if len(names) != len(set(names)): raise SystemExit("lib/ contains colliding module and package names") @@ -226,6 +321,15 @@ def synchronize_pydevices(source_root: Path, mip_root: Path, version: str) -> No desktop = destination_root / "pydevices-desktop" desktop.mkdir() desktop_payloads: list[str] = [] + # The other half of the split: every backend the device package left out. + for source, split in host_components: + host_files = sorted(path.name for path in source.glob("*.py") if path.stem in split) + destination = desktop / source.name + destination.mkdir(parents=True) + for name in host_files: + shutil.copy2(source / name, destination / name) + listed = ", ".join(f'"{name}"' for name in host_files) + desktop_payloads.append(f'package("{source.name}", files=({listed},))') for source in sorted(filter(publishable, (source_root / "utils").iterdir()), key=lambda path: path.name): copy_component(source, desktop / source.name) desktop_payloads.append(f'module("{source.name}")' if source.is_file() else f'package("{source.name}")') From 6725ef35b797b7229467964aa5fa82b9ea82703d Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:52:27 -0500 Subject: [PATCH 2/2] ruff: sort the import block and use an f-string for the pattern The guard was re-proved after the rewrite: the planted module-scope import still fails the sync with exit 1, the real tree still exits 0, and the device package is still 31 files. --- scripts/synchronize_mip_package.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/synchronize_mip_package.py b/scripts/synchronize_mip_package.py index 4c8d639..f7ba928 100755 --- a/scripts/synchronize_mip_package.py +++ b/scripts/synchronize_mip_package.py @@ -7,10 +7,10 @@ import json import re import shutil -import tomllib from dataclasses import dataclass from pathlib import Path +import tomllib from pydevices_package_metadata import PYDEVICES_DESCRIPTIONS @@ -244,9 +244,9 @@ def check_no_host_imports(package_dir: Path, host_only: frozenset[str]) -> None: """ package = package_dir.name pattern = re.compile( - r"^(?:from\s+(?:%s|\.)\s+import\s+(\w+)" - r"|from\s+%s\.(\w+)\s+import" - r"|import\s+%s\.(\w+))" % (package, package, package) + rf"^(?:from\s+(?:{package}|\.)\s+import\s+(\w+)" + rf"|from\s+{package}\.(\w+)\s+import" + rf"|import\s+{package}\.(\w+))" ) for path in sorted(package_dir.glob("*.py")): if path.stem in host_only: