From d8f7e7d2f7906fb8c49a82114aff9fda7ccf8b7c Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Tue, 10 Nov 2026 16:20:49 -0800 Subject: [PATCH 1/9] feat(learning): add local decision runtime Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../0002-decision-model-and-rl-environment.md | 3 + packages/learning/README.md | 2 + .../environments/brand-check/README.md | 102 +++ .../environments/brand-check/__init__.py | 15 +- .../brand-check/examples/runtime_demo.py | 156 ++++ .../environments/brand-check/pyproject.toml | 1 + .../environments/brand-check/runtime.py | 808 ++++++++++++++++++ .../brand-check/runtime_server.py | 190 ++++ .../brand-check/server/runtime.Dockerfile | 17 + .../brand-check/tests/test_runtime.py | 280 ++++++ 10 files changed, 1573 insertions(+), 1 deletion(-) create mode 100644 packages/learning/environments/brand-check/examples/runtime_demo.py create mode 100644 packages/learning/environments/brand-check/runtime.py create mode 100644 packages/learning/environments/brand-check/runtime_server.py create mode 100644 packages/learning/environments/brand-check/server/runtime.Dockerfile create mode 100644 packages/learning/environments/brand-check/tests/test_runtime.py diff --git a/docs/adr/0002-decision-model-and-rl-environment.md b/docs/adr/0002-decision-model-and-rl-environment.md index e4365c63d..1449ac97e 100644 --- a/docs/adr/0002-decision-model-and-rl-environment.md +++ b/docs/adr/0002-decision-model-and-rl-environment.md @@ -124,9 +124,12 @@ The list leaves out models above 10 billion parameters and models built on a bas A trained model replaces the base model only when a repository administrator approves it. The evaluation report gives them two numbers to compare against the base model on the test set: the mean reward, which should be higher, and the calibration error, which should not be worse. This version writes the report, but Hub does not record the decision or serve a model yet. +The customer-operated reference runtime records that decision outside Hub. It accepts a taskset, tracks a job bounded by update and wall-time limits, writes the adapter and evaluation report as an immutable candidate, and serves only the candidate an operator explicitly promotes. Its HTTP contracts separate jobs, candidates, promotion, and evaluation from the local filesystem and trainer, so a customer can later implement the same lifecycle on its cloud platform. A deterministic fixture backend runs the complete lifecycle without a GPU or external model download; the d1 backend invokes the environment, trainer, scoring code, and evaluator described above. Replica does not run or receive data from either backend. + ## Consequences - Teams can train on their own review outcomes without sending them anywhere they did not choose. +- Teams can practice and test the complete lifecycle with a deterministic backend before spending GPU budget. Its report proves the orchestration and scoring path, not real-model quality. - A taskset copies screenshots and learning text. It lives under `.replica/` in a checkout, or outside any checkout. When someone withdraws training consent, `replica-learning` lists the export the taskset came from, and the owner must delete the taskset too. - The Builds file holds screenshots of every published Build of each reviewed prototype. The operator should delete it after writing the taskset. - A "follows" label rests on the reviewer's check of the fix, which may have covered one width and theme. The other screenshots of that page count as fixed unless a later thread reports the same theme there. diff --git a/packages/learning/README.md b/packages/learning/README.md index e38e9c73d..56b335466 100644 --- a/packages/learning/README.md +++ b/packages/learning/README.md @@ -216,6 +216,8 @@ A task shows every screenshot of its page, desktop before narrow and light befor Hub names each screenshot by the address the build action was asked to capture. The build action follows redirects and Controls URL defaults without recording where the page ended up, so a comment made at the address a page redirects to finds no screenshot. +The [local decision-model runtime](environments/brand-check/README.md#run-the-local-training-and-inference-runtime) accepts that taskset, tracks a bounded fixture or d1 job, stores an immutable candidate and proper-scoring evaluation report, requires an explicit promotion, and serves the promoted visual critic through the same request shape as `decision ask`. The deterministic fixture backend demonstrates the complete lifecycle without a GPU or model download. Customers run the runtime and keep its tasksets, adapters, reports, and credentials; Replica does not host a training or inference service. + The output directory must be new and, inside a checkout, under `.replica/`. It holds: | File | Contents | diff --git a/packages/learning/environments/brand-check/README.md b/packages/learning/environments/brand-check/README.md index b6d3ee47e..80145a591 100644 --- a/packages/learning/environments/brand-check/README.md +++ b/packages/learning/environments/brand-check/README.md @@ -159,6 +159,108 @@ The command prints a JSON report: A missing or invalid answer counts as wrong for accuracy and as `invalid`, and it is left out of the other scores. A metric without data is `null`. +## Run the local training and inference runtime + +The local runtime turns the environment, trainer, and evaluator into one customer-operated lifecycle. It keeps mutable job records under `jobs/`, writes each completed candidate once under `candidates/`, and serves only the candidate an operator explicitly promotes. Replica does not receive the taskset, adapter, reports, token, or requests. + +Run the deterministic lifecycle first. It uses the real taskset loader, proper-scoring evaluation, candidate store, promotion gate, and inference contract, but its fixture readout needs no GPU or model download: + +```sh +cd packages/learning/environments/brand-check +python3 examples/runtime_demo.py --out /tmp/replica-runtime-demo +``` + +The output shows the bounded job, immutable candidate, baseline and candidate test reports, promotion, and one structured Build evaluation. `candidate.evaluation.evidence.meanRewardDelta` states the measured gain over the 50% baseline, and `calibrationErrorDelta` states whether calibration improved. The demo leaves its state in `/tmp/replica-runtime-demo/runtime` so an operator can inspect it. + +For a taskset created by `replica-learning decision taskset`, make a runtime token, install the package, and start the service: + +```sh +(umask 077 && python3 -c 'import secrets; print(secrets.token_urlsafe(32))' > ~/.replica-runtime-token) +export REPLICA_RUNTIME_TOKEN="$(cat ~/.replica-runtime-token)" +uv venv --python 3.12 +uv pip install -e . +.venv/bin/replica-decision-runtime \ + --taskset /path/to/taskset --state /path/to/runtime-state --port 8010 +``` + +Submit and track a bounded fixture job, then promote its candidate: + +```sh +curl -sS -H "Authorization: Bearer $REPLICA_RUNTIME_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"backend":"fixture","maxUpdates":12,"maxSeconds":30}' \ + http://127.0.0.1:8010/v1/jobs + +curl -sS -H "Authorization: Bearer $REPLICA_RUNTIME_TOKEN" \ + http://127.0.0.1:8010/v1/jobs/ + +curl -sS -H "Authorization: Bearer $REPLICA_RUNTIME_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"candidateId":"","promotedBy":""}' \ + http://127.0.0.1:8010/v1/promotions +``` + +`POST /score` and `POST /v1/evaluations` accept the request that `replica-learning decision ask` already sends. Set `REPLICA_DECISION_ENDPOINT=http://127.0.0.1:8010/score` and set `REPLICA_DECISION_KEY` to the runtime token so a prototype or authoring agent can use the promoted visual critic through that command. The response keeps its `answers` object and adds the promoted candidate ID, promotion generation, and one `followsProbability` and `breaksProbability` pair per learning. Inference returns `409 candidate_not_promoted` until an operator promotes a candidate. + +The HTTP contract stays independent of the local store and trainer implementation: + +| Route | Result | +| --- | --- | +| `GET /health` | Readiness only; this is the only route without the token | +| `POST /v1/jobs` | Enqueues one job bounded by `maxUpdates` and `maxSeconds` | +| `GET /v1/jobs/` | Returns `queued`, `running`, `succeeded`, or `failed` | +| `GET /v1/candidates/` | Returns the immutable manifest and evaluation report | +| `POST /v1/promotions` | Moves the explicit promotion pointer to one candidate | +| `GET /v1/promotion` | Returns the promoted candidate and generation | +| `POST /score`, `POST /v1/evaluations` | Evaluates one page with the promoted candidate | + +### Run the fixture runtime in a container + +Build the runtime image, mount the taskset read-only, and mount a separate durable directory read-write: + +```sh +docker build -f server/runtime.Dockerfile -t replica-decision-runtime:dev . +mkdir -p /path/to/runtime-state +docker run --rm \ + -e REPLICA_RUNTIME_TOKEN \ + -v /path/to/taskset:/data/taskset:ro \ + -v /path/to/runtime-state:/data/runtime \ + -p 127.0.0.1:8010:8010 \ + replica-decision-runtime:dev +``` + +The image contains no taskset, adapter, model, or credential. It runs as user 10001, which needs read access to the taskset and write access to the runtime-state directory. Add `--user "$(id -u):$(id -g)"` when the mounted directories allow only your account; rootless Podman also needs `--userns=keep-id`. + +### Run bounded d1 training + +Start the brand-check environment on the taskset, install the d1 dependencies, and download the reviewed model revision as described below. Then start the runtime with the d1 backend: + +```sh +export REPLICA_ENV_TOKEN="$(cat ~/.replica-env-token)" +REPLICA_TASKSET_DIR=/path/to/taskset .venv/bin/server --port 8000 + +.venv/bin/replica-decision-runtime \ + --taskset /path/to/taskset \ + --state /path/to/runtime-state \ + --d1-model models/d1-omni-600M \ + --environment-url http://127.0.0.1:8000 \ + --port 8010 +``` + +Submit `{"backend":"d1-omni","maxUpdates":600,"maxSeconds":21600}` to `POST /v1/jobs`. The backend runs the existing d1 trainer, stops it when either limit is reached, scores the released and trained readouts on the test split with the existing evaluator, and stores only `best.pt` plus the reports as the candidate. The runtime process needs the downloaded base model to serve a promoted d1 candidate; the candidate does not copy the base model. + +The real d1 path needs PyTorch, Transformers, the downloaded model code, and Apple silicon or NVIDIA hardware. It was not executed in this environment. The deterministic demo and tests cover the same job, candidate, promotion, and inference contracts without claiming a real-model quality result. + +### Trust, data, and credential boundaries + +- The operator chooses the host. The local runtime never calls Replica or a cloud service. A later cloud implementation can keep the same job, candidate, promotion, and inference contracts. +- The taskset contains private screenshots, learning text, and labels. Mount it read-only. A training-consent withdrawal still requires the operator to delete the taskset and every candidate derived from that export. +- The runtime state contains adapters, evaluation reports, job errors, and the promotion pointer. Back it up or delete it under the customer's retention rules. Candidate files are created read-only and never updated in place. +- The runtime token authorizes training, promotion, candidate reads, and inference. Keep it in a protected file or secret store and pass it through the environment, not the command line or image. +- The OpenEnv token is separate. It protects the taskset served to a trainer. The d1 runtime inherits it only while invoking the existing trainer. +- The service uses plain HTTP. Bind it to localhost as shown, or put customer-operated TLS and authentication in front of it before another host can connect. +- A backend is trusted code. The local runtime checks the taskset first and bounds time and updates, but a real model backend can read the screenshots and write its candidate artifacts. + ## Train d1-omni-600M [examples/train_d1_omni.py](examples/train_d1_omni.py) trains Liquid AI's d1-omni-600M against this environment with reinforcement learning. It keeps the released model frozen and, by default, trains a small readout of 1,025 numbers on top of it. The readout reads the model's internal state at each yes-or-no question's answer options and corrects the released model's answer. `--part head` trains the whole decision head instead, about 26 million of the model's 587 million parameters. On a synthetic taskset of 518 practice pages, the head's dev score rose from 0.685 to at best 0.753, barely above the 0.750 that answering 50% to every question earns, while its practice score rose much more. The readout raised the dev score to 0.807. Either way, the trainer learns from the rewards alone. Every 10 updates it scores the dev split and keeps the weights with the best dev reward, because a small taskset is easy to memorize. diff --git a/packages/learning/environments/brand-check/__init__.py b/packages/learning/environments/brand-check/__init__.py index 21cb99b35..56f5bffae 100644 --- a/packages/learning/environments/brand-check/__init__.py +++ b/packages/learning/environments/brand-check/__init__.py @@ -5,7 +5,16 @@ from .taskset import question_options -__all__ = ["BrandCheckAction", "BrandCheckEnv", "BrandCheckObservation", "question_options"] +__all__ = [ + "BrandCheckAction", + "BrandCheckEnv", + "BrandCheckObservation", + "D1OmniBackend", + "FixtureBackend", + "LocalRuntime", + "RuntimeFailure", + "question_options", +] # The scoring modules use only the standard library. Importing the OpenEnv classes on first use # lets them, and their tests, run where openenv-core is not installed. @@ -13,6 +22,10 @@ "BrandCheckAction": ".models", "BrandCheckObservation": ".models", "BrandCheckEnv": ".client", + "D1OmniBackend": ".runtime", + "FixtureBackend": ".runtime", + "LocalRuntime": ".runtime", + "RuntimeFailure": ".runtime", } diff --git a/packages/learning/environments/brand-check/examples/runtime_demo.py b/packages/learning/environments/brand-check/examples/runtime_demo.py new file mode 100644 index 000000000..7054ded19 --- /dev/null +++ b/packages/learning/environments/brand-check/examples/runtime_demo.py @@ -0,0 +1,156 @@ +"""Run the complete local training, promotion, and inference lifecycle without a model download.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import shutil +import sys +import time +import importlib.util +from pathlib import Path + +try: + from replica_brand_check.runtime import LocalRuntime +except ModuleNotFoundError: + root = Path(__file__).resolve().parent.parent + spec = importlib.util.spec_from_file_location( + "replica_brand_check", root / "__init__.py", submodule_search_locations=[str(root)] + ) + assert spec is not None and spec.loader is not None + package = importlib.util.module_from_spec(spec) + sys.modules["replica_brand_check"] = package + spec.loader.exec_module(package) + from replica_brand_check.runtime import LocalRuntime + +PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" +) +LABELS = { + "mem_111111111111111111111111": ( + "Put a visible label above every form field.", + True, + ), + "mem_222222222222222222222222": ( + "Keep destructive actions away from the primary action.", + False, + ), +} + + +def task(number: int, split: str) -> dict: + questions = { + key: { + "type": "noul", + "instructions": "Does the candidate follow this team learning?", + "criteria": {"true": "It follows the learning", "false": "It breaks the learning"}, + } + for key in LABELS + } + return { + "schemaVersion": "replica.decision-task/v1", + "id": f"task_{number:024x}", + "split": split, + "state": { + "rules": [{"id": key, "text": text} for key, (text, _) in LABELS.items()], + "imagesNote": "Image 1 is the candidate: route / at desktop width in light theme.", + }, + "images": [ + { + "role": "candidate", + "path": "images/screen.png", + "mediaType": "image/png", + "digest": "sha256:" + hashlib.sha256(PNG).hexdigest(), + "size": "desktop", + "theme": "light", + } + ], + "questions": questions, + "labels": {key: {"answer": answer} for key, (_, answer) in LABELS.items()}, + "provenance": { + "exportId": "export_fixture", + "prototypeId": number, + "buildNumber": 1, + "route": "/", + }, + } + + +def write_taskset(directory: Path) -> None: + directory.mkdir(parents=True) + (directory / "images").mkdir() + (directory / "images" / "screen.png").write_bytes(PNG) + tasks = [ + task(1, "train"), + task(2, "train"), + task(3, "train"), + task(4, "dev"), + task(5, "test"), + task(6, "test"), + ] + manifest = { + "schemaVersion": "replica.decision-taskset/v1", + "exportId": "export_fixture", + "createdAt": "2026-11-10T00:00:00Z", + "counts": {"tasks": len(tasks)}, + } + (directory / "taskset.json").write_text(json.dumps(manifest), encoding="utf-8") + (directory / "tasks.jsonl").write_text( + "".join(json.dumps(item) + "\n" for item in tasks), encoding="utf-8" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", required=True, type=Path, help="new directory for demo data") + args = parser.parse_args() + if args.out.exists(): + print(f"error: {args.out} already exists; choose a new demo directory.", file=sys.stderr) + return 1 + taskset = args.out / "taskset" + state = args.out / "runtime" + write_taskset(taskset) + runtime = LocalRuntime(state, taskset) + try: + submitted = runtime.submit(backend="fixture", max_updates=12, max_seconds=30) + while True: + job = runtime.job(submitted["id"]) + if job["state"] in ("succeeded", "failed"): + break + time.sleep(0.01) + if job["state"] != "succeeded": + raise RuntimeError(job["error"]["message"]) + candidate = runtime.candidate(job["candidateId"]) + promotion = runtime.promote(job["candidateId"], promoted_by="demo-operator") + encoded = base64.b64encode(PNG).decode() + response = runtime.evaluate( + { + "state": task(7, "test")["state"], + "images": [f"data:image/png;base64,{encoded}"], + "questions": task(7, "test")["questions"], + } + ) + print( + json.dumps( + { + "job": job, + "candidate": candidate, + "promotion": promotion, + "evaluation": response, + "stateDirectory": str(state.resolve()), + }, + indent=2, + ) + ) + except Exception: + shutil.rmtree(args.out, ignore_errors=True) + raise + finally: + runtime.close() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/learning/environments/brand-check/pyproject.toml b/packages/learning/environments/brand-check/pyproject.toml index 138f4a4c0..bc2c5956d 100644 --- a/packages/learning/environments/brand-check/pyproject.toml +++ b/packages/learning/environments/brand-check/pyproject.toml @@ -13,6 +13,7 @@ dependencies = ["openenv-core==0.3.0"] [project.scripts] server = "replica_brand_check.server.app:main" +replica-decision-runtime = "replica_brand_check.runtime_server:main" [tool.setuptools] packages = ["replica_brand_check", "replica_brand_check.server"] diff --git a/packages/learning/environments/brand-check/runtime.py b/packages/learning/environments/brand-check/runtime.py new file mode 100644 index 000000000..4d2065961 --- /dev/null +++ b/packages/learning/environments/brand-check/runtime.py @@ -0,0 +1,808 @@ +"""Customer-operated training, promotion, and inference for Replica decision models. + +The runtime keeps orchestration separate from model code. A backend trains one bounded job, +returns an adapter and evaluation reports, and answers requests with that adapter. The local +runtime owns durable job state, immutable candidates, and the explicit promotion pointer. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import math +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import uuid +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, Protocol + +from .evaluate import EvaluationError, evaluate +from .taskset import Taskset, load_taskset, question_options + +JOB_SCHEMA = "replica.decision-training-job/v1" +CANDIDATE_SCHEMA = "replica.decision-candidate/v1" +CANDIDATE_REPORT_SCHEMA = "replica.decision-candidate-evaluation/v1" +PROMOTION_SCHEMA = "replica.decision-promotion/v1" +INFERENCE_SCHEMA = "replica.decision-evaluation-response/v1" +MIN_UPDATES = 1 +MAX_UPDATES = 100_000 +MIN_SECONDS = 1 +MAX_SECONDS = 86_400 + + +class RuntimeFailure(ValueError): + """A runtime request failed. code and status are safe to return through the HTTP service.""" + + def __init__(self, code: str, message: str, status: int = 400): + super().__init__(message) + self.code = code + self.status = status + + +@dataclass(frozen=True) +class TrainingResult: + """Artifacts a backend produced for one completed training job.""" + + adapter_name: str + adapter: bytes + baseline_report: Mapping[str, Any] + candidate_report: Mapping[str, Any] + metadata: Mapping[str, Any] + + +class TrainingBackend(Protocol): + """A provider-neutral trainer and inference adapter.""" + + name: str + + def train( + self, + taskset: Taskset, + work_directory: Path, + *, + max_updates: int, + max_seconds: float, + ) -> TrainingResult: ... + + def predict( + self, adapter: Path, request: Mapping[str, Any], *, timeout_seconds: float + ) -> Mapping[str, Any]: ... + + +class FixtureBackend: + """A deterministic readout for tests, demos, and operator practice without model downloads.""" + + name = "fixture" + + def train( + self, + taskset: Taskset, + work_directory: Path, + *, + max_updates: int, + max_seconds: float, + ) -> TrainingResult: + del work_directory, max_seconds + counts: dict[str, list[int]] = {} + for task in taskset.by_split["train"]: + for key, label in task.labels.items(): + if task.questions[key].type != "noul": + continue + totals = counts.setdefault(key, [0, 0]) + totals[0] += int(bool(label)) + totals[1] += 1 + if not counts: + raise RuntimeFailure( + "training_data_missing", + "The taskset's train split has no labeled yes-or-no questions.", + ) + # Updates control how far the fixture readout moves from the released 50% baseline. Eight + # updates fit the observed training frequency exactly, which makes the demo's gain and + # calibration evidence easy to inspect while keeping shorter jobs meaningfully bounded. + strength = min(1.0, max_updates / 8.0) + probabilities = { + key: 0.5 + strength * ((yes / total) - 0.5) + for key, (yes, total) in sorted(counts.items()) + } + adapter = { + "schemaVersion": "replica.decision-fixture-readout/v1", + "updates": max_updates, + "probabilities": probabilities, + } + baseline = _evaluate_fixture(taskset, {}) + candidate = _evaluate_fixture(taskset, probabilities) + return TrainingResult( + adapter_name="adapter.json", + adapter=_json_bytes(adapter), + baseline_report=baseline, + candidate_report=candidate, + metadata={"kind": "deterministic-fixture", "updates": max_updates}, + ) + + def predict( + self, adapter: Path, request: Mapping[str, Any], *, timeout_seconds: float + ) -> Mapping[str, Any]: + del timeout_seconds + try: + data = _read_json(adapter) + probabilities = data["probabilities"] + except (OSError, KeyError, ValueError, TypeError): + raise RuntimeFailure( + "candidate_damaged", f"Candidate adapter {adapter} is not valid fixture data.", 500 + ) from None + return _fixture_answers(request["questions"], probabilities) + + +class D1OmniBackend: + """Run the existing d1-omni trainer and predictor as a bounded subprocess backend.""" + + name = "d1-omni" + + def __init__( + self, + model_directory: str | os.PathLike[str], + *, + environment_url: str = "http://127.0.0.1:8000", + python: str = sys.executable, + ): + self.model_directory = Path(model_directory).resolve() + self.environment_url = environment_url + self.python = python + self.script = Path(__file__).parent / "examples" / "train_d1_omni.py" + + def train( + self, + taskset: Taskset, + work_directory: Path, + *, + max_updates: int, + max_seconds: float, + ) -> TrainingResult: + if not self.model_directory.is_dir(): + raise RuntimeFailure( + "model_missing", + f"The d1 model directory {self.model_directory} does not exist.", + ) + output = work_directory / "d1" + output.mkdir() + deadline = _monotonic() + max_seconds + common = [ + self.python, + str(self.script), + "--model", + str(self.model_directory), + "--taskset", + str(taskset.directory), + "--out", + str(output), + "--url", + self.environment_url, + ] + self._run( + [common[0], common[1], "train", *common[2:], "--updates", str(max_updates)], + deadline, + ) + trained = output / "best.pt" + if not trained.is_file(): + raise RuntimeFailure( + "training_failed", "The d1 trainer completed without writing best.pt.", 500 + ) + self._run( + [common[0], common[1], "evaluate", *common[2:], "--split", "test", "--name", "released"], + deadline, + ) + self._run( + [ + common[0], + common[1], + "evaluate", + *common[2:], + "--trained", + str(trained), + "--split", + "test", + "--name", + "trained", + ], + deadline, + ) + return TrainingResult( + adapter_name="adapter.pt", + adapter=trained.read_bytes(), + baseline_report=_read_json(output / "released-test-report.json"), + candidate_report=_read_json(output / "trained-test-report.json"), + metadata={ + "kind": "d1-omni-readout", + "baseModel": "LiquidAI/d1-omni-600M", + "updates": max_updates, + }, + ) + + def predict( + self, adapter: Path, request: Mapping[str, Any], *, timeout_seconds: float + ) -> Mapping[str, Any]: + if not self.model_directory.is_dir(): + raise RuntimeFailure( + "model_missing", + f"The d1 model directory {self.model_directory} does not exist.", + 500, + ) + with tempfile.TemporaryDirectory(prefix="replica-d1-request-") as holder: + directory = Path(holder) + document = dict(request) + document["images"] = _write_request_images(directory, request["images"]) + request_file = directory / "request.json" + request_file.write_bytes(_json_bytes(document)) + command = [ + self.python, + str(self.script), + "predict", + "--model", + str(self.model_directory), + "--trained", + str(adapter), + "--request", + str(request_file), + ] + completed = self._run(command, _monotonic() + timeout_seconds) + try: + return json.loads(completed.stdout)["answers"] + except (json.JSONDecodeError, KeyError, TypeError): + raise RuntimeFailure( + "inference_failed", "The d1 predictor did not return probability answers.", 500 + ) from None + + @staticmethod + def _run(command: list[str], deadline: float) -> subprocess.CompletedProcess[str]: + remaining = deadline - _monotonic() + if remaining <= 0: + raise RuntimeFailure("job_timed_out", "The bounded training job reached maxSeconds.") + try: + return subprocess.run( + command, + check=True, + capture_output=True, + text=True, + timeout=remaining, + env=_backend_environment(), + ) + except subprocess.TimeoutExpired: + raise RuntimeFailure( + "job_timed_out", "The bounded training job reached maxSeconds." + ) from None + except subprocess.CalledProcessError as error: + detail = (error.stderr or error.stdout or "no error output").strip()[-1000:] + raise RuntimeFailure( + "backend_failed", f"The d1 backend failed: {detail}", 500 + ) from None + + +class LocalRuntime: + """Durable local jobs, immutable candidates, one promotion pointer, and inference.""" + + def __init__( + self, + state_directory: str | os.PathLike[str], + taskset_directory: str | os.PathLike[str], + backends: Mapping[str, TrainingBackend] | None = None, + ): + self.state_directory = Path(state_directory).resolve() + self.taskset = load_taskset(taskset_directory) + self.taskset_digest = _taskset_digest(self.taskset) + self.backends = dict(backends or {"fixture": FixtureBackend()}) + if not self.backends: + raise RuntimeFailure("backend_missing", "Configure at least one training backend.") + self._lock = threading.RLock() + self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="replica-training") + self.state_directory.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(self.state_directory, 0o700) + for name in ("jobs", "candidates", "work"): + directory = self.state_directory / name + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(directory, 0o700) + self._recover_interrupted_jobs() + + def close(self) -> None: + """Wait for the current job and stop the worker.""" + self._executor.shutdown(wait=True) + + def submit( + self, *, backend: str = "fixture", max_updates: int, max_seconds: float + ) -> dict[str, Any]: + """Persist and enqueue one bounded job, returning its initial state.""" + _check_limits(max_updates, max_seconds) + if backend not in self.backends: + raise RuntimeFailure( + "backend_unknown", + f"Backend {json.dumps(backend)} is not configured. Choose one of " + f"{', '.join(sorted(self.backends))}.", + ) + job_id = f"job_{uuid.uuid4().hex}" + job = { + "schemaVersion": JOB_SCHEMA, + "id": job_id, + "state": "queued", + "backend": backend, + "tasksetDigest": self.taskset_digest, + "limits": {"maxUpdates": max_updates, "maxSeconds": float(max_seconds)}, + "createdAt": _now(), + "updatedAt": _now(), + } + _write_json_atomic(self._job_path(job_id), job) + self._executor.submit(self._run_job, job_id) + return job + + def job(self, job_id: str) -> dict[str, Any]: + """Return one persisted job.""" + return self._record(self._job_path(_checked_id(job_id, "job_")), "job", job_id) + + def candidate(self, candidate_id: str) -> dict[str, Any]: + """Return one immutable candidate manifest and evaluation report.""" + identifier = _checked_id(candidate_id, "candidate_") + directory = self.state_directory / "candidates" / identifier + if not directory.is_dir(): + raise RuntimeFailure("candidate_not_found", f"Candidate {candidate_id} was not found.", 404) + manifest = self._record(directory / "manifest.json", "candidate", candidate_id) + report = self._record(directory / "evaluation.json", "candidate report", candidate_id) + adapter = manifest.get("adapter", {}) + adapter_file = adapter.get("file") if isinstance(adapter, dict) else None + if ( + not isinstance(adapter, dict) + or not isinstance(adapter_file, str) + or Path(adapter_file).name != adapter_file + or not isinstance(adapter.get("digest"), str) + or manifest.get("schemaVersion") != CANDIDATE_SCHEMA + or manifest.get("id") != candidate_id + or manifest.get("tasksetDigest") != report.get("tasksetDigest") + ): + raise RuntimeFailure( + "candidate_damaged", f"Candidate {candidate_id} has an invalid adapter record.", 500 + ) + try: + adapter_bytes = (directory / adapter_file).read_bytes() + except OSError: + raise RuntimeFailure( + "candidate_damaged", f"Candidate {candidate_id} has no readable adapter.", 500 + ) from None + adapter_digest = "sha256:" + hashlib.sha256(adapter_bytes).hexdigest() + identity = hashlib.sha256() + identity.update(str(manifest.get("backend", "")).encode()) + identity.update(adapter_bytes) + identity.update(_json_bytes(report)) + expected_id = f"candidate_{identity.hexdigest()[:32]}" + if adapter_digest != adapter["digest"] or expected_id != candidate_id: + raise RuntimeFailure( + "candidate_damaged", + f"Candidate {candidate_id} no longer matches its immutable identity.", + 500, + ) + return {"manifest": manifest, "evaluation": report} + + def promote(self, candidate_id: str, *, promoted_by: str) -> dict[str, Any]: + """Explicitly make one candidate the version inference serves.""" + candidate = self.candidate(candidate_id)["manifest"] + if not isinstance(promoted_by, str) or not promoted_by.strip(): + raise RuntimeFailure("invalid_input", "promotedBy must name the operator.") + if candidate["tasksetDigest"] != self.taskset_digest: + raise RuntimeFailure( + "candidate_mismatch", "The candidate belongs to another taskset.", 409 + ) + with self._lock: + current = self.promotion(required=False) + promotion = { + "schemaVersion": PROMOTION_SCHEMA, + "generation": (current["generation"] + 1) if current else 1, + "candidateId": candidate_id, + "previousCandidateId": current["candidateId"] if current else None, + "promotedBy": promoted_by.strip(), + "promotedAt": _now(), + } + _write_json_atomic(self.state_directory / "promotion.json", promotion) + return promotion + + def promotion(self, *, required: bool = True) -> dict[str, Any] | None: + """Return the current promotion, or fail closed when inference has no promoted model.""" + path = self.state_directory / "promotion.json" + if not path.is_file(): + if required: + raise RuntimeFailure( + "candidate_not_promoted", + "No candidate is promoted. Promote an evaluated candidate before inference.", + 409, + ) + return None + return self._record(path, "promotion", "current") + + def evaluate( + self, request: Mapping[str, Any], *, timeout_seconds: float = 120 + ) -> dict[str, Any]: + """Answer one prototype page with the promoted candidate.""" + _check_request(request) + promotion = self.promotion() + candidate_id = promotion["candidateId"] + manifest = self.candidate(candidate_id)["manifest"] + backend = self.backends.get(manifest["backend"]) + if backend is None: + raise RuntimeFailure( + "backend_unavailable", + f"Promoted candidate backend {manifest['backend']} is not configured.", + 503, + ) + adapter = self.state_directory / "candidates" / candidate_id / manifest["adapter"]["file"] + answers = dict(backend.predict(adapter, request, timeout_seconds=timeout_seconds)) + evaluations = [] + for key, question in request["questions"].items(): + if question.get("type") != "noul": + continue + probabilities = answers.get(key, {}).get("probabilities") + if not _valid_yes_no(probabilities): + raise RuntimeFailure( + "inference_failed", + f"The promoted candidate did not answer question {json.dumps(key)}.", + 500, + ) + total = probabilities.get("yes", 0) + probabilities.get("no", 0) + yes = probabilities.get("yes", 0) / total + answers[key] = {"probabilities": {"yes": yes, "no": 1.0 - yes}} + evaluations.append( + { + "learningId": key, + "followsProbability": yes, + "breaksProbability": 1.0 - yes, + } + ) + return { + "schemaVersion": INFERENCE_SCHEMA, + "candidateId": candidate_id, + "promotionGeneration": promotion["generation"], + "answers": answers, + "evaluations": evaluations, + } + + def _run_job(self, job_id: str) -> None: + job = self.job(job_id) + job.update(state="running", startedAt=_now(), updatedAt=_now()) + _write_json_atomic(self._job_path(job_id), job) + work = self.state_directory / "work" / job_id + shutil.rmtree(work, ignore_errors=True) + work.mkdir(mode=0o700) + try: + backend = self.backends[job["backend"]] + result = backend.train( + self.taskset, + work, + max_updates=job["limits"]["maxUpdates"], + max_seconds=job["limits"]["maxSeconds"], + ) + candidate_id = self._persist_candidate(job, result) + job.update( + state="succeeded", + candidateId=candidate_id, + completedAt=_now(), + updatedAt=_now(), + ) + except Exception as error: + code = error.code if isinstance(error, RuntimeFailure) else "backend_failed" + job.update( + state="failed", + error={"code": code, "message": str(error)}, + completedAt=_now(), + updatedAt=_now(), + ) + finally: + _write_json_atomic(self._job_path(job_id), job) + shutil.rmtree(work, ignore_errors=True) + + def _persist_candidate(self, job: Mapping[str, Any], result: TrainingResult) -> str: + if ( + Path(result.adapter_name).name != result.adapter_name + or result.adapter_name in ("manifest.json", "evaluation.json") + ): + raise RuntimeFailure( + "backend_failed", "The backend returned an unsafe adapter file name.", 500 + ) + report = _candidate_evaluation( + self.taskset_digest, result.baseline_report, result.candidate_report + ) + identity = hashlib.sha256() + identity.update(job["backend"].encode()) + identity.update(result.adapter) + identity.update(_json_bytes(report)) + candidate_id = f"candidate_{identity.hexdigest()[:32]}" + candidates = self.state_directory / "candidates" + target = candidates / candidate_id + if target.exists(): + existing = self.candidate(candidate_id) + if existing["evaluation"] != report: + raise RuntimeFailure( + "candidate_collision", f"Candidate id {candidate_id} already has other data.", 500 + ) + return candidate_id + staging = candidates / f".{candidate_id}.{uuid.uuid4().hex}" + staging.mkdir(mode=0o700) + adapter_digest = "sha256:" + hashlib.sha256(result.adapter).hexdigest() + manifest = { + "schemaVersion": CANDIDATE_SCHEMA, + "id": candidate_id, + "backend": job["backend"], + "tasksetDigest": self.taskset_digest, + "trainingJobId": job["id"], + "createdAt": _now(), + "adapter": { + "file": result.adapter_name, + "digest": adapter_digest, + "bytes": len(result.adapter), + }, + "metadata": dict(result.metadata), + } + try: + _write_immutable(staging / result.adapter_name, result.adapter) + _write_immutable(staging / "evaluation.json", _json_bytes(report)) + _write_immutable(staging / "manifest.json", _json_bytes(manifest)) + os.replace(staging, target) + finally: + if staging.exists(): + shutil.rmtree(staging) + return candidate_id + + def _recover_interrupted_jobs(self) -> None: + for path in (self.state_directory / "jobs").glob("job_*.json"): + try: + job = _read_json(path) + except (OSError, ValueError): + continue + if job.get("state") in ("queued", "running"): + job.update( + state="failed", + error={ + "code": "runtime_restarted", + "message": "The runtime stopped before this job completed. Submit it again.", + }, + completedAt=_now(), + updatedAt=_now(), + ) + _write_json_atomic(path, job) + + def _record(self, path: Path, kind: str, identifier: str) -> dict[str, Any]: + try: + value = _read_json(path) + except FileNotFoundError: + raise RuntimeFailure( + f"{kind.replace(' ', '_')}_not_found", + f"{kind.title()} {identifier} was not found.", + 404, + ) from None + except (OSError, ValueError): + raise RuntimeFailure( + "state_damaged", f"Stored {kind} {identifier} is not valid JSON.", 500 + ) from None + if not isinstance(value, dict): + raise RuntimeFailure( + "state_damaged", f"Stored {kind} {identifier} is not an object.", 500 + ) + return value + + def _job_path(self, job_id: str) -> Path: + return self.state_directory / "jobs" / f"{job_id}.json" + + +def _candidate_evaluation( + taskset_digest: str, + baseline: Mapping[str, Any], + candidate: Mapping[str, Any], +) -> dict[str, Any]: + base_reward = baseline.get("meanReward") + candidate_reward = candidate.get("meanReward") + if not isinstance(base_reward, (int, float)) or not isinstance(candidate_reward, (int, float)): + raise RuntimeFailure( + "evaluation_failed", "Baseline and candidate reports need numeric meanReward.", 500 + ) + base_ece = baseline.get("noul", {}).get("ece") + candidate_ece = candidate.get("noul", {}).get("ece") + ece_delta = ( + candidate_ece - base_ece + if isinstance(base_ece, (int, float)) and isinstance(candidate_ece, (int, float)) + else None + ) + return { + "schemaVersion": CANDIDATE_REPORT_SCHEMA, + "tasksetDigest": taskset_digest, + "split": candidate.get("split", "test"), + "baseline": dict(baseline), + "candidate": dict(candidate), + "evidence": { + "meanRewardDelta": candidate_reward - base_reward, + "improvesMeanReward": candidate_reward > base_reward, + "calibrationErrorDelta": ece_delta, + "calibrationNotWorse": ece_delta is None or ece_delta <= 0, + }, + } + + +def _evaluate_fixture(taskset: Taskset, probabilities: Mapping[str, float]) -> dict[str, Any]: + split = "test" + predictions = { + task.id: _fixture_answers( + {key: question.spec for key, question in task.questions.items()}, probabilities + ) + for task in taskset.by_split[split] + } + try: + return evaluate(taskset, predictions, split) + except EvaluationError as error: + raise RuntimeFailure("evaluation_failed", str(error)) from None + + +def _fixture_answers( + questions: Mapping[str, Any], probabilities: Mapping[str, float] +) -> dict[str, Any]: + answers = {} + for key, question in questions.items(): + options = question_options(question) + if question["type"] == "noul": + yes = float(probabilities.get(key, 0.5)) + answers[key] = {"probabilities": {"yes": yes, "no": 1.0 - yes}} + else: + answers[key] = { + "probabilities": {option: 1.0 / len(options) for option in options} + } + return answers + + +def _taskset_digest(taskset: Taskset) -> str: + identity = { + "manifest": dict(taskset.metadata), + "tasks": [ + { + "id": task.id, + "split": task.split, + "images": [image.digest for image in task.images], + "questions": sorted(task.questions), + "labels": dict(task.labels), + } + for task in taskset.tasks + ], + } + return "sha256:" + hashlib.sha256(_json_bytes(identity)).hexdigest() + + +def _check_limits(max_updates: int, max_seconds: float) -> None: + if ( + isinstance(max_updates, bool) + or not isinstance(max_updates, int) + or not MIN_UPDATES <= max_updates <= MAX_UPDATES + ): + raise RuntimeFailure( + "invalid_limit", f"maxUpdates must be an integer from {MIN_UPDATES} to {MAX_UPDATES}." + ) + if ( + isinstance(max_seconds, bool) + or not isinstance(max_seconds, (int, float)) + or not math.isfinite(max_seconds) + or not MIN_SECONDS <= max_seconds <= MAX_SECONDS + ): + raise RuntimeFailure( + "invalid_limit", f"maxSeconds must be from {MIN_SECONDS} to {MAX_SECONDS}." + ) + + +def _check_request(request: Mapping[str, Any]) -> None: + if not isinstance(request, Mapping): + raise RuntimeFailure("invalid_input", "The evaluation request must be an object.") + if not isinstance(request.get("state"), Mapping): + raise RuntimeFailure("invalid_input", "The evaluation request state must be an object.") + images = request.get("images") + if not isinstance(images, list) or not images or not all( + isinstance(image, str) and image.startswith("data:image/") for image in images + ): + raise RuntimeFailure( + "invalid_input", "The evaluation request images must be image data URIs." + ) + questions = request.get("questions") + if not isinstance(questions, Mapping) or not any( + isinstance(question, Mapping) and question.get("type") == "noul" + for question in questions.values() + ): + raise RuntimeFailure( + "invalid_input", "The evaluation request needs at least one yes-or-no question." + ) + + +def _valid_yes_no(value: Any) -> bool: + if not isinstance(value, Mapping) or any(key not in ("yes", "no") for key in value): + return False + yes, no = value.get("yes", 0), value.get("no", 0) + return ( + not isinstance(yes, bool) + and not isinstance(no, bool) + and isinstance(yes, (int, float)) + and isinstance(no, (int, float)) + and math.isfinite(yes) + and math.isfinite(no) + and 0 <= yes <= 1 + and 0 <= no <= 1 + and abs(yes + no - 1) <= 0.001 + and yes + no > 0 + ) + + +def _write_request_images(directory: Path, images: list[str]) -> list[str]: + paths = [] + for index, uri in enumerate(images): + try: + header, encoded = uri.split(",", 1) + media_type = header.removeprefix("data:").removesuffix(";base64") + if media_type not in ("image/png", "image/jpeg", "image/webp"): + raise ValueError + data = base64.b64decode(encoded, validate=True) + except (ValueError, TypeError): + raise RuntimeFailure("invalid_input", f"Image {index + 1} is not a valid data URI.") + suffix = {"image/png": ".png", "image/jpeg": ".jpg", "image/webp": ".webp"}[media_type] + path = directory / f"image-{index + 1}{suffix}" + path.write_bytes(data) + paths.append(path.name) + return paths + + +def _backend_environment() -> dict[str, str]: + environment = dict(os.environ) + # The trainer needs REPLICA_ENV_TOKEN, but model code never needs credentials that authorize + # runtime administration or promoted inference. + environment.pop("REPLICA_RUNTIME_TOKEN", None) + environment.pop("REPLICA_DECISION_KEY", None) + return environment + + +def _checked_id(value: str, prefix: str) -> str: + if ( + not isinstance(value, str) + or not value.startswith(prefix) + or not value[len(prefix) :].isalnum() + ): + raise RuntimeFailure("invalid_input", f"The id must start with {prefix}.") + return value + + +def _read_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _json_bytes(value: Any) -> bytes: + return ( + json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n" + ).encode() + + +def _write_json_atomic(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}") + try: + temporary.write_bytes(_json_bytes(value)) + os.chmod(temporary, 0o600) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _write_immutable(path: Path, data: bytes) -> None: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o400) + with os.fdopen(descriptor, "wb") as handle: + handle.write(data) + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def _monotonic() -> float: + import time + + return time.monotonic() diff --git a/packages/learning/environments/brand-check/runtime_server.py b/packages/learning/environments/brand-check/runtime_server.py new file mode 100644 index 000000000..3576b3e81 --- /dev/null +++ b/packages/learning/environments/brand-check/runtime_server.py @@ -0,0 +1,190 @@ +"""Serve the local Replica decision-model runtime over a small authenticated HTTP API.""" + +from __future__ import annotations + +import argparse +import hmac +import json +import os +import re +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from .runtime import D1OmniBackend, FixtureBackend, LocalRuntime, RuntimeFailure + +TOKEN_VARIABLE = "REPLICA_RUNTIME_TOKEN" +MIN_TOKEN_LENGTH = 32 +MAX_REQUEST_BYTES = 180 * 1024 * 1024 +_SAFE_TOKEN = re.compile(r"^[\x21-\x7e]+$") + + +def runtime_token(environ: dict[str, str]) -> str: + """Return a strong runtime token without ever including it in an error.""" + token = environ.get(TOKEN_VARIABLE, "") + if len(token) < MIN_TOKEN_LENGTH or not _SAFE_TOKEN.fullmatch(token): + raise RuntimeFailure( + "token_missing", + f"Set {TOKEN_VARIABLE} to a random secret of at least {MIN_TOKEN_LENGTH} printable " + "ASCII characters.", + ) + return token + + +class RuntimeHTTPServer(ThreadingHTTPServer): + """HTTP server carrying one local runtime and its bearer token.""" + + daemon_threads = True + + def __init__(self, address: tuple[str, int], runtime: LocalRuntime, token: str): + super().__init__(address, RuntimeHandler) + self.runtime = runtime + self.token = token.encode() + + +class RuntimeHandler(BaseHTTPRequestHandler): + """Route runtime requests and return JSON errors with stable codes.""" + + server: RuntimeHTTPServer + protocol_version = "HTTP/1.1" + + def do_GET(self) -> None: + try: + path = urlparse(self.path).path + if path == "/health": + self._send(200, {"status": "healthy"}) + return + self._authorize() + if path == "/v1/promotion": + promotion = self.server.runtime.promotion(required=False) + self._send(200, promotion or {"candidateId": None}) + return + if path.startswith("/v1/jobs/"): + self._send(200, self.server.runtime.job(path.removeprefix("/v1/jobs/"))) + return + if path.startswith("/v1/candidates/"): + self._send( + 200, self.server.runtime.candidate(path.removeprefix("/v1/candidates/")) + ) + return + raise RuntimeFailure("not_found", f"Route {path} was not found.", 404) + except RuntimeFailure as error: + self._failure(error) + except Exception as error: + self._failure(RuntimeFailure("internal_error", str(error), 500)) + + def do_POST(self) -> None: + try: + self._authorize() + path = urlparse(self.path).path + body = self._body() + if path == "/v1/jobs": + job = self.server.runtime.submit( + backend=body.get("backend", "fixture"), + max_updates=body.get("maxUpdates"), + max_seconds=body.get("maxSeconds"), + ) + self._send(202, job) + return + if path == "/v1/promotions": + self._send( + 200, + self.server.runtime.promote( + body.get("candidateId"), promoted_by=body.get("promotedBy") + ), + ) + return + if path in ("/score", "/v1/evaluations"): + self._send(200, self.server.runtime.evaluate(body)) + return + raise RuntimeFailure("not_found", f"Route {path} was not found.", 404) + except RuntimeFailure as error: + self._failure(error) + except Exception as error: + self._failure(RuntimeFailure("internal_error", str(error), 500)) + + def log_message(self, format: str, *args: Any) -> None: + """Keep the default access log, which never includes Authorization header values.""" + super().log_message(format, *args) + + def _authorize(self) -> None: + value = self.headers.get("Authorization", "") + scheme, _, credentials = value.partition(" ") + if scheme.lower() != "bearer" or not hmac.compare_digest( + credentials.strip().encode(), self.server.token + ): + raise RuntimeFailure( + "unauthorized", + f"Send {TOKEN_VARIABLE} in Authorization: Bearer .", + 401, + ) + + def _body(self) -> dict[str, Any]: + try: + length = int(self.headers.get("Content-Length", "")) + except ValueError: + raise RuntimeFailure("invalid_input", "Content-Length must be an integer.") + if not 0 < length <= MAX_REQUEST_BYTES: + raise RuntimeFailure( + "invalid_input", f"The JSON request must be from 1 to {MAX_REQUEST_BYTES} bytes." + ) + try: + value = json.loads(self.rfile.read(length)) + except (UnicodeDecodeError, json.JSONDecodeError): + raise RuntimeFailure("invalid_input", "The request body must be one JSON object.") + if not isinstance(value, dict): + raise RuntimeFailure("invalid_input", "The request body must be one JSON object.") + return value + + def _failure(self, error: RuntimeFailure) -> None: + headers = {"WWW-Authenticate": "Bearer"} if error.status == 401 else None + self._send(error.status, {"error": {"code": error.code, "message": str(error)}}, headers) + + def _send( + self, status: int, value: Any, headers: dict[str, str] | None = None + ) -> None: + data = json.dumps(value, ensure_ascii=True, separators=(",", ":")).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + for name, value in (headers or {}).items(): + self.send_header(name, value) + self.end_headers() + self.wfile.write(data) + + +def main() -> None: + """Run one runtime bound to one taskset and one durable state directory.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--state", required=True, type=Path, help="durable runtime state directory") + parser.add_argument("--taskset", required=True, type=Path, help="read-only decision taskset") + parser.add_argument("--host", default="127.0.0.1", help="address to listen on") + parser.add_argument("--port", type=int, default=8010, help="port to listen on") + parser.add_argument("--d1-model", type=Path, help="downloaded d1-omni-600M model directory") + parser.add_argument( + "--environment-url", + default="http://127.0.0.1:8000", + help="brand-check OpenEnv server used by d1 training", + ) + args = parser.parse_args() + token = runtime_token(dict(os.environ)) + os.environ.pop(TOKEN_VARIABLE, None) + backends: dict[str, Any] = {"fixture": FixtureBackend()} + if args.d1_model: + backends["d1-omni"] = D1OmniBackend( + args.d1_model, environment_url=args.environment_url + ) + runtime = LocalRuntime(args.state, args.taskset, backends) + server = RuntimeHTTPServer((args.host, args.port), runtime, token) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + runtime.close() + + +if __name__ == "__main__": + main() diff --git a/packages/learning/environments/brand-check/server/runtime.Dockerfile b/packages/learning/environments/brand-check/server/runtime.Dockerfile new file mode 100644 index 000000000..2605352dd --- /dev/null +++ b/packages/learning/environments/brand-check/server/runtime.Dockerfile @@ -0,0 +1,17 @@ +# Local customer-operated training and inference runtime. +# Mount a taskset read-only at /data/taskset and durable state read-write at /data/runtime. +# Pass REPLICA_RUNTIME_TOKEN when the container starts; never bake data or credentials in. +FROM python:3.12.16-slim-trixie + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app + +COPY *.py /app/replica_brand_check/ +RUN useradd --uid 10001 --create-home --shell /usr/sbin/nolog replica + +USER replica +EXPOSE 8010 +HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \ + CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8010/health', timeout=3)"] +CMD ["python", "-m", "replica_brand_check.runtime_server", "--host", "0.0.0.0", "--port", "8010", "--state", "/data/runtime", "--taskset", "/data/taskset"] diff --git a/packages/learning/environments/brand-check/tests/test_runtime.py b/packages/learning/environments/brand-check/tests/test_runtime.py new file mode 100644 index 000000000..7965ff2aa --- /dev/null +++ b/packages/learning/environments/brand-check/tests/test_runtime.py @@ -0,0 +1,280 @@ +"""Customer-operated training and serving through the local runtime.""" + +from __future__ import annotations + +import base64 +import json +import threading +import time +import unittest +from collections.abc import Mapping +from pathlib import Path +from urllib import error, request + +from replica_brand_check.runtime import FixtureBackend, LocalRuntime +from replica_brand_check.runtime_server import RuntimeHTTPServer +from tests.support import PNG, RULE, make_task, write_taskset + +TOKEN = "runtime-test-token-with-32-characters" + + +class FailingBackend(FixtureBackend): + """A backend that fails after the runtime has accepted its job.""" + + def train(self, *args, **kwargs): + raise RuntimeError("deliberate backend failure") + + +class LocalRuntimeTest(unittest.TestCase): + def setUp(self): + self.taskset_dir = write_taskset( + self, + [ + make_task(1, answer=True), + make_task(2, answer=True), + make_task(3, split="test", answer=True), + make_task(4, split="test", answer=True), + ], + ) + self.state_dir = self.taskset_dir.parent / f"{self.taskset_dir.name}-state" + self.addCleanup(self._remove_state) + + def test_trains_immutable_candidate_then_requires_promotion_to_serve_it(self): + runtime = self._runtime() + + submitted = runtime.submit(backend="fixture", max_updates=8, max_seconds=5) + self.assertEqual(submitted["state"], "queued") + finished = self._wait_for_job(runtime, submitted["id"]) + self.assertEqual(finished["state"], "succeeded") + candidate_id = finished["candidateId"] + + candidate = runtime.candidate(candidate_id) + manifest, report = self._candidate_documents(candidate) + self.assertEqual(manifest["id"], candidate_id) + self.assertGreater(report["evidence"]["meanRewardDelta"], 0) + self.assertIn("calibrationErrorDelta", report["evidence"]) + before = self._candidate_snapshot(candidate) + candidate["manifest"]["backend"] = "tampered" + self.assertEqual(self._candidate_snapshot(runtime.candidate(candidate_id)), before) + for artifact in (self.state_dir / "candidates" / candidate_id).iterdir(): + self.assertEqual(artifact.stat().st_mode & 0o222, 0) + + promotion = runtime.promote(candidate_id, promoted_by="test-operator") + self.assertEqual(promotion["candidateId"], candidate_id) + response = runtime.evaluate(self._request()) + + self.assertEqual(response["candidateId"], candidate_id) + self.assertIn("schemaVersion", response) + self.assertIn(RULE, response["answers"]) + self.assertEqual(response["evaluations"][0]["learningId"], RULE) + self.assertEqual(self._candidate_snapshot(runtime.candidate(candidate_id)), before) + + def test_refuses_inference_until_a_candidate_is_explicitly_promoted(self): + runtime = self._runtime() + finished = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=2, max_seconds=5)["id"], + ) + self.assertEqual(finished["state"], "succeeded") + + with self.assertRaises(ValueError): + runtime.evaluate(self._request()) + + def test_rejects_invalid_limits_without_persisting_a_job(self): + runtime = self._runtime() + before = self._tree_snapshot(self.state_dir) + + invalid_limits = ( + {"max_updates": 0, "max_seconds": 1}, + {"max_updates": -1, "max_seconds": 1}, + {"max_updates": 1, "max_seconds": 0}, + {"max_updates": 1, "max_seconds": -1}, + {"max_updates": 10**12, "max_seconds": 1}, + {"max_updates": 1, "max_seconds": 10**12}, + ) + for limits in invalid_limits: + with self.subTest(**limits): + with self.assertRaises(ValueError): + runtime.submit(backend="fixture", **limits) + + self.assertEqual(self._tree_snapshot(self.state_dir), before) + + def test_persists_backend_failure_without_creating_a_candidate(self): + runtime = LocalRuntime( + self.state_dir, + self.taskset_dir, + backends={"failed": FailingBackend()}, + ) + self.addCleanup(runtime.close) + + submitted = runtime.submit(backend="failed", max_updates=1, max_seconds=5) + failed = self._wait_for_job(runtime, submitted["id"]) + + self.assertEqual(failed["state"], "failed") + self.assertNotIn("candidateId", failed) + self.assertEqual(list((self.state_dir / "candidates").iterdir()), []) + self.assertIsNone(runtime.promotion(required=False)) + runtime.close() + reopened = LocalRuntime( + self.state_dir, + self.taskset_dir, + backends={"failed": FailingBackend()}, + ) + self.addCleanup(reopened.close) + self.assertEqual(reopened.job(submitted["id"])["state"], "failed") + with self.assertRaises(ValueError): + reopened.candidate(submitted["id"]) + with self.assertRaises(ValueError): + reopened.promote(submitted["id"], promoted_by="test-operator") + + def test_reopening_state_preserves_job_candidate_and_promotion(self): + runtime = self._runtime() + finished = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=4, max_seconds=5)["id"], + ) + candidate_id = finished["candidateId"] + runtime.promote(candidate_id, promoted_by="test-operator") + candidate_before = self._candidate_snapshot(runtime.candidate(candidate_id)) + runtime.close() + + reopened = self._runtime() + + self.assertEqual(reopened.job(finished["id"]), finished) + self.assertEqual( + self._candidate_snapshot(reopened.candidate(candidate_id)), + candidate_before, + ) + self.assertEqual(reopened.promotion()["candidateId"], candidate_id) + self.assertEqual(reopened.evaluate(self._request())["candidateId"], candidate_id) + + def test_http_routes_require_the_token_and_track_an_authorized_job(self): + runtime = self._runtime() + server = RuntimeHTTPServer(("127.0.0.1", 0), runtime, TOKEN) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join, 1) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + base_url = f"http://127.0.0.1:{server.server_address[1]}" + body = json.dumps( + {"backend": "fixture", "maxUpdates": 2, "maxSeconds": 5} + ).encode() + + with self.assertRaises(error.HTTPError) as refused: + request.urlopen( + request.Request( + f"{base_url}/v1/jobs", + data=body, + headers={"Content-Type": "application/json"}, + ) + ) + self.assertEqual(refused.exception.code, 401) + + submitted = self._http_json( + request.Request( + f"{base_url}/v1/jobs", + data=body, + headers={ + "Authorization": f"Bearer {TOKEN}", + "Content-Type": "application/json", + }, + ) + ) + finished = self._wait_for_http_job(base_url, submitted["id"]) + self.assertEqual(finished["state"], "succeeded") + + def _runtime(self): + runtime = LocalRuntime( + self.state_dir, + self.taskset_dir, + backends={"fixture": FixtureBackend()}, + ) + self.addCleanup(runtime.close) + return runtime + + def _request(self): + task = make_task(99, split="test", answer=True) + return { + "state": task["state"], + "images": [ + "data:image/png;base64," + base64.b64encode(PNG).decode("ascii") + ], + "questions": task["questions"], + } + + def _wait_for_job(self, runtime, job_id): + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + job = runtime.job(job_id) + if job["state"] in {"succeeded", "failed"}: + return job + time.sleep(0.01) + self.fail(f"job {job_id} did not finish") + + def _wait_for_http_job(self, base_url, job_id): + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + job = self._http_json( + request.Request( + f"{base_url}/v1/jobs/{job_id}", + headers={"Authorization": f"Bearer {TOKEN}"}, + ) + ) + if job["state"] in {"succeeded", "failed"}: + return job + time.sleep(0.01) + self.fail(f"job {job_id} did not finish") + + def _http_json(self, http_request): + with request.urlopen(http_request) as response: + return json.load(response) + + def _candidate_documents(self, candidate): + self.assertIsInstance(candidate, Mapping) + manifest = self._document(candidate, "manifest") + report = self._document(candidate, "evaluation") + self.assertIsInstance(manifest, Mapping) + self.assertIsInstance(report, Mapping) + return manifest, report + + def _document(self, candidate, name): + for key in (name, f"{name}Path", f"{name}_path"): + if key not in candidate: + continue + value = candidate[key] + if isinstance(value, Mapping): + return value + return json.loads(Path(value).read_text(encoding="utf-8")) + self.fail(f"candidate does not expose its {name}") + + def _candidate_snapshot(self, candidate): + manifest, report = self._candidate_documents(candidate) + return json.dumps( + {"manifest": manifest, "report": report}, + sort_keys=True, + separators=(",", ":"), + ) + + def _tree_snapshot(self, root): + if not root.exists(): + return () + return tuple( + (path.relative_to(root).as_posix(), path.read_bytes()) + for path in sorted(root.rglob("*")) + if path.is_file() + ) + + def _remove_state(self): + if not self.state_dir.exists(): + return + for path in sorted(self.state_dir.rglob("*"), reverse=True): + if path.is_file() or path.is_symlink(): + path.unlink() + else: + path.rmdir() + self.state_dir.rmdir() + + +if __name__ == "__main__": + unittest.main() From 236148a436d3e03800cd122a05833d76490737e6 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Wed, 11 Nov 2026 08:31:36 -0800 Subject: [PATCH 2/9] docs(learning): define customer deployment slice Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/decision-model.md | 72 +++++++++++++++++++ hub/docs/learnings.md | 2 + .../environments/brand-check/README.md | 3 +- .../environments/brand-check/runtime.py | 11 +++ .../brand-check/runtime_server.py | 8 +++ .../brand-check/tests/test_runtime.py | 19 +++++ 6 files changed, 114 insertions(+), 1 deletion(-) diff --git a/docs/decision-model.md b/docs/decision-model.md index c3287cca9..0969fb611 100644 --- a/docs/decision-model.md +++ b/docs/decision-model.md @@ -24,6 +24,78 @@ flowchart LR - The Hugging Face tool, `hf`, to download the reference model. - An NVIDIA GPU or an Apple silicon Mac for the reference trainer. +## Recommended first customer deployment + +The smallest credible production deployment keeps Hub as the control plane and runs training and inference in the customer's environment. Hub currently records the base model and whether training uses a container or managed service; connecting it to the runtime is the next bounded product slice. + +```mermaid +flowchart LR + admin["Repository administrator"] --> hub["Replica Hub
control plane"] + hub --> runtime["Runtime API
CPU, always available"] + runtime --> store["Private object storage
tasksets, candidates, reports"] + runtime --> worker["Training worker
GPU, scale to zero"] + worker --> env["Brand-check OpenEnv
same training job"] + worker --> store + hub --> inference["Inference service
promoted candidate only"] + inference --> store + hub --> reviews["Build checks and
authoring feedback"] +``` + +Use one runtime API, one training-worker profile, one inference deployment, and one private storage boundary: + +| Piece | First deployment responsibility | +| --- | --- | +| Hub control plane | A repository administrator starts a bounded job, sees its state and evaluation report, and explicitly promotes a candidate. Hub stores only runtime connection metadata, job and promotion references, and the administrator's decisions. | +| Runtime API | Authenticates Hub, validates taskset and limits, records jobs, leases one job to a worker, publishes immutable candidate metadata, and changes the promotion pointer. It runs on CPU and has no model loaded. | +| Training worker | Starts only for a queued job, mounts or downloads one taskset, runs the existing brand-check environment and d1 trainer, uploads the adapter and reports, then exits. The first version runs one training job at a time per repository. | +| Inference service | Loads the promoted base model and adapter once, answers the existing `/score` contract, and reports its candidate ID. It never accepts training or promotion requests. | +| Model and taskset storage | A private object container keeps tasksets, immutable candidate adapters, and reports under repository-scoped paths. The base model is a pinned image layer or read-only model cache, not copied into every candidate. | +| Runtime state | A small persistent database or volume keeps job state, leases, candidate manifests, and promotion generations. Back it up with Hub's control-plane data. The local reference uses JSON files to prove this contract, not as the multi-instance production store. | + +### Identity, secrets, and networking + +Use separate workload identities rather than one shared cloud credential: + +| Identity | Minimum access | +| --- | --- | +| Hub | Call the runtime control API and inference service for repositories it administers. It does not read object storage directly. | +| Runtime API | Read and write runtime metadata; create repository-scoped object paths and worker jobs. | +| Training worker | Read its named taskset and base model; write only its named candidate prefix. | +| Inference service | Read the promoted candidate and pinned base model. It cannot write candidates or change promotion. | + +Use managed or workload identity for cloud resources. Keep only the Hub-to-runtime and inference credentials in the deployment's secret store, exposed to the correct process as environment variables or mounted files. The local bearer token is the single-host reference, not the preferred cloud credential. Do not put cloud keys, model-registry tokens, or runtime credentials in Hub's database, tasksets, images, or repository secrets. + +Keep the runtime API, worker, object storage, and inference service on private networking. Hub is the only control-plane caller. Run Build evaluation from Hub after it imports a successful Build, rather than exposing the inference service to GitHub-hosted runners. Authoring agents can reach the same inference contract through Hub or a customer-authenticated private ingress. Do not make the training API public. + +### CPU and GPU profiles + +| Profile | Purpose | Scaling | +| --- | --- | --- | +| Fixture, CPU only | Installation checks, lifecycle demos, contract tests, and operator practice. It does not establish real visual-model quality. | Runtime and inference can share one small CPU container. | +| d1 training | Customer-specific post-training with the existing d1 trainer and OpenEnv reward. | Start one GPU worker for a bounded job and return to zero workers afterward. | +| d1 inference | Build and authoring evaluation with the promoted candidate. | Keep one warm GPU replica when interactive latency matters; otherwise scale to zero with an accepted cold-start delay. Do not train and serve in the same GPU process. | + +### Persistence, observability, and cost controls + +- Keep tasksets and candidates immutable. Promotion changes one small pointer; rollback promotes an earlier intact candidate. +- Record the training export ID and taskset digest on each candidate. When consent withdrawal names an export, delete its tasksets and derived candidates under the customer's retention process. +- Log job, candidate, repository, backend, limit, state, and duration. Do not log screenshots, learning text, tokens, or probability request bodies. +- Emit traces or metrics for queued jobs, lease age, training duration, failures by code, candidate reward delta, inference latency, active candidate, and GPU runtime. Use customer-operated OpenTelemetry collection as Hub does. +- Require `maxUpdates` and `maxSeconds`, allow one active training job per repository, cap retries, and stop failed workers. Set storage lifecycle rules for superseded tasksets and candidates only after the customer's retention policy permits deletion. +- Keep GPU workers at zero when idle. Treat a warm inference replica as an explicit monthly cost choice and show that choice beside promotion. + +### Next bounded Hub integration + +The next pull request should connect one Hub deployment to one customer runtime and let a repository administrator opt one repository in, without adding a general cloud scheduler: + +1. Add a Hub-administrator runtime connection with one HTTPS base URL. Read its credential from `REPLICA_DECISION_RUNTIME_TOKEN`, following Hub's existing operator-secret pattern, and store only its digest for refusal tracking. +2. Add a Hub job that prepares the repository taskset in a private staging directory, uploads it through the runtime API, submits bounded training, and polls the job with capped retries. +3. Show the candidate's baseline and trained mean reward and calibration error on **Learnings**. Promotion requires a repository administrator and records a security-audit event. +4. After promotion, have Hub call the inference service for newly imported Build screenshots and expose the same endpoint to the prototype plugin for interactive authoring feedback. +5. Add provider interfaces for taskset transfer, job execution, and candidate storage only where this slice needs them. Keep the local filesystem implementation as the reference and add one workload-identity cloud implementation. + +This slice depends on authenticated Hub-to-runtime calls, private taskset transfer, durable job polling, and a long-lived inference process that loads a promoted d1 model once. It deliberately leaves multi-region scheduling, pooled GPUs across customers, automatic promotion, and a Replica-hosted service out of scope. + ## 1. Prepare a taskset A taskset is a private directory of screenshots paired with the repository's questions and the answers that review outcomes imply. Each yes-or-no question asks whether the page follows one accepted learning. diff --git a/hub/docs/learnings.md b/hub/docs/learnings.md index 29dd7556a..cf02bca58 100644 --- a/hub/docs/learnings.md +++ b/hub/docs/learnings.md @@ -59,6 +59,8 @@ When a person works with Copilot in a repository that has a Hub connection, the A vision-language model is a model that takes both an image and text as input. A decision model is a small vision-language model that looks at a Build's screenshot and answers questions about the accepted learnings with probabilities instead of prose. A team starts from a pretrained base model and post-trains it with reinforcement learning, a training method where the model answers tasks and receives a numeric reward for each answer, using the team's own review outcomes so the model can later check new Builds against the team's learnings. [ADR 0002](../../docs/adr/0002-decision-model-and-rl-environment.md) explains the design. Hub prepares training inputs, but it does not train, keep, or run the model. [Train a decision model](../../docs/decision-model.md) shows how a team runs and evaluates the reference trainer. +Hub does not connect to the customer runtime yet. The [recommended first customer deployment](../../docs/decision-model.md#recommended-first-customer-deployment) defines the next bounded integration: Hub submits private tasksets, tracks bounded jobs, shows evaluation evidence, records an administrator's promotion, and calls the promoted inference service. The customer keeps every runtime, model, credential, and storage resource. + The **Decision model** section of the **Learnings** page shows a repository administrator what is configured for the repository: - It lists the questions the model answers about each page: one yes-or-no question per accepted learning, which learning the page breaks most clearly, and how far it is from all of them. diff --git a/packages/learning/environments/brand-check/README.md b/packages/learning/environments/brand-check/README.md index 80145a591..d1d0df8ac 100644 --- a/packages/learning/environments/brand-check/README.md +++ b/packages/learning/environments/brand-check/README.md @@ -206,7 +206,8 @@ The HTTP contract stays independent of the local store and trainer implementatio | Route | Result | | --- | --- | -| `GET /health` | Readiness only; this is the only route without the token | +| `GET /health` | Process liveness; no token required | +| `GET /ready` | Returns 200 only when an intact promoted candidate has a configured backend; no token required | | `POST /v1/jobs` | Enqueues one job bounded by `maxUpdates` and `maxSeconds` | | `GET /v1/jobs/` | Returns `queued`, `running`, `succeeded`, or `failed` | | `GET /v1/candidates/` | Returns the immutable manifest and evaluation report | diff --git a/packages/learning/environments/brand-check/runtime.py b/packages/learning/environments/brand-check/runtime.py index 4d2065961..fa05544da 100644 --- a/packages/learning/environments/brand-check/runtime.py +++ b/packages/learning/environments/brand-check/runtime.py @@ -421,6 +421,17 @@ def promotion(self, *, required: bool = True) -> dict[str, Any] | None: return None return self._record(path, "promotion", "current") + def ready(self) -> None: + """Fail unless the promoted candidate is intact and its backend can serve requests.""" + promotion = self.promotion() + manifest = self.candidate(promotion["candidateId"])["manifest"] + if manifest["backend"] not in self.backends: + raise RuntimeFailure( + "backend_unavailable", + f"Promoted candidate backend {manifest['backend']} is not configured.", + 503, + ) + def evaluate( self, request: Mapping[str, Any], *, timeout_seconds: float = 120 ) -> dict[str, Any]: diff --git a/packages/learning/environments/brand-check/runtime_server.py b/packages/learning/environments/brand-check/runtime_server.py index 3576b3e81..fd7293452 100644 --- a/packages/learning/environments/brand-check/runtime_server.py +++ b/packages/learning/environments/brand-check/runtime_server.py @@ -55,6 +55,14 @@ def do_GET(self) -> None: if path == "/health": self._send(200, {"status": "healthy"}) return + if path == "/ready": + try: + self.server.runtime.ready() + except RuntimeFailure as error: + self._send(503, {"status": "not_ready", "reason": error.code}) + else: + self._send(200, {"status": "ready"}) + return self._authorize() if path == "/v1/promotion": promotion = self.server.runtime.promotion(required=False) diff --git a/packages/learning/environments/brand-check/tests/test_runtime.py b/packages/learning/environments/brand-check/tests/test_runtime.py index 7965ff2aa..a1b73598d 100644 --- a/packages/learning/environments/brand-check/tests/test_runtime.py +++ b/packages/learning/environments/brand-check/tests/test_runtime.py @@ -170,6 +170,9 @@ def test_http_routes_require_the_token_and_track_an_authorized_job(self): ) ) self.assertEqual(refused.exception.code, 401) + with self.assertRaises(error.HTTPError) as not_ready: + request.urlopen(f"{base_url}/ready") + self.assertEqual(not_ready.exception.code, 503) submitted = self._http_json( request.Request( @@ -183,6 +186,22 @@ def test_http_routes_require_the_token_and_track_an_authorized_job(self): ) finished = self._wait_for_http_job(base_url, submitted["id"]) self.assertEqual(finished["state"], "succeeded") + self._http_json( + request.Request( + f"{base_url}/v1/promotions", + data=json.dumps( + { + "candidateId": finished["candidateId"], + "promotedBy": "test-operator", + } + ).encode(), + headers={ + "Authorization": f"Bearer {TOKEN}", + "Content-Type": "application/json", + }, + ) + ) + self.assertEqual(self._http_json(f"{base_url}/ready"), {"status": "ready"}) def _runtime(self): runtime = LocalRuntime( From be4d7b6c5998af269952ef42497130ba5a96b03e Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Mon, 16 Nov 2026 09:12:36 -0800 Subject: [PATCH 3/9] refactor(learning): narrow local runtime scope Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../0002-decision-model-and-rl-environment.md | 2 +- docs/decision-model.md | 72 --- hub/docs/learnings.md | 2 - packages/learning/README.md | 2 +- .../environments/brand-check/README.md | 40 +- .../environments/brand-check/__init__.py | 2 - .../brand-check/examples/runtime_demo.py | 33 +- .../environments/brand-check/runtime.py | 442 +++++++++--------- .../brand-check/runtime_server.py | 38 +- .../brand-check/tests/test_runtime.py | 260 ++++++++++- 10 files changed, 526 insertions(+), 367 deletions(-) diff --git a/docs/adr/0002-decision-model-and-rl-environment.md b/docs/adr/0002-decision-model-and-rl-environment.md index 1449ac97e..6675b4b9e 100644 --- a/docs/adr/0002-decision-model-and-rl-environment.md +++ b/docs/adr/0002-decision-model-and-rl-environment.md @@ -124,7 +124,7 @@ The list leaves out models above 10 billion parameters and models built on a bas A trained model replaces the base model only when a repository administrator approves it. The evaluation report gives them two numbers to compare against the base model on the test set: the mean reward, which should be higher, and the calibration error, which should not be worse. This version writes the report, but Hub does not record the decision or serve a model yet. -The customer-operated reference runtime records that decision outside Hub. It accepts a taskset, tracks a job bounded by update and wall-time limits, writes the adapter and evaluation report as an immutable candidate, and serves only the candidate an operator explicitly promotes. Its HTTP contracts separate jobs, candidates, promotion, and evaluation from the local filesystem and trainer, so a customer can later implement the same lifecycle on its cloud platform. A deterministic fixture backend runs the complete lifecycle without a GPU or external model download; the d1 backend invokes the environment, trainer, scoring code, and evaluator described above. Replica does not run or receive data from either backend. +The customer-operated local reference runtime records that decision outside Hub. It accepts a taskset, tracks a job bounded by update and wall-time limits, writes the adapter and evaluation report as an immutable candidate, and serves only the candidate an operator explicitly promotes. A deterministic fixture backend runs this lifecycle without a GPU or external model download. Fixture candidates are for development and tests only and cannot be promoted in production. The runtime keeps a backend interface for a later real-model implementation without adding one here. ## Consequences diff --git a/docs/decision-model.md b/docs/decision-model.md index 0969fb611..c3287cca9 100644 --- a/docs/decision-model.md +++ b/docs/decision-model.md @@ -24,78 +24,6 @@ flowchart LR - The Hugging Face tool, `hf`, to download the reference model. - An NVIDIA GPU or an Apple silicon Mac for the reference trainer. -## Recommended first customer deployment - -The smallest credible production deployment keeps Hub as the control plane and runs training and inference in the customer's environment. Hub currently records the base model and whether training uses a container or managed service; connecting it to the runtime is the next bounded product slice. - -```mermaid -flowchart LR - admin["Repository administrator"] --> hub["Replica Hub
control plane"] - hub --> runtime["Runtime API
CPU, always available"] - runtime --> store["Private object storage
tasksets, candidates, reports"] - runtime --> worker["Training worker
GPU, scale to zero"] - worker --> env["Brand-check OpenEnv
same training job"] - worker --> store - hub --> inference["Inference service
promoted candidate only"] - inference --> store - hub --> reviews["Build checks and
authoring feedback"] -``` - -Use one runtime API, one training-worker profile, one inference deployment, and one private storage boundary: - -| Piece | First deployment responsibility | -| --- | --- | -| Hub control plane | A repository administrator starts a bounded job, sees its state and evaluation report, and explicitly promotes a candidate. Hub stores only runtime connection metadata, job and promotion references, and the administrator's decisions. | -| Runtime API | Authenticates Hub, validates taskset and limits, records jobs, leases one job to a worker, publishes immutable candidate metadata, and changes the promotion pointer. It runs on CPU and has no model loaded. | -| Training worker | Starts only for a queued job, mounts or downloads one taskset, runs the existing brand-check environment and d1 trainer, uploads the adapter and reports, then exits. The first version runs one training job at a time per repository. | -| Inference service | Loads the promoted base model and adapter once, answers the existing `/score` contract, and reports its candidate ID. It never accepts training or promotion requests. | -| Model and taskset storage | A private object container keeps tasksets, immutable candidate adapters, and reports under repository-scoped paths. The base model is a pinned image layer or read-only model cache, not copied into every candidate. | -| Runtime state | A small persistent database or volume keeps job state, leases, candidate manifests, and promotion generations. Back it up with Hub's control-plane data. The local reference uses JSON files to prove this contract, not as the multi-instance production store. | - -### Identity, secrets, and networking - -Use separate workload identities rather than one shared cloud credential: - -| Identity | Minimum access | -| --- | --- | -| Hub | Call the runtime control API and inference service for repositories it administers. It does not read object storage directly. | -| Runtime API | Read and write runtime metadata; create repository-scoped object paths and worker jobs. | -| Training worker | Read its named taskset and base model; write only its named candidate prefix. | -| Inference service | Read the promoted candidate and pinned base model. It cannot write candidates or change promotion. | - -Use managed or workload identity for cloud resources. Keep only the Hub-to-runtime and inference credentials in the deployment's secret store, exposed to the correct process as environment variables or mounted files. The local bearer token is the single-host reference, not the preferred cloud credential. Do not put cloud keys, model-registry tokens, or runtime credentials in Hub's database, tasksets, images, or repository secrets. - -Keep the runtime API, worker, object storage, and inference service on private networking. Hub is the only control-plane caller. Run Build evaluation from Hub after it imports a successful Build, rather than exposing the inference service to GitHub-hosted runners. Authoring agents can reach the same inference contract through Hub or a customer-authenticated private ingress. Do not make the training API public. - -### CPU and GPU profiles - -| Profile | Purpose | Scaling | -| --- | --- | --- | -| Fixture, CPU only | Installation checks, lifecycle demos, contract tests, and operator practice. It does not establish real visual-model quality. | Runtime and inference can share one small CPU container. | -| d1 training | Customer-specific post-training with the existing d1 trainer and OpenEnv reward. | Start one GPU worker for a bounded job and return to zero workers afterward. | -| d1 inference | Build and authoring evaluation with the promoted candidate. | Keep one warm GPU replica when interactive latency matters; otherwise scale to zero with an accepted cold-start delay. Do not train and serve in the same GPU process. | - -### Persistence, observability, and cost controls - -- Keep tasksets and candidates immutable. Promotion changes one small pointer; rollback promotes an earlier intact candidate. -- Record the training export ID and taskset digest on each candidate. When consent withdrawal names an export, delete its tasksets and derived candidates under the customer's retention process. -- Log job, candidate, repository, backend, limit, state, and duration. Do not log screenshots, learning text, tokens, or probability request bodies. -- Emit traces or metrics for queued jobs, lease age, training duration, failures by code, candidate reward delta, inference latency, active candidate, and GPU runtime. Use customer-operated OpenTelemetry collection as Hub does. -- Require `maxUpdates` and `maxSeconds`, allow one active training job per repository, cap retries, and stop failed workers. Set storage lifecycle rules for superseded tasksets and candidates only after the customer's retention policy permits deletion. -- Keep GPU workers at zero when idle. Treat a warm inference replica as an explicit monthly cost choice and show that choice beside promotion. - -### Next bounded Hub integration - -The next pull request should connect one Hub deployment to one customer runtime and let a repository administrator opt one repository in, without adding a general cloud scheduler: - -1. Add a Hub-administrator runtime connection with one HTTPS base URL. Read its credential from `REPLICA_DECISION_RUNTIME_TOKEN`, following Hub's existing operator-secret pattern, and store only its digest for refusal tracking. -2. Add a Hub job that prepares the repository taskset in a private staging directory, uploads it through the runtime API, submits bounded training, and polls the job with capped retries. -3. Show the candidate's baseline and trained mean reward and calibration error on **Learnings**. Promotion requires a repository administrator and records a security-audit event. -4. After promotion, have Hub call the inference service for newly imported Build screenshots and expose the same endpoint to the prototype plugin for interactive authoring feedback. -5. Add provider interfaces for taskset transfer, job execution, and candidate storage only where this slice needs them. Keep the local filesystem implementation as the reference and add one workload-identity cloud implementation. - -This slice depends on authenticated Hub-to-runtime calls, private taskset transfer, durable job polling, and a long-lived inference process that loads a promoted d1 model once. It deliberately leaves multi-region scheduling, pooled GPUs across customers, automatic promotion, and a Replica-hosted service out of scope. - ## 1. Prepare a taskset A taskset is a private directory of screenshots paired with the repository's questions and the answers that review outcomes imply. Each yes-or-no question asks whether the page follows one accepted learning. diff --git a/hub/docs/learnings.md b/hub/docs/learnings.md index cf02bca58..29dd7556a 100644 --- a/hub/docs/learnings.md +++ b/hub/docs/learnings.md @@ -59,8 +59,6 @@ When a person works with Copilot in a repository that has a Hub connection, the A vision-language model is a model that takes both an image and text as input. A decision model is a small vision-language model that looks at a Build's screenshot and answers questions about the accepted learnings with probabilities instead of prose. A team starts from a pretrained base model and post-trains it with reinforcement learning, a training method where the model answers tasks and receives a numeric reward for each answer, using the team's own review outcomes so the model can later check new Builds against the team's learnings. [ADR 0002](../../docs/adr/0002-decision-model-and-rl-environment.md) explains the design. Hub prepares training inputs, but it does not train, keep, or run the model. [Train a decision model](../../docs/decision-model.md) shows how a team runs and evaluates the reference trainer. -Hub does not connect to the customer runtime yet. The [recommended first customer deployment](../../docs/decision-model.md#recommended-first-customer-deployment) defines the next bounded integration: Hub submits private tasksets, tracks bounded jobs, shows evaluation evidence, records an administrator's promotion, and calls the promoted inference service. The customer keeps every runtime, model, credential, and storage resource. - The **Decision model** section of the **Learnings** page shows a repository administrator what is configured for the repository: - It lists the questions the model answers about each page: one yes-or-no question per accepted learning, which learning the page breaks most clearly, and how far it is from all of them. diff --git a/packages/learning/README.md b/packages/learning/README.md index 56b335466..25e08aef0 100644 --- a/packages/learning/README.md +++ b/packages/learning/README.md @@ -216,7 +216,7 @@ A task shows every screenshot of its page, desktop before narrow and light befor Hub names each screenshot by the address the build action was asked to capture. The build action follows redirects and Controls URL defaults without recording where the page ended up, so a comment made at the address a page redirects to finds no screenshot. -The [local decision-model runtime](environments/brand-check/README.md#run-the-local-training-and-inference-runtime) accepts that taskset, tracks a bounded fixture or d1 job, stores an immutable candidate and proper-scoring evaluation report, requires an explicit promotion, and serves the promoted visual critic through the same request shape as `decision ask`. The deterministic fixture backend demonstrates the complete lifecycle without a GPU or model download. Customers run the runtime and keep its tasksets, adapters, reports, and credentials; Replica does not host a training or inference service. +The [local decision-model runtime](environments/brand-check/README.md#run-the-local-training-and-inference-runtime) accepts that taskset, tracks a bounded deterministic fixture job, stores an immutable candidate and proper-scoring evaluation report, requires explicit promotion or rollback, and serves the promoted result through the same request shape as `decision ask`. The fixture proves the local lifecycle without a GPU or model download, but its candidates are development/test-only and cannot be promoted in production. The runtime keeps a small backend interface for later real-model work. The output directory must be new and, inside a checkout, under `.replica/`. It holds: diff --git a/packages/learning/environments/brand-check/README.md b/packages/learning/environments/brand-check/README.md index d1d0df8ac..5fdd76d29 100644 --- a/packages/learning/environments/brand-check/README.md +++ b/packages/learning/environments/brand-check/README.md @@ -161,16 +161,16 @@ A missing or invalid answer counts as wrong for accuracy and as `invalid`, and i ## Run the local training and inference runtime -The local runtime turns the environment, trainer, and evaluator into one customer-operated lifecycle. It keeps mutable job records under `jobs/`, writes each completed candidate once under `candidates/`, and serves only the candidate an operator explicitly promotes. Replica does not receive the taskset, adapter, reports, token, or requests. +The local runtime proves one complete customer-operated lifecycle. It keeps mutable job records under `jobs/`, writes each completed candidate once under `candidates/`, keeps immutable promotion generations under `promotions/`, and serves only the candidate an operator explicitly promotes or restores. Replica does not receive the taskset, adapter, reports, token, or requests. -Run the deterministic lifecycle first. It uses the real taskset loader, proper-scoring evaluation, candidate store, promotion gate, and inference contract, but its fixture readout needs no GPU or model download: +The only backend in this first reference is deterministic and ignores image content. It uses the real taskset loader, proper-scoring evaluation, candidate store, promotion gate, rollback, and inference contract without a GPU or model download. Its candidates are for development and tests only; the runtime refuses to promote them when `--environment production`. ```sh cd packages/learning/environments/brand-check python3 examples/runtime_demo.py --out /tmp/replica-runtime-demo ``` -The output shows the bounded job, immutable candidate, baseline and candidate test reports, promotion, and one structured Build evaluation. `candidate.evaluation.evidence.meanRewardDelta` states the measured gain over the 50% baseline, and `calibrationErrorDelta` states whether calibration improved. The demo leaves its state in `/tmp/replica-runtime-demo/runtime` so an operator can inspect it. +The output shows two bounded jobs, immutable candidates, baseline and candidate test reports, two promotions, an explicit rollback, and one structured Build evaluation from the restored candidate. `candidate.evaluation.evidence.meanRewardDelta` states the fixture's measured gain over the 50% baseline, not real visual-model quality. The demo leaves its state in `/tmp/replica-runtime-demo/runtime` so an operator can inspect it. For a taskset created by `replica-learning decision taskset`, make a runtime token, install the package, and start the service: @@ -180,6 +180,7 @@ export REPLICA_RUNTIME_TOKEN="$(cat ~/.replica-runtime-token)" uv venv --python 3.12 uv pip install -e . .venv/bin/replica-decision-runtime \ + --environment development \ --taskset /path/to/taskset --state /path/to/runtime-state --port 8010 ``` @@ -198,9 +199,14 @@ curl -sS -H "Authorization: Bearer $REPLICA_RUNTIME_TOKEN" \ -H "Content-Type: application/json" \ -d '{"candidateId":"","promotedBy":""}' \ http://127.0.0.1:8010/v1/promotions + +curl -sS -H "Authorization: Bearer $REPLICA_RUNTIME_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"rolledBackBy":""}' \ + http://127.0.0.1:8010/v1/rollbacks ``` -`POST /score` and `POST /v1/evaluations` accept the request that `replica-learning decision ask` already sends. Set `REPLICA_DECISION_ENDPOINT=http://127.0.0.1:8010/score` and set `REPLICA_DECISION_KEY` to the runtime token so a prototype or authoring agent can use the promoted visual critic through that command. The response keeps its `answers` object and adds the promoted candidate ID, promotion generation, and one `followsProbability` and `breaksProbability` pair per learning. Inference returns `409 candidate_not_promoted` until an operator promotes a candidate. +`POST /score` and `POST /v1/evaluations` accept the request that `replica-learning decision ask` already sends. Set `REPLICA_DECISION_ENDPOINT=http://127.0.0.1:8010/score` and set `REPLICA_DECISION_KEY` to the runtime token to exercise that endpoint contract. The fixture ignores the screenshots, so do not use its answers to judge a Build or guide authoring. The response keeps its `answers` object and adds the promoted candidate ID, promotion generation, and one `followsProbability` and `breaksProbability` pair per learning. Inference returns `409 candidate_not_promoted` until an operator promotes a candidate. The HTTP contract stays independent of the local store and trainer implementation: @@ -212,6 +218,7 @@ The HTTP contract stays independent of the local store and trainer implementatio | `GET /v1/jobs/` | Returns `queued`, `running`, `succeeded`, or `failed` | | `GET /v1/candidates/` | Returns the immutable manifest and evaluation report | | `POST /v1/promotions` | Moves the explicit promotion pointer to one candidate | +| `POST /v1/rollbacks` | Restores the immediately previous candidate as a new promotion generation | | `GET /v1/promotion` | Returns the promoted candidate and generation | | `POST /score`, `POST /v1/evaluations` | Evaluates one page with the promoted candidate | @@ -230,27 +237,11 @@ docker run --rm \ replica-decision-runtime:dev ``` -The image contains no taskset, adapter, model, or credential. It runs as user 10001, which needs read access to the taskset and write access to the runtime-state directory. Add `--user "$(id -u):$(id -g)"` when the mounted directories allow only your account; rootless Podman also needs `--userns=keep-id`. - -### Run bounded d1 training - -Start the brand-check environment on the taskset, install the d1 dependencies, and download the reviewed model revision as described below. Then start the runtime with the d1 backend: - -```sh -export REPLICA_ENV_TOKEN="$(cat ~/.replica-env-token)" -REPLICA_TASKSET_DIR=/path/to/taskset .venv/bin/server --port 8000 - -.venv/bin/replica-decision-runtime \ - --taskset /path/to/taskset \ - --state /path/to/runtime-state \ - --d1-model models/d1-omni-600M \ - --environment-url http://127.0.0.1:8000 \ - --port 8010 -``` +The image contains no taskset, candidate, model, or credential. It runs as user 10001, which needs read access to the taskset and write access to the runtime-state directory. Add `--user "$(id -u):$(id -g)"` when the mounted directories allow only your account; rootless Podman also needs `--userns=keep-id`. -Submit `{"backend":"d1-omni","maxUpdates":600,"maxSeconds":21600}` to `POST /v1/jobs`. The backend runs the existing d1 trainer, stops it when either limit is reached, scores the released and trained readouts on the test split with the existing evaluator, and stores only `best.pt` plus the reports as the candidate. The runtime process needs the downloaded base model to serve a promoted d1 candidate; the candidate does not copy the base model. +### Add another backend later -The real d1 path needs PyTorch, Transformers, the downloaded model code, and Apple silicon or NVIDIA hardware. It was not executed in this environment. The deterministic demo and tests cover the same job, candidate, promotion, and inference contracts without claiming a real-model quality result. +`TrainingBackend` is the extension seam. A backend declares its name and whether its candidates may be promoted in production, implements one bounded `train` call that returns an adapter plus baseline and candidate reports, and implements `predict` for that adapter. The runtime owns job state, candidate integrity, promotion, rollback, and HTTP authentication. This first reference does not connect the existing d1 trainer to that interface. ### Trust, data, and credential boundaries @@ -258,9 +249,8 @@ The real d1 path needs PyTorch, Transformers, the downloaded model code, and App - The taskset contains private screenshots, learning text, and labels. Mount it read-only. A training-consent withdrawal still requires the operator to delete the taskset and every candidate derived from that export. - The runtime state contains adapters, evaluation reports, job errors, and the promotion pointer. Back it up or delete it under the customer's retention rules. Candidate files are created read-only and never updated in place. - The runtime token authorizes training, promotion, candidate reads, and inference. Keep it in a protected file or secret store and pass it through the environment, not the command line or image. -- The OpenEnv token is separate. It protects the taskset served to a trainer. The d1 runtime inherits it only while invoking the existing trainer. - The service uses plain HTTP. Bind it to localhost as shown, or put customer-operated TLS and authentication in front of it before another host can connect. -- A backend is trusted code. The local runtime checks the taskset first and bounds time and updates, but a real model backend can read the screenshots and write its candidate artifacts. +- A backend is trusted code. The local runtime checks the taskset first and bounds time and updates, but a later model backend can read the screenshots and return candidate artifacts. ## Train d1-omni-600M diff --git a/packages/learning/environments/brand-check/__init__.py b/packages/learning/environments/brand-check/__init__.py index 56f5bffae..03393960e 100644 --- a/packages/learning/environments/brand-check/__init__.py +++ b/packages/learning/environments/brand-check/__init__.py @@ -9,7 +9,6 @@ "BrandCheckAction", "BrandCheckEnv", "BrandCheckObservation", - "D1OmniBackend", "FixtureBackend", "LocalRuntime", "RuntimeFailure", @@ -22,7 +21,6 @@ "BrandCheckAction": ".models", "BrandCheckObservation": ".models", "BrandCheckEnv": ".client", - "D1OmniBackend": ".runtime", "FixtureBackend": ".runtime", "LocalRuntime": ".runtime", "RuntimeFailure": ".runtime", diff --git a/packages/learning/environments/brand-check/examples/runtime_demo.py b/packages/learning/environments/brand-check/examples/runtime_demo.py index 7054ded19..29c682db3 100644 --- a/packages/learning/environments/brand-check/examples/runtime_demo.py +++ b/packages/learning/environments/brand-check/examples/runtime_demo.py @@ -5,11 +5,11 @@ import argparse import base64 import hashlib +import importlib.util import json import shutil import sys import time -import importlib.util from pathlib import Path try: @@ -114,16 +114,23 @@ def main() -> int: write_taskset(taskset) runtime = LocalRuntime(state, taskset) try: - submitted = runtime.submit(backend="fixture", max_updates=12, max_seconds=30) - while True: - job = runtime.job(submitted["id"]) - if job["state"] in ("succeeded", "failed"): - break - time.sleep(0.01) - if job["state"] != "succeeded": - raise RuntimeError(job["error"]["message"]) - candidate = runtime.candidate(job["candidateId"]) - promotion = runtime.promote(job["candidateId"], promoted_by="demo-operator") + jobs = [] + for updates in (4, 8): + submitted = runtime.submit( + backend="fixture", max_updates=updates, max_seconds=30 + ) + while True: + job = runtime.job(submitted["id"]) + if job["state"] in ("succeeded", "failed"): + break + time.sleep(0.01) + if job["state"] != "succeeded": + raise RuntimeError(job["error"]["message"]) + jobs.append(job) + first = runtime.promote(jobs[0]["candidateId"], promoted_by="demo-operator") + second = runtime.promote(jobs[1]["candidateId"], promoted_by="demo-operator") + rollback = runtime.rollback(rolled_back_by="demo-operator") + candidate = runtime.candidate(rollback["candidateId"]) encoded = base64.b64encode(PNG).decode() response = runtime.evaluate( { @@ -135,9 +142,9 @@ def main() -> int: print( json.dumps( { - "job": job, + "jobs": jobs, "candidate": candidate, - "promotion": promotion, + "promotions": [first, second, rollback], "evaluation": response, "stateDirectory": str(state.resolve()), }, diff --git a/packages/learning/environments/brand-check/runtime.py b/packages/learning/environments/brand-check/runtime.py index fa05544da..74560ca83 100644 --- a/packages/learning/environments/brand-check/runtime.py +++ b/packages/learning/environments/brand-check/runtime.py @@ -7,15 +7,11 @@ from __future__ import annotations -import base64 import hashlib import json import math import os import shutil -import subprocess -import sys -import tempfile import threading import uuid from concurrent.futures import ThreadPoolExecutor @@ -36,6 +32,7 @@ MAX_UPDATES = 100_000 MIN_SECONDS = 1 MAX_SECONDS = 86_400 +ENVIRONMENTS = ("development", "test", "production") class RuntimeFailure(ValueError): @@ -62,6 +59,7 @@ class TrainingBackend(Protocol): """A provider-neutral trainer and inference adapter.""" name: str + production_eligible: bool def train( self, @@ -81,6 +79,7 @@ class FixtureBackend: """A deterministic readout for tests, demos, and operator practice without model downloads.""" name = "fixture" + production_eligible = False def train( self, @@ -141,150 +140,6 @@ def predict( return _fixture_answers(request["questions"], probabilities) -class D1OmniBackend: - """Run the existing d1-omni trainer and predictor as a bounded subprocess backend.""" - - name = "d1-omni" - - def __init__( - self, - model_directory: str | os.PathLike[str], - *, - environment_url: str = "http://127.0.0.1:8000", - python: str = sys.executable, - ): - self.model_directory = Path(model_directory).resolve() - self.environment_url = environment_url - self.python = python - self.script = Path(__file__).parent / "examples" / "train_d1_omni.py" - - def train( - self, - taskset: Taskset, - work_directory: Path, - *, - max_updates: int, - max_seconds: float, - ) -> TrainingResult: - if not self.model_directory.is_dir(): - raise RuntimeFailure( - "model_missing", - f"The d1 model directory {self.model_directory} does not exist.", - ) - output = work_directory / "d1" - output.mkdir() - deadline = _monotonic() + max_seconds - common = [ - self.python, - str(self.script), - "--model", - str(self.model_directory), - "--taskset", - str(taskset.directory), - "--out", - str(output), - "--url", - self.environment_url, - ] - self._run( - [common[0], common[1], "train", *common[2:], "--updates", str(max_updates)], - deadline, - ) - trained = output / "best.pt" - if not trained.is_file(): - raise RuntimeFailure( - "training_failed", "The d1 trainer completed without writing best.pt.", 500 - ) - self._run( - [common[0], common[1], "evaluate", *common[2:], "--split", "test", "--name", "released"], - deadline, - ) - self._run( - [ - common[0], - common[1], - "evaluate", - *common[2:], - "--trained", - str(trained), - "--split", - "test", - "--name", - "trained", - ], - deadline, - ) - return TrainingResult( - adapter_name="adapter.pt", - adapter=trained.read_bytes(), - baseline_report=_read_json(output / "released-test-report.json"), - candidate_report=_read_json(output / "trained-test-report.json"), - metadata={ - "kind": "d1-omni-readout", - "baseModel": "LiquidAI/d1-omni-600M", - "updates": max_updates, - }, - ) - - def predict( - self, adapter: Path, request: Mapping[str, Any], *, timeout_seconds: float - ) -> Mapping[str, Any]: - if not self.model_directory.is_dir(): - raise RuntimeFailure( - "model_missing", - f"The d1 model directory {self.model_directory} does not exist.", - 500, - ) - with tempfile.TemporaryDirectory(prefix="replica-d1-request-") as holder: - directory = Path(holder) - document = dict(request) - document["images"] = _write_request_images(directory, request["images"]) - request_file = directory / "request.json" - request_file.write_bytes(_json_bytes(document)) - command = [ - self.python, - str(self.script), - "predict", - "--model", - str(self.model_directory), - "--trained", - str(adapter), - "--request", - str(request_file), - ] - completed = self._run(command, _monotonic() + timeout_seconds) - try: - return json.loads(completed.stdout)["answers"] - except (json.JSONDecodeError, KeyError, TypeError): - raise RuntimeFailure( - "inference_failed", "The d1 predictor did not return probability answers.", 500 - ) from None - - @staticmethod - def _run(command: list[str], deadline: float) -> subprocess.CompletedProcess[str]: - remaining = deadline - _monotonic() - if remaining <= 0: - raise RuntimeFailure("job_timed_out", "The bounded training job reached maxSeconds.") - try: - return subprocess.run( - command, - check=True, - capture_output=True, - text=True, - timeout=remaining, - env=_backend_environment(), - ) - except subprocess.TimeoutExpired: - raise RuntimeFailure( - "job_timed_out", "The bounded training job reached maxSeconds." - ) from None - except subprocess.CalledProcessError as error: - detail = (error.stderr or error.stdout or "no error output").strip()[-1000:] - raise RuntimeFailure( - "backend_failed", f"The d1 backend failed: {detail}", 500 - ) from None - - class LocalRuntime: """Durable local jobs, immutable candidates, one promotion pointer, and inference.""" @@ -293,10 +148,18 @@ def __init__( state_directory: str | os.PathLike[str], taskset_directory: str | os.PathLike[str], backends: Mapping[str, TrainingBackend] | None = None, + *, + environment: str = "development", ): + if environment not in ENVIRONMENTS: + raise RuntimeFailure( + "environment_invalid", + f"Environment must be one of {', '.join(ENVIRONMENTS)}.", + ) self.state_directory = Path(state_directory).resolve() self.taskset = load_taskset(taskset_directory) self.taskset_digest = _taskset_digest(self.taskset) + self.environment = environment self.backends = dict(backends or {"fixture": FixtureBackend()}) if not self.backends: raise RuntimeFailure("backend_missing", "Configure at least one training backend.") @@ -304,10 +167,11 @@ def __init__( self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="replica-training") self.state_directory.mkdir(parents=True, exist_ok=True, mode=0o700) os.chmod(self.state_directory, 0o700) - for name in ("jobs", "candidates", "work"): + for name in ("jobs", "candidates", "promotions", "work"): directory = self.state_directory / name directory.mkdir(parents=True, exist_ok=True, mode=0o700) os.chmod(directory, 0o700) + self._recover_promotion() self._recover_interrupted_jobs() def close(self) -> None: @@ -362,6 +226,7 @@ def candidate(self, candidate_id: str) -> dict[str, Any]: or manifest.get("schemaVersion") != CANDIDATE_SCHEMA or manifest.get("id") != candidate_id or manifest.get("tasksetDigest") != report.get("tasksetDigest") + or not isinstance(manifest.get("productionEligible"), bool) ): raise RuntimeFailure( "candidate_damaged", f"Candidate {candidate_id} has an invalid adapter record.", 500 @@ -373,11 +238,9 @@ def candidate(self, candidate_id: str) -> dict[str, Any]: "candidate_damaged", f"Candidate {candidate_id} has no readable adapter.", 500 ) from None adapter_digest = "sha256:" + hashlib.sha256(adapter_bytes).hexdigest() - identity = hashlib.sha256() - identity.update(str(manifest.get("backend", "")).encode()) - identity.update(adapter_bytes) - identity.update(_json_bytes(report)) - expected_id = f"candidate_{identity.hexdigest()[:32]}" + manifest_identity = dict(manifest) + manifest_identity.pop("id", None) + expected_id = _candidate_id(manifest_identity, adapter_bytes, report) if adapter_digest != adapter["digest"] or expected_id != candidate_id: raise RuntimeFailure( "candidate_damaged", @@ -388,7 +251,7 @@ def candidate(self, candidate_id: str) -> dict[str, Any]: def promote(self, candidate_id: str, *, promoted_by: str) -> dict[str, Any]: """Explicitly make one candidate the version inference serves.""" - candidate = self.candidate(candidate_id)["manifest"] + candidate = self._promotable_candidate(candidate_id) if not isinstance(promoted_by, str) or not promoted_by.strip(): raise RuntimeFailure("invalid_input", "promotedBy must name the operator.") if candidate["tasksetDigest"] != self.taskset_digest: @@ -397,15 +260,38 @@ def promote(self, candidate_id: str, *, promoted_by: str) -> dict[str, Any]: ) with self._lock: current = self.promotion(required=False) - promotion = { - "schemaVersion": PROMOTION_SCHEMA, - "generation": (current["generation"] + 1) if current else 1, - "candidateId": candidate_id, - "previousCandidateId": current["candidateId"] if current else None, - "promotedBy": promoted_by.strip(), - "promotedAt": _now(), - } - _write_json_atomic(self.state_directory / "promotion.json", promotion) + promotion = self._promotion( + candidate_id, + previous_candidate_id=current["candidateId"] if current else None, + generation=(current["generation"] + 1) if current else 1, + promoted_by=promoted_by.strip(), + operation="promote", + ) + self._write_promotion(promotion) + return promotion + + def rollback(self, *, rolled_back_by: str) -> dict[str, Any]: + """Promote the immediately previous candidate as a new explicit generation.""" + if not isinstance(rolled_back_by, str) or not rolled_back_by.strip(): + raise RuntimeFailure("invalid_input", "rolledBackBy must name the operator.") + with self._lock: + current = self.promotion() + candidate_id = current.get("previousCandidateId") + if not isinstance(candidate_id, str): + raise RuntimeFailure( + "rollback_unavailable", + "The current promotion has no previous candidate to restore.", + 409, + ) + self._promotable_candidate(candidate_id) + promotion = self._promotion( + candidate_id, + previous_candidate_id=current["candidateId"], + generation=current["generation"] + 1, + promoted_by=rolled_back_by.strip(), + operation="rollback", + ) + self._write_promotion(promotion) return promotion def promotion(self, *, required: bool = True) -> dict[str, Any] | None: @@ -419,18 +305,27 @@ def promotion(self, *, required: bool = True) -> dict[str, Any] | None: 409, ) return None - return self._record(path, "promotion", "current") + promotion = self._record(path, "promotion", "current") + generation = promotion.get("generation") + if not isinstance(generation, int) or generation < 1: + raise RuntimeFailure("state_damaged", "Stored promotion generation is invalid.", 500) + history = self._record( + self.state_directory / "promotions" / f"{generation:08d}.json", + "promotion generation", + str(generation), + ) + if history != promotion: + raise RuntimeFailure( + "state_damaged", + "The current promotion does not match its immutable generation.", + 500, + ) + return promotion def ready(self) -> None: """Fail unless the promoted candidate is intact and its backend can serve requests.""" promotion = self.promotion() - manifest = self.candidate(promotion["candidateId"])["manifest"] - if manifest["backend"] not in self.backends: - raise RuntimeFailure( - "backend_unavailable", - f"Promoted candidate backend {manifest['backend']} is not configured.", - 503, - ) + self._promotable_candidate(promotion["candidateId"]) def evaluate( self, request: Mapping[str, Any], *, timeout_seconds: float = 120 @@ -439,7 +334,7 @@ def evaluate( _check_request(request) promotion = self.promotion() candidate_id = promotion["candidateId"] - manifest = self.candidate(candidate_id)["manifest"] + manifest = self._promotable_candidate(candidate_id) backend = self.backends.get(manifest["backend"]) if backend is None: raise RuntimeFailure( @@ -523,26 +418,9 @@ def _persist_candidate(self, job: Mapping[str, Any], result: TrainingResult) -> report = _candidate_evaluation( self.taskset_digest, result.baseline_report, result.candidate_report ) - identity = hashlib.sha256() - identity.update(job["backend"].encode()) - identity.update(result.adapter) - identity.update(_json_bytes(report)) - candidate_id = f"candidate_{identity.hexdigest()[:32]}" - candidates = self.state_directory / "candidates" - target = candidates / candidate_id - if target.exists(): - existing = self.candidate(candidate_id) - if existing["evaluation"] != report: - raise RuntimeFailure( - "candidate_collision", f"Candidate id {candidate_id} already has other data.", 500 - ) - return candidate_id - staging = candidates / f".{candidate_id}.{uuid.uuid4().hex}" - staging.mkdir(mode=0o700) adapter_digest = "sha256:" + hashlib.sha256(result.adapter).hexdigest() - manifest = { + manifest_identity = { "schemaVersion": CANDIDATE_SCHEMA, - "id": candidate_id, "backend": job["backend"], "tasksetDigest": self.taskset_digest, "trainingJobId": job["id"], @@ -552,8 +430,24 @@ def _persist_candidate(self, job: Mapping[str, Any], result: TrainingResult) -> "digest": adapter_digest, "bytes": len(result.adapter), }, + "productionEligible": bool( + self.backends[job["backend"]].production_eligible + ), "metadata": dict(result.metadata), } + candidate_id = _candidate_id(manifest_identity, result.adapter, report) + manifest = {"id": candidate_id, **manifest_identity} + candidates = self.state_directory / "candidates" + target = candidates / candidate_id + if target.exists(): + existing = self.candidate(candidate_id) + if existing["manifest"] != manifest or existing["evaluation"] != report: + raise RuntimeFailure( + "candidate_collision", f"Candidate id {candidate_id} already has other data.", 500 + ) + return candidate_id + staging = candidates / f".{candidate_id}.{uuid.uuid4().hex}" + staging.mkdir(mode=0o700) try: _write_immutable(staging / result.adapter_name, result.adapter) _write_immutable(staging / "evaluation.json", _json_bytes(report)) @@ -582,6 +476,41 @@ def _recover_interrupted_jobs(self) -> None: ) _write_json_atomic(path, job) + def _recover_promotion(self) -> None: + history = sorted((self.state_directory / "promotions").glob("*.json")) + if not history: + return + latest = self._record(history[-1], "promotion generation", history[-1].stem) + latest_generation = latest.get("generation") + if ( + not isinstance(latest_generation, int) + or history[-1].name != f"{latest_generation:08d}.json" + ): + raise RuntimeFailure( + "state_damaged", "The latest promotion generation is invalid.", 500 + ) + current_path = self.state_directory / "promotion.json" + current = ( + self._record(current_path, "promotion", "current") + if current_path.is_file() + else None + ) + current_generation = current.get("generation", 0) if current else 0 + if latest_generation == current_generation: + return + if ( + latest_generation == current_generation + 1 + and latest.get("previousCandidateId") + == (current.get("candidateId") if current else None) + ): + _write_json_atomic(current_path, latest) + return + raise RuntimeFailure( + "state_damaged", + "Promotion history is not one generation ahead of the current pointer.", + 500, + ) + def _record(self, path: Path, kind: str, identifier: str) -> dict[str, Any]: try: value = _read_json(path) @@ -604,6 +533,60 @@ def _record(self, path: Path, kind: str, identifier: str) -> dict[str, Any]: def _job_path(self, job_id: str) -> Path: return self.state_directory / "jobs" / f"{job_id}.json" + def _promotable_candidate(self, candidate_id: str) -> dict[str, Any]: + candidate = self.candidate(candidate_id)["manifest"] + backend = self.backends.get(candidate["backend"]) + if backend is None: + raise RuntimeFailure( + "backend_unavailable", + f"Candidate backend {candidate['backend']} is not configured.", + 503, + ) + if self.environment == "production" and not candidate.get("productionEligible"): + raise RuntimeFailure( + "candidate_not_production_eligible", + f"Backend {candidate['backend']} produces development and test candidates only.", + 409, + ) + return candidate + + def _write_promotion(self, promotion: Mapping[str, Any]) -> None: + history = ( + self.state_directory + / "promotions" + / f"{promotion['generation']:08d}.json" + ) + data = _json_bytes(promotion) + if history.exists(): + if history.read_bytes() != data: + raise RuntimeFailure( + "state_damaged", + f"Promotion generation {promotion['generation']} already has other data.", + 500, + ) + else: + _write_immutable(history, data) + _write_json_atomic(self.state_directory / "promotion.json", promotion) + + @staticmethod + def _promotion( + candidate_id: str, + *, + previous_candidate_id: str | None, + generation: int, + promoted_by: str, + operation: str, + ) -> dict[str, Any]: + return { + "schemaVersion": PROMOTION_SCHEMA, + "generation": generation, + "candidateId": candidate_id, + "previousCandidateId": previous_candidate_id, + "operation": operation, + "promotedBy": promoted_by, + "promotedAt": _now(), + } + def _candidate_evaluation( taskset_digest: str, @@ -638,6 +621,19 @@ def _candidate_evaluation( } +def _candidate_id( + manifest: Mapping[str, Any], + adapter: bytes, + report: Mapping[str, Any], +) -> str: + identity = hashlib.sha256() + identity.update(_json_bytes(manifest)) + identity.update(b"\0") + identity.update(adapter) + identity.update(_json_bytes(report)) + return f"candidate_{identity.hexdigest()[:32]}" + + def _evaluate_fixture(taskset: Taskset, probabilities: Mapping[str, float]) -> dict[str, Any]: split = "test" predictions = { @@ -670,14 +666,29 @@ def _fixture_answers( def _taskset_digest(taskset: Taskset) -> str: identity = { - "manifest": dict(taskset.metadata), + "manifest": _json_data(taskset.metadata), "tasks": [ { "id": task.id, "split": task.split, - "images": [image.digest for image in task.images], - "questions": sorted(task.questions), - "labels": dict(task.labels), + "state": _json_data(task.state), + "images": [ + { + "role": image.role, + "path": image.name, + "mediaType": image.media_type, + "digest": image.digest, + "size": image.size, + "theme": image.theme, + } + for image in task.images + ], + "questions": { + key: _json_data(question.spec) + for key, question in task.questions.items() + }, + "labels": _json_data(task.labels), + "provenance": _json_data(task.provenance), } for task in taskset.tasks ], @@ -685,6 +696,14 @@ def _taskset_digest(taskset: Taskset) -> str: return "sha256:" + hashlib.sha256(_json_bytes(identity)).hexdigest() +def _json_data(value: Any) -> Any: + if isinstance(value, Mapping): + return {key: _json_data(member) for key, member in value.items()} + if isinstance(value, (list, tuple)): + return [_json_data(member) for member in value] + return value + + def _check_limits(max_updates: int, max_seconds: float) -> None: if ( isinstance(max_updates, bool) @@ -745,33 +764,6 @@ def _valid_yes_no(value: Any) -> bool: ) -def _write_request_images(directory: Path, images: list[str]) -> list[str]: - paths = [] - for index, uri in enumerate(images): - try: - header, encoded = uri.split(",", 1) - media_type = header.removeprefix("data:").removesuffix(";base64") - if media_type not in ("image/png", "image/jpeg", "image/webp"): - raise ValueError - data = base64.b64decode(encoded, validate=True) - except (ValueError, TypeError): - raise RuntimeFailure("invalid_input", f"Image {index + 1} is not a valid data URI.") - suffix = {"image/png": ".png", "image/jpeg": ".jpg", "image/webp": ".webp"}[media_type] - path = directory / f"image-{index + 1}{suffix}" - path.write_bytes(data) - paths.append(path.name) - return paths - - -def _backend_environment() -> dict[str, str]: - environment = dict(os.environ) - # The trainer needs REPLICA_ENV_TOKEN, but model code never needs credentials that authorize - # runtime administration or promoted inference. - environment.pop("REPLICA_RUNTIME_TOKEN", None) - environment.pop("REPLICA_DECISION_KEY", None) - return environment - - def _checked_id(value: str, prefix: str) -> str: if ( not isinstance(value, str) @@ -811,9 +803,3 @@ def _write_immutable(path: Path, data: bytes) -> None: def _now() -> str: return datetime.now(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") - - -def _monotonic() -> float: - import time - - return time.monotonic() diff --git a/packages/learning/environments/brand-check/runtime_server.py b/packages/learning/environments/brand-check/runtime_server.py index fd7293452..96f584f94 100644 --- a/packages/learning/environments/brand-check/runtime_server.py +++ b/packages/learning/environments/brand-check/runtime_server.py @@ -12,7 +12,7 @@ from typing import Any from urllib.parse import urlparse -from .runtime import D1OmniBackend, FixtureBackend, LocalRuntime, RuntimeFailure +from .runtime import ENVIRONMENTS, FixtureBackend, LocalRuntime, RuntimeFailure TOKEN_VARIABLE = "REPLICA_RUNTIME_TOKEN" MIN_TOKEN_LENGTH = 32 @@ -80,7 +80,10 @@ def do_GET(self) -> None: except RuntimeFailure as error: self._failure(error) except Exception as error: - self._failure(RuntimeFailure("internal_error", str(error), 500)) + self.log_error("Unhandled runtime error: %r", error) + self._failure( + RuntimeFailure("internal_error", "The runtime failed unexpectedly.", 500) + ) def do_POST(self) -> None: try: @@ -103,6 +106,12 @@ def do_POST(self) -> None: ), ) return + if path == "/v1/rollbacks": + self._send( + 200, + self.server.runtime.rollback(rolled_back_by=body.get("rolledBackBy")), + ) + return if path in ("/score", "/v1/evaluations"): self._send(200, self.server.runtime.evaluate(body)) return @@ -110,7 +119,10 @@ def do_POST(self) -> None: except RuntimeFailure as error: self._failure(error) except Exception as error: - self._failure(RuntimeFailure("internal_error", str(error), 500)) + self.log_error("Unhandled runtime error: %r", error) + self._failure( + RuntimeFailure("internal_error", "The runtime failed unexpectedly.", 500) + ) def log_message(self, format: str, *args: Any) -> None: """Keep the default access log, which never includes Authorization header values.""" @@ -169,21 +181,21 @@ def main() -> None: parser.add_argument("--taskset", required=True, type=Path, help="read-only decision taskset") parser.add_argument("--host", default="127.0.0.1", help="address to listen on") parser.add_argument("--port", type=int, default=8010, help="port to listen on") - parser.add_argument("--d1-model", type=Path, help="downloaded d1-omni-600M model directory") parser.add_argument( - "--environment-url", - default="http://127.0.0.1:8000", - help="brand-check OpenEnv server used by d1 training", + "--environment", + choices=ENVIRONMENTS, + default="development", + help="runtime environment; fixture candidates cannot be promoted in production", ) args = parser.parse_args() token = runtime_token(dict(os.environ)) os.environ.pop(TOKEN_VARIABLE, None) - backends: dict[str, Any] = {"fixture": FixtureBackend()} - if args.d1_model: - backends["d1-omni"] = D1OmniBackend( - args.d1_model, environment_url=args.environment_url - ) - runtime = LocalRuntime(args.state, args.taskset, backends) + runtime = LocalRuntime( + args.state, + args.taskset, + {"fixture": FixtureBackend()}, + environment=args.environment, + ) server = RuntimeHTTPServer((args.host, args.port), runtime, token) try: server.serve_forever() diff --git a/packages/learning/environments/brand-check/tests/test_runtime.py b/packages/learning/environments/brand-check/tests/test_runtime.py index a1b73598d..c6e0c124a 100644 --- a/packages/learning/environments/brand-check/tests/test_runtime.py +++ b/packages/learning/environments/brand-check/tests/test_runtime.py @@ -11,7 +11,7 @@ from pathlib import Path from urllib import error, request -from replica_brand_check.runtime import FixtureBackend, LocalRuntime +from replica_brand_check.runtime import FixtureBackend, LocalRuntime, RuntimeFailure from replica_brand_check.runtime_server import RuntimeHTTPServer from tests.support import PNG, RULE, make_task, write_taskset @@ -51,6 +51,7 @@ def test_trains_immutable_candidate_then_requires_promotion_to_serve_it(self): candidate = runtime.candidate(candidate_id) manifest, report = self._candidate_documents(candidate) self.assertEqual(manifest["id"], candidate_id) + self.assertFalse(manifest["productionEligible"]) self.assertGreater(report["evidence"]["meanRewardDelta"], 0) self.assertIn("calibrationErrorDelta", report["evidence"]) before = self._candidate_snapshot(candidate) @@ -69,6 +70,12 @@ def test_trains_immutable_candidate_then_requires_promotion_to_serve_it(self): self.assertEqual(response["evaluations"][0]["learningId"], RULE) self.assertEqual(self._candidate_snapshot(runtime.candidate(candidate_id)), before) + before_rollback = runtime.promotion() + with self.assertRaises(RuntimeFailure) as unavailable: + runtime.rollback(rolled_back_by="test-operator") + self.assertEqual(unavailable.exception.code, "rollback_unavailable") + self.assertEqual(runtime.promotion(), before_rollback) + def test_refuses_inference_until_a_candidate_is_explicitly_promoted(self): runtime = self._runtime() finished = self._wait_for_job( @@ -80,6 +87,25 @@ def test_refuses_inference_until_a_candidate_is_explicitly_promoted(self): with self.assertRaises(ValueError): runtime.evaluate(self._request()) + def test_detects_a_candidate_changed_to_claim_production_eligibility(self): + runtime = self._runtime() + finished = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=8, max_seconds=5)["id"], + ) + manifest_path = ( + self.state_dir / "candidates" / finished["candidateId"] / "manifest.json" + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["productionEligible"] = True + manifest_path.chmod(0o600) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with self.assertRaises(RuntimeFailure) as damaged: + runtime.candidate(finished["candidateId"]) + + self.assertEqual(damaged.exception.code, "candidate_damaged") + def test_rejects_invalid_limits_without_persisting_a_job(self): runtime = self._runtime() before = self._tree_snapshot(self.state_dir) @@ -99,6 +125,22 @@ def test_rejects_invalid_limits_without_persisting_a_job(self): self.assertEqual(self._tree_snapshot(self.state_dir), before) + def test_taskset_digest_changes_when_question_content_changes(self): + runtime = self._runtime() + before = runtime.taskset_digest + runtime.close() + tasks_path = self.taskset_dir / "tasks.jsonl" + tasks = [json.loads(line) for line in tasks_path.read_text(encoding="utf-8").splitlines()] + tasks[0]["questions"][RULE]["instructions"] = "Changed question" + tasks_path.write_text( + "".join(json.dumps(task) + "\n" for task in tasks), + encoding="utf-8", + ) + + reopened = self._runtime() + + self.assertNotEqual(reopened.taskset_digest, before) + def test_persists_backend_failure_without_creating_a_candidate(self): runtime = LocalRuntime( self.state_dir, @@ -127,26 +169,170 @@ def test_persists_backend_failure_without_creating_a_candidate(self): with self.assertRaises(ValueError): reopened.promote(submitted["id"], promoted_by="test-operator") - def test_reopening_state_preserves_job_candidate_and_promotion(self): + def test_restart_marks_an_interrupted_job_failed(self): + jobs = self.state_dir / "jobs" + jobs.mkdir(parents=True) + job_id = "job_0123456789abcdef0123456789abcdef" + (jobs / f"{job_id}.json").write_text( + json.dumps( + { + "schemaVersion": "replica.decision-training-job/v1", + "id": job_id, + "state": "running", + "backend": "fixture", + } + ), + encoding="utf-8", + ) + runtime = self._runtime() - finished = self._wait_for_job( + recovered = runtime.job(job_id) + + self.assertEqual(recovered["state"], "failed") + self.assertEqual(recovered["error"]["code"], "runtime_restarted") + self.assertIn("completedAt", recovered) + + def test_rollback_promotes_the_previous_candidate_as_a_new_generation(self): + runtime = self._runtime() + first = self._wait_for_job( runtime, runtime.submit(backend="fixture", max_updates=4, max_seconds=5)["id"], ) - candidate_id = finished["candidateId"] - runtime.promote(candidate_id, promoted_by="test-operator") - candidate_before = self._candidate_snapshot(runtime.candidate(candidate_id)) + second = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=8, max_seconds=5)["id"], + ) + first_candidate = first["candidateId"] + second_candidate = second["candidateId"] + self.assertNotEqual(first_candidate, second_candidate) + + first_promotion = runtime.promote(first_candidate, promoted_by="first-operator") + second_promotion = runtime.promote(second_candidate, promoted_by="second-operator") + rollback = runtime.rollback(rolled_back_by="rollback-operator") + + self.assertEqual(first_promotion["generation"], 1) + self.assertEqual(first_promotion["previousCandidateId"], None) + self.assertEqual(first_promotion["operation"], "promote") + self.assertEqual(second_promotion["generation"], 2) + self.assertEqual(second_promotion["previousCandidateId"], first_candidate) + self.assertEqual(rollback["generation"], 3) + self.assertEqual(rollback["candidateId"], first_candidate) + self.assertEqual(rollback["previousCandidateId"], second_candidate) + self.assertEqual(rollback["operation"], "rollback") + self.assertEqual(rollback["promotedBy"], "rollback-operator") + self.assertEqual( + [ + json.loads(path.read_text(encoding="utf-8")) + for path in sorted((self.state_dir / "promotions").iterdir()) + ], + [first_promotion, second_promotion, rollback], + ) + candidate_before = self._candidate_snapshot(runtime.candidate(first_candidate)) runtime.close() reopened = self._runtime() - self.assertEqual(reopened.job(finished["id"]), finished) + self.assertEqual(reopened.job(first["id"]), first) + self.assertEqual(reopened.job(second["id"]), second) self.assertEqual( - self._candidate_snapshot(reopened.candidate(candidate_id)), + self._candidate_snapshot(reopened.candidate(first_candidate)), candidate_before, ) - self.assertEqual(reopened.promotion()["candidateId"], candidate_id) - self.assertEqual(reopened.evaluate(self._request())["candidateId"], candidate_id) + self.assertEqual(reopened.promotion(), rollback) + response = reopened.evaluate(self._request()) + self.assertEqual(response["candidateId"], first_candidate) + self.assertEqual(response["promotionGeneration"], 3) + + def test_restart_recovers_a_promotion_generation_written_before_the_pointer(self): + runtime = self._runtime() + first = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=4, max_seconds=5)["id"], + ) + second = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=8, max_seconds=5)["id"], + ) + first_promotion = runtime.promote( + first["candidateId"], promoted_by="first-operator" + ) + second_promotion = runtime.promote( + second["candidateId"], promoted_by="second-operator" + ) + runtime.close() + (self.state_dir / "promotion.json").write_text( + json.dumps(first_promotion), + encoding="utf-8", + ) + + reopened = self._runtime() + + self.assertEqual(reopened.promotion(), second_promotion) + + def test_production_rejects_fixture_promotion_without_changing_the_current_generation(self): + runtime = self._runtime() + first = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=2, max_seconds=5)["id"], + ) + second = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=8, max_seconds=5)["id"], + ) + runtime.promote(first["candidateId"], promoted_by="test-operator") + before = runtime.promotion() + runtime.close() + + production = LocalRuntime( + self.state_dir, + self.taskset_dir, + backends={"fixture": FixtureBackend()}, + environment="production", + ) + self.addCleanup(production.close) + + with self.assertRaises(RuntimeFailure) as refused: + production.promote(second["candidateId"], promoted_by="production-operator") + + self.assertEqual(refused.exception.code, "candidate_not_production_eligible") + self.assertEqual(refused.exception.status, 409) + self.assertEqual(production.promotion(), before) + with self.assertRaises(RuntimeFailure) as not_ready: + production.ready() + self.assertEqual(not_ready.exception.code, "candidate_not_production_eligible") + with self.assertRaises(RuntimeFailure) as not_served: + production.evaluate(self._request()) + self.assertEqual(not_served.exception.code, "candidate_not_production_eligible") + + def test_production_rejects_rollback_to_a_fixture_candidate(self): + runtime = self._runtime() + first = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=2, max_seconds=5)["id"], + ) + second = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=8, max_seconds=5)["id"], + ) + runtime.promote(first["candidateId"], promoted_by="first-operator") + runtime.promote(second["candidateId"], promoted_by="second-operator") + before = runtime.promotion() + runtime.close() + + production = LocalRuntime( + self.state_dir, + self.taskset_dir, + backends={"fixture": FixtureBackend()}, + environment="production", + ) + self.addCleanup(production.close) + + with self.assertRaises(RuntimeFailure) as refused: + production.rollback(rolled_back_by="production-operator") + + self.assertEqual(refused.exception.code, "candidate_not_production_eligible") + self.assertEqual(refused.exception.status, 409) + self.assertEqual(production.promotion(), before) def test_http_routes_require_the_token_and_track_an_authorized_job(self): runtime = self._runtime() @@ -201,6 +387,59 @@ def test_http_routes_require_the_token_and_track_an_authorized_job(self): }, ) ) + second = self._http_json( + request.Request( + f"{base_url}/v1/jobs", + data=json.dumps( + {"backend": "fixture", "maxUpdates": 8, "maxSeconds": 5} + ).encode(), + headers={ + "Authorization": f"Bearer {TOKEN}", + "Content-Type": "application/json", + }, + ) + ) + second = self._wait_for_http_job(base_url, second["id"]) + self._http_json( + request.Request( + f"{base_url}/v1/promotions", + data=json.dumps( + { + "candidateId": second["candidateId"], + "promotedBy": "test-operator", + } + ).encode(), + headers={ + "Authorization": f"Bearer {TOKEN}", + "Content-Type": "application/json", + }, + ) + ) + rollback = self._http_json( + request.Request( + f"{base_url}/v1/rollbacks", + data=json.dumps({"rolledBackBy": "test-operator"}).encode(), + headers={ + "Authorization": f"Bearer {TOKEN}", + "Content-Type": "application/json", + }, + ) + ) + scored = self._http_json( + request.Request( + f"{base_url}/score", + data=json.dumps(self._request()).encode(), + headers={ + "Authorization": f"Bearer {TOKEN}", + "Content-Type": "application/json", + }, + ) + ) + self.assertEqual(rollback["generation"], 3) + self.assertEqual(rollback["candidateId"], finished["candidateId"]) + self.assertEqual(rollback["previousCandidateId"], second["candidateId"]) + self.assertEqual(scored["candidateId"], finished["candidateId"]) + self.assertEqual(scored["promotionGeneration"], 3) self.assertEqual(self._http_json(f"{base_url}/ready"), {"status": "ready"}) def _runtime(self): @@ -208,6 +447,7 @@ def _runtime(self): self.state_dir, self.taskset_dir, backends={"fixture": FixtureBackend()}, + environment="test", ) self.addCleanup(runtime.close) return runtime From 20d7466821492100ee9f36916f9935b3c18f89fe Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Tue, 17 Nov 2026 09:58:11 -0800 Subject: [PATCH 4/9] fix(learning): harden local runtime lifecycle Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- packages/learning/README.md | 2 +- .../environments/brand-check/README.md | 9 +- .../environments/brand-check/runtime.py | 191 +++++++++++++++--- .../brand-check/runtime_server.py | 39 +++- .../brand-check/tests/test_runtime.py | 188 ++++++++++++++++- 5 files changed, 397 insertions(+), 32 deletions(-) diff --git a/packages/learning/README.md b/packages/learning/README.md index 25e08aef0..5f307889d 100644 --- a/packages/learning/README.md +++ b/packages/learning/README.md @@ -216,7 +216,7 @@ A task shows every screenshot of its page, desktop before narrow and light befor Hub names each screenshot by the address the build action was asked to capture. The build action follows redirects and Controls URL defaults without recording where the page ended up, so a comment made at the address a page redirects to finds no screenshot. -The [local decision-model runtime](environments/brand-check/README.md#run-the-local-training-and-inference-runtime) accepts that taskset, tracks a bounded deterministic fixture job, stores an immutable candidate and proper-scoring evaluation report, requires explicit promotion or rollback, and serves the promoted result through the same request shape as `decision ask`. The fixture proves the local lifecycle without a GPU or model download, but its candidates are development/test-only and cannot be promoted in production. The runtime keeps a small backend interface for later real-model work. +The [local decision-model runtime](environments/brand-check/README.md#run-the-local-training-and-inference-runtime) accepts that taskset, tracks a bounded deterministic fixture job, stores an immutable candidate and proper-scoring evaluation report, requires explicit promotion or rollback, and serves the promoted result through an authenticated scoring endpoint. The fixture proves the local lifecycle without a GPU or model download, but its candidates are development/test-only and cannot be promoted in production. The runtime keeps a small backend interface for later real-model work. The output directory must be new and, inside a checkout, under `.replica/`. It holds: diff --git a/packages/learning/environments/brand-check/README.md b/packages/learning/environments/brand-check/README.md index 5fdd76d29..cb418ad55 100644 --- a/packages/learning/environments/brand-check/README.md +++ b/packages/learning/environments/brand-check/README.md @@ -206,7 +206,14 @@ curl -sS -H "Authorization: Bearer $REPLICA_RUNTIME_TOKEN" \ http://127.0.0.1:8010/v1/rollbacks ``` -`POST /score` and `POST /v1/evaluations` accept the request that `replica-learning decision ask` already sends. Set `REPLICA_DECISION_ENDPOINT=http://127.0.0.1:8010/score` and set `REPLICA_DECISION_KEY` to the runtime token to exercise that endpoint contract. The fixture ignores the screenshots, so do not use its answers to judge a Build or guide authoring. The response keeps its `answers` object and adds the promoted candidate ID, promotion generation, and one `followsProbability` and `breaksProbability` pair per learning. Inference returns `409 candidate_not_promoted` until an operator promotes a candidate. +`POST /score` and `POST /v1/evaluations` accept a task-shaped request with `state`, image data URIs, and `questions`. Send the runtime token as a bearer token. The fixture ignores the screenshots, so do not use its answers to judge a Build or guide authoring. The response includes probability `answers`, the promoted candidate ID, promotion generation, and one `followsProbability` and `breaksProbability` pair per learning. Inference returns `409 candidate_not_promoted` until an operator promotes a candidate. + +```sh +curl -sS -H "Authorization: Bearer $REPLICA_RUNTIME_TOKEN" \ + -H "Content-Type: application/json" \ + --data-binary @request.json \ + http://127.0.0.1:8010/score +``` The HTTP contract stays independent of the local store and trainer implementation: diff --git a/packages/learning/environments/brand-check/runtime.py b/packages/learning/environments/brand-check/runtime.py index 74560ca83..9a36aff1e 100644 --- a/packages/learning/environments/brand-check/runtime.py +++ b/packages/learning/environments/brand-check/runtime.py @@ -11,8 +11,11 @@ import json import math import os +import queue +import re import shutil import threading +import time import uuid from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass @@ -23,6 +26,11 @@ from .evaluate import EvaluationError, evaluate from .taskset import Taskset, load_taskset, question_options +try: + import fcntl +except ImportError: # pragma: no cover - the reference runtime supports Unix hosts and containers. + fcntl = None + JOB_SCHEMA = "replica.decision-training-job/v1" CANDIDATE_SCHEMA = "replica.decision-candidate/v1" CANDIDATE_REPORT_SCHEMA = "replica.decision-candidate-evaluation/v1" @@ -33,6 +41,7 @@ MIN_SECONDS = 1 MAX_SECONDS = 86_400 ENVIRONMENTS = ("development", "test", "production") +_QUESTION_KEY = re.compile(r"^[A-Za-z0-9_]{1,64}$") class RuntimeFailure(ValueError): @@ -67,7 +76,7 @@ def train( work_directory: Path, *, max_updates: int, - max_seconds: float, + deadline: float, ) -> TrainingResult: ... def predict( @@ -87,11 +96,12 @@ def train( work_directory: Path, *, max_updates: int, - max_seconds: float, + deadline: float, ) -> TrainingResult: - del work_directory, max_seconds + del work_directory counts: dict[str, list[int]] = {} for task in taskset.by_split["train"]: + _check_deadline(deadline) for key, label in task.labels.items(): if task.questions[key].type != "noul": continue @@ -116,8 +126,11 @@ def train( "updates": max_updates, "probabilities": probabilities, } + _check_deadline(deadline) baseline = _evaluate_fixture(taskset, {}) + _check_deadline(deadline) candidate = _evaluate_fixture(taskset, probabilities) + _check_deadline(deadline) return TrainingResult( adapter_name="adapter.json", adapter=_json_bytes(adapter), @@ -160,23 +173,49 @@ def __init__( self.taskset = load_taskset(taskset_directory) self.taskset_digest = _taskset_digest(self.taskset) self.environment = environment - self.backends = dict(backends or {"fixture": FixtureBackend()}) + self.backends = dict({"fixture": FixtureBackend()} if backends is None else backends) if not self.backends: raise RuntimeFailure("backend_missing", "Configure at least one training backend.") + if fcntl is None: + raise RuntimeFailure( + "runtime_unsupported", "The local runtime needs Unix file locking.", 500 + ) self._lock = threading.RLock() - self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="replica-training") self.state_directory.mkdir(parents=True, exist_ok=True, mode=0o700) os.chmod(self.state_directory, 0o700) - for name in ("jobs", "candidates", "promotions", "work"): - directory = self.state_directory / name - directory.mkdir(parents=True, exist_ok=True, mode=0o700) - os.chmod(directory, 0o700) - self._recover_promotion() - self._recover_interrupted_jobs() + self._owner = (self.state_directory / ".runtime.lock").open("a+b") + try: + fcntl.flock(self._owner.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + self._owner.close() + raise RuntimeFailure( + "runtime_in_use", + f"Another runtime owns {self.state_directory}.", + 409, + ) from None + try: + self._executor = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="replica-training" + ) + for name in ("jobs", "candidates", "promotions", "work"): + directory = self.state_directory / name + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(directory, 0o700) + self._recover_promotion() + self._recover_interrupted_jobs() + except Exception: + if hasattr(self, "_executor"): + self._executor.shutdown(wait=False, cancel_futures=True) + fcntl.flock(self._owner.fileno(), fcntl.LOCK_UN) + self._owner.close() + raise def close(self) -> None: """Wait for the current job and stop the worker.""" - self._executor.shutdown(wait=True) + self._executor.shutdown(wait=True, cancel_futures=True) + if not self._owner.closed: + fcntl.flock(self._owner.fileno(), fcntl.LOCK_UN) + self._owner.close() def submit( self, *, backend: str = "fixture", max_updates: int, max_seconds: float @@ -260,6 +299,8 @@ def promote(self, candidate_id: str, *, promoted_by: str) -> dict[str, Any]: ) with self._lock: current = self.promotion(required=False) + if current and current["candidateId"] == candidate_id: + return current promotion = self._promotion( candidate_id, previous_candidate_id=current["candidateId"] if current else None, @@ -375,14 +416,15 @@ def evaluate( def _run_job(self, job_id: str) -> None: job = self.job(job_id) - job.update(state="running", startedAt=_now(), updatedAt=_now()) - _write_json_atomic(self._job_path(job_id), job) work = self.state_directory / "work" / job_id - shutil.rmtree(work, ignore_errors=True) - work.mkdir(mode=0o700) try: + job.update(state="running", startedAt=_now(), updatedAt=_now()) + _write_json_atomic(self._job_path(job_id), job) + shutil.rmtree(work, ignore_errors=True) + work.mkdir(mode=0o700) backend = self.backends[job["backend"]] - result = backend.train( + result = _run_backend( + backend, self.taskset, work, max_updates=job["limits"]["maxUpdates"], @@ -535,6 +577,10 @@ def _job_path(self, job_id: str) -> Path: def _promotable_candidate(self, candidate_id: str) -> dict[str, Any]: candidate = self.candidate(candidate_id)["manifest"] + if candidate["tasksetDigest"] != self.taskset_digest: + raise RuntimeFailure( + "candidate_mismatch", "The candidate belongs to another taskset.", 409 + ) backend = self.backends.get(candidate["backend"]) if backend is None: raise RuntimeFailure( @@ -737,10 +783,59 @@ def _check_request(request: Mapping[str, Any]) -> None: "invalid_input", "The evaluation request images must be image data URIs." ) questions = request.get("questions") - if not isinstance(questions, Mapping) or not any( - isinstance(question, Mapping) and question.get("type") == "noul" - for question in questions.values() - ): + if not isinstance(questions, Mapping) or not questions: + raise RuntimeFailure( + "invalid_input", "The evaluation request questions must be a non-empty object." + ) + has_noul = False + for key, question in questions.items(): + if not isinstance(key, str) or not _QUESTION_KEY.fullmatch(key): + raise RuntimeFailure("invalid_input", "Each question needs a valid key.") + if not isinstance(question, Mapping): + raise RuntimeFailure("invalid_input", f"Question {key} must be an object.") + unknown = set(question) - {"type", "instructions", "criteria", "learning_text"} + if unknown: + raise RuntimeFailure("invalid_input", f"Question {key} has unknown fields.") + kind = question.get("type") + instructions = question.get("instructions") + criteria = question.get("criteria") + if ( + kind not in ("noul", "choice", "score") + or not isinstance(instructions, str) + or not instructions.strip() + ): + raise RuntimeFailure("invalid_input", f"Question {key} is not valid.") + if "learning_text" in question and ( + kind != "noul" + or not isinstance(question["learning_text"], str) + or not question["learning_text"].strip() + ): + raise RuntimeFailure("invalid_input", f"Question {key} has invalid learning text.") + if kind == "noul": + has_noul = True + if "criteria" in question and not ( + isinstance(criteria, Mapping) + and set(criteria) <= {"true", "false"} + and all(isinstance(text, str) for text in criteria.values()) + ): + raise RuntimeFailure("invalid_input", f"Question {key} has invalid criteria.") + elif kind == "choice": + if not isinstance(criteria, Mapping) or not 2 <= len(criteria) <= 255: + raise RuntimeFailure("invalid_input", f"Question {key} needs 2 to 255 options.") + if any( + not isinstance(option, str) + or not _QUESTION_KEY.fullmatch(option) + or not isinstance(text, str) + for option, text in criteria.items() + ): + raise RuntimeFailure("invalid_input", f"Question {key} has invalid options.") + elif not ( + isinstance(criteria, list) + and 2 <= len(criteria) <= 10 + and all(isinstance(level, str) for level in criteria) + ): + raise RuntimeFailure("invalid_input", f"Question {key} needs 2 to 10 score levels.") + if not has_noul: raise RuntimeFailure( "invalid_input", "The evaluation request needs at least one yes-or-no question." ) @@ -796,10 +891,58 @@ def _write_json_atomic(path: Path, value: Any) -> None: def _write_immutable(path: Path, data: bytes) -> None: - descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o400) - with os.fdopen(descriptor, "wb") as handle: - handle.write(data) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}") + try: + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o400) + with os.fdopen(descriptor, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.link(temporary, path) + finally: + temporary.unlink(missing_ok=True) def _now() -> str: return datetime.now(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def _check_deadline(deadline: float) -> None: + if time.monotonic() > deadline: + raise RuntimeFailure("job_timed_out", "The bounded training job reached maxSeconds.") + + +def _run_backend( + backend: TrainingBackend, + taskset: Taskset, + work_directory: Path, + *, + max_updates: int, + max_seconds: float, +) -> TrainingResult: + """Run trusted backend code without letting it block the runtime past the job deadline.""" + completed: queue.Queue[TrainingResult | Exception] = queue.Queue(maxsize=1) + deadline = time.monotonic() + max_seconds + + def run() -> None: + try: + result: TrainingResult | Exception = backend.train( + taskset, + work_directory, + max_updates=max_updates, + deadline=deadline, + ) + except Exception as error: + result = error + completed.put(result) + + thread = threading.Thread(target=run, name=f"replica-{backend.name}-job", daemon=True) + thread.start() + try: + result = completed.get(timeout=max_seconds) + except queue.Empty: + raise RuntimeFailure("job_timed_out", "The bounded training job reached maxSeconds.") from None + if isinstance(result, Exception): + raise result + _check_deadline(deadline) + return result diff --git a/packages/learning/environments/brand-check/runtime_server.py b/packages/learning/environments/brand-check/runtime_server.py index 96f584f94..7b5003a1b 100644 --- a/packages/learning/environments/brand-check/runtime_server.py +++ b/packages/learning/environments/brand-check/runtime_server.py @@ -7,6 +7,9 @@ import json import os import re +import signal +import socket +import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any @@ -17,9 +20,15 @@ TOKEN_VARIABLE = "REPLICA_RUNTIME_TOKEN" MIN_TOKEN_LENGTH = 32 MAX_REQUEST_BYTES = 180 * 1024 * 1024 +MAX_CONNECTIONS = 16 +SOCKET_TIMEOUT_SECONDS = 10 _SAFE_TOKEN = re.compile(r"^[\x21-\x7e]+$") +def _stop(_signum: int, _frame: Any) -> None: + raise KeyboardInterrupt + + def runtime_token(environ: dict[str, str]) -> str: """Return a strong runtime token without ever including it in an error.""" token = environ.get(TOKEN_VARIABLE, "") @@ -38,16 +47,40 @@ class RuntimeHTTPServer(ThreadingHTTPServer): daemon_threads = True def __init__(self, address: tuple[str, int], runtime: LocalRuntime, token: str): + if len(token) < MIN_TOKEN_LENGTH or not _SAFE_TOKEN.fullmatch(token): + raise RuntimeFailure("token_missing", "The runtime token is not valid.") super().__init__(address, RuntimeHandler) self.runtime = runtime self.token = token.encode() + self._connections = threading.BoundedSemaphore(MAX_CONNECTIONS) + + def get_request(self) -> tuple[socket.socket, Any]: + connection, address = super().get_request() + connection.settimeout(SOCKET_TIMEOUT_SECONDS) + return connection, address + + def process_request(self, request: socket.socket, client_address: Any) -> None: + if not self._connections.acquire(blocking=False): + request.close() + return + try: + super().process_request(request, client_address) + except Exception: + self._connections.release() + raise + + def process_request_thread(self, request: socket.socket, client_address: Any) -> None: + try: + super().process_request_thread(request, client_address) + finally: + self._connections.release() class RuntimeHandler(BaseHTTPRequestHandler): """Route runtime requests and return JSON errors with stable codes.""" server: RuntimeHTTPServer - protocol_version = "HTTP/1.1" + protocol_version = "HTTP/1.0" def do_GET(self) -> None: try: @@ -158,6 +191,7 @@ def _body(self) -> dict[str, Any]: return value def _failure(self, error: RuntimeFailure) -> None: + self.close_connection = True headers = {"WWW-Authenticate": "Bearer"} if error.status == 401 else None self._send(error.status, {"error": {"code": error.code, "message": str(error)}}, headers) @@ -197,6 +231,8 @@ def main() -> None: environment=args.environment, ) server = RuntimeHTTPServer((args.host, args.port), runtime, token) + previous = signal.getsignal(signal.SIGTERM) + signal.signal(signal.SIGTERM, _stop) try: server.serve_forever() except KeyboardInterrupt: @@ -204,6 +240,7 @@ def main() -> None: finally: server.server_close() runtime.close() + signal.signal(signal.SIGTERM, previous) if __name__ == "__main__": diff --git a/packages/learning/environments/brand-check/tests/test_runtime.py b/packages/learning/environments/brand-check/tests/test_runtime.py index c6e0c124a..fac3e4124 100644 --- a/packages/learning/environments/brand-check/tests/test_runtime.py +++ b/packages/learning/environments/brand-check/tests/test_runtime.py @@ -4,15 +4,17 @@ import base64 import json +import shutil import threading import time import unittest from collections.abc import Mapping from pathlib import Path +from unittest import mock from urllib import error, request from replica_brand_check.runtime import FixtureBackend, LocalRuntime, RuntimeFailure -from replica_brand_check.runtime_server import RuntimeHTTPServer +from replica_brand_check.runtime_server import RuntimeHTTPServer, runtime_token from tests.support import PNG, RULE, make_task, write_taskset TOKEN = "runtime-test-token-with-32-characters" @@ -25,6 +27,27 @@ def train(self, *args, **kwargs): raise RuntimeError("deliberate backend failure") +class SlowBackend(FixtureBackend): + """A backend that ignores its deadline, to prove the runtime still bounds the job.""" + + def train(self, *args, **kwargs): + time.sleep(2) + return super().train(*args, **kwargs) + + +class BlockingBackend(FixtureBackend): + """A backend whose first job waits so a second job stays queued during close.""" + + def __init__(self): + self.started = threading.Event() + self.release = threading.Event() + + def train(self, *args, **kwargs): + self.started.set() + self.release.wait(5) + return super().train(*args, **kwargs) + + class LocalRuntimeTest(unittest.TestCase): def setUp(self): self.taskset_dir = write_taskset( @@ -125,6 +148,67 @@ def test_rejects_invalid_limits_without_persisting_a_job(self): self.assertEqual(self._tree_snapshot(self.state_dir), before) + def test_refuses_an_empty_backend_configuration(self): + with self.assertRaises(RuntimeFailure) as missing: + LocalRuntime(self.state_dir, self.taskset_dir, backends={}) + + self.assertEqual(missing.exception.code, "backend_missing") + + def test_refuses_a_second_runtime_without_changing_the_first_jobs(self): + runtime = self._runtime() + submitted = runtime.submit(backend="fixture", max_updates=8, max_seconds=5) + + with self.assertRaises(RuntimeFailure) as in_use: + LocalRuntime(self.state_dir, self.taskset_dir) + + self.assertEqual(in_use.exception.code, "runtime_in_use") + self.assertNotEqual(runtime.job(submitted["id"])["state"], "failed") + + def test_marks_a_backend_that_exceeds_the_wall_time_failed(self): + runtime = LocalRuntime( + self.state_dir, + self.taskset_dir, + backends={"slow": SlowBackend()}, + environment="test", + ) + self.addCleanup(runtime.close) + + submitted = runtime.submit(backend="slow", max_updates=1, max_seconds=1) + failed = self._wait_for_job(runtime, submitted["id"]) + + self.assertEqual(failed["state"], "failed") + self.assertEqual(failed["error"]["code"], "job_timed_out") + self.assertEqual(list((self.state_dir / "candidates").iterdir()), []) + + def test_close_cancels_queued_jobs_instead_of_starting_them(self): + backend = BlockingBackend() + runtime = LocalRuntime( + self.state_dir, + self.taskset_dir, + backends={"blocking": backend}, + environment="test", + ) + first = runtime.submit(backend="blocking", max_updates=1, max_seconds=5) + self.assertTrue(backend.started.wait(1)) + second = runtime.submit(backend="blocking", max_updates=1, max_seconds=5) + closing = threading.Thread(target=runtime.close) + closing.start() + time.sleep(0.05) + backend.release.set() + closing.join(2) + self.assertFalse(closing.is_alive()) + self.assertEqual(runtime.job(first["id"])["state"], "succeeded") + self.assertEqual(runtime.job(second["id"])["state"], "queued") + + reopened = LocalRuntime( + self.state_dir, + self.taskset_dir, + backends={"blocking": backend}, + environment="test", + ) + self.addCleanup(reopened.close) + self.assertEqual(reopened.job(second["id"])["error"]["code"], "runtime_restarted") + def test_taskset_digest_changes_when_question_content_changes(self): runtime = self._runtime() before = runtime.taskset_digest @@ -164,10 +248,24 @@ def test_persists_backend_failure_without_creating_a_candidate(self): ) self.addCleanup(reopened.close) self.assertEqual(reopened.job(submitted["id"])["state"], "failed") - with self.assertRaises(ValueError): - reopened.candidate(submitted["id"]) - with self.assertRaises(ValueError): - reopened.promote(submitted["id"], promoted_by="test-operator") + self.assertEqual(list((self.state_dir / "candidates").iterdir()), []) + self.assertIsNone(reopened.promotion(required=False)) + + def test_persists_work_directory_setup_failure(self): + runtime = self._runtime() + original = shutil.rmtree + + def fail_work(path, *args, **kwargs): + if Path(path).parent.name == "work" and Path(path).name.startswith("job_"): + raise OSError("work directory unavailable") + return original(path, *args, **kwargs) + + with mock.patch("replica_brand_check.runtime.shutil.rmtree", side_effect=fail_work): + submitted = runtime.submit(backend="fixture", max_updates=1, max_seconds=5) + failed = self._wait_for_job(runtime, submitted["id"]) + + self.assertEqual(failed["state"], "failed") + self.assertEqual(failed["error"]["code"], "backend_failed") def test_restart_marks_an_interrupted_job_failed(self): jobs = self.state_dir / "jobs" @@ -208,6 +306,7 @@ def test_rollback_promotes_the_previous_candidate_as_a_new_generation(self): first_promotion = runtime.promote(first_candidate, promoted_by="first-operator") second_promotion = runtime.promote(second_candidate, promoted_by="second-operator") + repeated = runtime.promote(second_candidate, promoted_by="retrying-operator") rollback = runtime.rollback(rolled_back_by="rollback-operator") self.assertEqual(first_promotion["generation"], 1) @@ -215,6 +314,7 @@ def test_rollback_promotes_the_previous_candidate_as_a_new_generation(self): self.assertEqual(first_promotion["operation"], "promote") self.assertEqual(second_promotion["generation"], 2) self.assertEqual(second_promotion["previousCandidateId"], first_candidate) + self.assertEqual(repeated, second_promotion) self.assertEqual(rollback["generation"], 3) self.assertEqual(rollback["candidateId"], first_candidate) self.assertEqual(rollback["previousCandidateId"], second_candidate) @@ -334,6 +434,39 @@ def test_production_rejects_rollback_to_a_fixture_candidate(self): self.assertEqual(refused.exception.status, 409) self.assertEqual(production.promotion(), before) + def test_reopened_runtime_refuses_a_promoted_candidate_from_another_taskset(self): + runtime = self._runtime() + first = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=4, max_seconds=5)["id"], + ) + second = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=8, max_seconds=5)["id"], + ) + runtime.promote(first["candidateId"], promoted_by="test-operator") + runtime.promote(second["candidateId"], promoted_by="test-operator") + runtime.close() + tasks_path = self.taskset_dir / "tasks.jsonl" + tasks = [json.loads(line) for line in tasks_path.read_text(encoding="utf-8").splitlines()] + tasks[0]["questions"][RULE]["instructions"] = "A different taskset" + tasks_path.write_text( + "".join(json.dumps(task) + "\n" for task in tasks), + encoding="utf-8", + ) + reopened = self._runtime() + + with self.assertRaises(RuntimeFailure) as not_ready: + reopened.ready() + with self.assertRaises(RuntimeFailure) as not_served: + reopened.evaluate(self._request()) + with self.assertRaises(RuntimeFailure) as not_rolled_back: + reopened.rollback(rolled_back_by="test-operator") + + self.assertEqual(not_ready.exception.code, "candidate_mismatch") + self.assertEqual(not_served.exception.code, "candidate_mismatch") + self.assertEqual(not_rolled_back.exception.code, "candidate_mismatch") + def test_http_routes_require_the_token_and_track_an_authorized_job(self): runtime = self._runtime() server = RuntimeHTTPServer(("127.0.0.1", 0), runtime, TOKEN) @@ -442,6 +575,38 @@ def test_http_routes_require_the_token_and_track_an_authorized_job(self): self.assertEqual(scored["promotionGeneration"], 3) self.assertEqual(self._http_json(f"{base_url}/ready"), {"status": "ready"}) + def test_refuses_malformed_questions_before_calling_the_backend(self): + runtime = self._runtime() + finished = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=8, max_seconds=5)["id"], + ) + runtime.promote(finished["candidateId"], promoted_by="test-operator") + malformed = self._request() + malformed["questions"]["severity"] = { + "type": "score", + "instructions": "How severe is it?", + "criteria": "x" * 10_000, + } + + with self.assertRaises(RuntimeFailure) as invalid: + runtime.evaluate(malformed) + + self.assertEqual(invalid.exception.code, "invalid_input") + + def test_immutable_write_failure_does_not_publish_a_partial_generation(self): + runtime = self._runtime() + finished = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=8, max_seconds=5)["id"], + ) + with mock.patch("replica_brand_check.runtime.os.link", side_effect=OSError("stopped")): + with self.assertRaises(OSError): + runtime.promote(finished["candidateId"], promoted_by="test-operator") + + self.assertEqual(list((self.state_dir / "promotions").iterdir()), []) + self.assertIsNone(runtime.promotion(required=False)) + def _runtime(self): runtime = LocalRuntime( self.state_dir, @@ -535,5 +700,18 @@ def _remove_state(self): self.state_dir.rmdir() +class RuntimeTokenTest(unittest.TestCase): + def test_requires_a_strong_printable_token_without_revealing_it(self): + for token in ("", "short", "x" * 31, "x" * 31 + " "): + with self.subTest(token=token): + with self.assertRaises(RuntimeFailure) as refused: + runtime_token({"REPLICA_RUNTIME_TOKEN": token}) + if token: + self.assertNotIn(token, str(refused.exception)) + + token = "runtime-token-with-at-least-32-characters" + self.assertEqual(runtime_token({"REPLICA_RUNTIME_TOKEN": token}), token) + + if __name__ == "__main__": unittest.main() From 82a6568cc3a2fefb3066380d6471d8d87df58c49 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Tue, 17 Nov 2026 13:30:17 -0800 Subject: [PATCH 5/9] fix(learning): isolate fixture backend input Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../environments/brand-check/runtime.py | 48 ++++++++++++++++--- .../brand-check/tests/test_runtime.py | 37 ++++++++++++++ 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/packages/learning/environments/brand-check/runtime.py b/packages/learning/environments/brand-check/runtime.py index 9a36aff1e..33c14ee8c 100644 --- a/packages/learning/environments/brand-check/runtime.py +++ b/packages/learning/environments/brand-check/runtime.py @@ -24,7 +24,7 @@ from typing import Any, Mapping, Protocol from .evaluate import EvaluationError, evaluate -from .taskset import Taskset, load_taskset, question_options +from .taskset import Taskset, load_taskset try: import fcntl @@ -384,7 +384,13 @@ def evaluate( 503, ) adapter = self.state_directory / "candidates" / candidate_id / manifest["adapter"]["file"] - answers = dict(backend.predict(adapter, request, timeout_seconds=timeout_seconds)) + answers = dict( + backend.predict( + adapter, + _backend_request(request), + timeout_seconds=timeout_seconds, + ) + ) evaluations = [] for key, question in request["questions"].items(): if question.get("type") != "noul": @@ -699,14 +705,18 @@ def _fixture_answers( ) -> dict[str, Any]: answers = {} for key, question in questions.items(): - options = question_options(question) if question["type"] == "noul": yes = float(probabilities.get(key, 0.5)) answers[key] = {"probabilities": {"yes": yes, "no": 1.0 - yes}} else: - answers[key] = { - "probabilities": {option: 1.0 / len(options) for option in options} - } + if question["type"] == "choice": + criteria = question["criteria"] + options = tuple(criteria if isinstance(criteria, list) else criteria.keys()) + else: + criteria = question["criteria"] + levels = criteria if isinstance(criteria, int) else len(criteria) + options = tuple(str(level) for level in range(levels)) + answers[key] = {"probabilities": {option: 1.0 / len(options) for option in options}} return answers @@ -859,6 +869,32 @@ def _valid_yes_no(value: Any) -> bool: ) +def _backend_request(request: Mapping[str, Any]) -> dict[str, Any]: + """Project validated input to the data-only contract the local fixture backend needs. + + Free-form state, instructions, learning text, and option descriptions remain outside backend + execution. A later model backend must define and secure its own text-bearing input contract. + """ + return { + "images": list(request["images"]), + "questions": { + key: ( + {"type": "noul"} + if question["type"] == "noul" + else { + "type": question["type"], + "criteria": ( + list(question["criteria"]) + if question["type"] == "choice" + else len(question["criteria"]) + ), + } + ) + for key, question in request["questions"].items() + }, + } + + def _checked_id(value: str, prefix: str) -> str: if ( not isinstance(value, str) diff --git a/packages/learning/environments/brand-check/tests/test_runtime.py b/packages/learning/environments/brand-check/tests/test_runtime.py index fac3e4124..2befc7972 100644 --- a/packages/learning/environments/brand-check/tests/test_runtime.py +++ b/packages/learning/environments/brand-check/tests/test_runtime.py @@ -48,6 +48,17 @@ def train(self, *args, **kwargs): return super().train(*args, **kwargs) +class CapturingBackend(FixtureBackend): + """A fixture backend that records the projected inference request.""" + + def __init__(self): + self.request = None + + def predict(self, adapter, request, *, timeout_seconds): + self.request = request + return super().predict(adapter, request, timeout_seconds=timeout_seconds) + + class LocalRuntimeTest(unittest.TestCase): def setUp(self): self.taskset_dir = write_taskset( @@ -110,6 +121,32 @@ def test_refuses_inference_until_a_candidate_is_explicitly_promoted(self): with self.assertRaises(ValueError): runtime.evaluate(self._request()) + def test_backend_receives_no_free_form_task_or_question_text(self): + backend = CapturingBackend() + runtime = LocalRuntime( + self.state_dir, + self.taskset_dir, + backends={"fixture": backend}, + environment="test", + ) + self.addCleanup(runtime.close) + finished = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=8, max_seconds=5)["id"], + ) + runtime.promote(finished["candidateId"], promoted_by="test-operator") + runtime.evaluate(self._request()) + + self.assertEqual( + backend.request, + { + "images": [ + "data:image/png;base64," + base64.b64encode(PNG).decode("ascii") + ], + "questions": {RULE: {"type": "noul"}}, + }, + ) + def test_detects_a_candidate_changed_to_claim_production_eligibility(self): runtime = self._runtime() finished = self._wait_for_job( From 3c8131f44aa0c0918fd240c75713aaee26587026 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Tue, 17 Nov 2026 15:56:40 -0800 Subject: [PATCH 6/9] fix(learning): close runtime lifecycle races Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../environments/brand-check/README.md | 2 +- .../environments/brand-check/runtime.py | 107 ++++++++++++++---- .../brand-check/runtime_server.py | 26 ++++- .../brand-check/tests/test_runtime.py | 82 +++++++++++++- 4 files changed, 189 insertions(+), 28 deletions(-) diff --git a/packages/learning/environments/brand-check/README.md b/packages/learning/environments/brand-check/README.md index cb418ad55..be2043680 100644 --- a/packages/learning/environments/brand-check/README.md +++ b/packages/learning/environments/brand-check/README.md @@ -248,7 +248,7 @@ The image contains no taskset, candidate, model, or credential. It runs as user ### Add another backend later -`TrainingBackend` is the extension seam. A backend declares its name and whether its candidates may be promoted in production, implements one bounded `train` call that returns an adapter plus baseline and candidate reports, and implements `predict` for that adapter. The runtime owns job state, candidate integrity, promotion, rollback, and HTTP authentication. This first reference does not connect the existing d1 trainer to that interface. +`TrainingBackend` is the extension seam. A backend can run in a separate terminable process, declares its name and whether its candidates may be promoted in production, implements one bounded `train` call that returns an adapter plus baseline and candidate reports, and implements `predict` for that adapter. Installed runtimes use a spawned worker process, so a backend and its configuration must be serializable. The runtime owns job state, candidate integrity, promotion, rollback, and HTTP authentication. This first reference does not connect the existing d1 trainer to that interface. ### Trust, data, and credential boundaries diff --git a/packages/learning/environments/brand-check/runtime.py b/packages/learning/environments/brand-check/runtime.py index 33c14ee8c..1158766c4 100644 --- a/packages/learning/environments/brand-check/runtime.py +++ b/packages/learning/environments/brand-check/runtime.py @@ -8,10 +8,11 @@ from __future__ import annotations import hashlib +import importlib.util import json import math +import multiprocessing import os -import queue import re import shutil import threading @@ -222,6 +223,8 @@ def submit( ) -> dict[str, Any]: """Persist and enqueue one bounded job, returning its initial state.""" _check_limits(max_updates, max_seconds) + if not isinstance(backend, str): + raise RuntimeFailure("invalid_input", "backend must be a string.") if backend not in self.backends: raise RuntimeFailure( "backend_unknown", @@ -298,6 +301,7 @@ def promote(self, candidate_id: str, *, promoted_by: str) -> dict[str, Any]: "candidate_mismatch", "The candidate belongs to another taskset.", 409 ) with self._lock: + self._recover_promotion() current = self.promotion(required=False) if current and current["candidateId"] == candidate_id: return current @@ -316,6 +320,7 @@ def rollback(self, *, rolled_back_by: str) -> dict[str, Any]: if not isinstance(rolled_back_by, str) or not rolled_back_by.strip(): raise RuntimeFailure("invalid_input", "rolledBackBy must name the operator.") with self._lock: + self._recover_promotion() current = self.promotion() candidate_id = current.get("previousCandidateId") if not isinstance(candidate_id, str): @@ -668,7 +673,7 @@ def _candidate_evaluation( "meanRewardDelta": candidate_reward - base_reward, "improvesMeanReward": candidate_reward > base_reward, "calibrationErrorDelta": ece_delta, - "calibrationNotWorse": ece_delta is None or ece_delta <= 0, + "calibrationNotWorse": None if ece_delta is None else ece_delta <= 0, }, } @@ -956,29 +961,83 @@ def _run_backend( max_updates: int, max_seconds: float, ) -> TrainingResult: - """Run trusted backend code without letting it block the runtime past the job deadline.""" - completed: queue.Queue[TrainingResult | Exception] = queue.Queue(maxsize=1) + """Run backend code in a child that the runtime can stop at the job deadline.""" deadline = time.monotonic() + max_seconds - - def run() -> None: - try: - result: TrainingResult | Exception = backend.train( - taskset, - work_directory, - max_updates=max_updates, - deadline=deadline, + context = _process_context() + receive, send = context.Pipe(duplex=False) + process = context.Process( + target=_backend_process, + args=( + send, + backend, + str(taskset.directory), + str(work_directory), + max_updates, + deadline, + ), + name=f"replica-{backend.name}-job", + ) + process.start() + send.close() + try: + if not receive.poll(max_seconds): + process.terminate() + process.join() + raise RuntimeFailure( + "job_timed_out", "The bounded training job reached maxSeconds." ) - except Exception as error: - result = error - completed.put(result) + kind, payload = receive.recv() + process.join() + if kind == "error": + code, message, status = payload + raise RuntimeFailure(code, message, status) + _check_deadline(deadline) + return payload + finally: + receive.close() + if process.is_alive(): + process.terminate() + process.join() - thread = threading.Thread(target=run, name=f"replica-{backend.name}-job", daemon=True) - thread.start() + +def _backend_process( + send: Any, + backend: TrainingBackend, + taskset_directory: str, + work_directory: str, + max_updates: int, + deadline: float, +) -> None: + """Load task data afresh and run one serializable backend in a spawned child process.""" try: - result = completed.get(timeout=max_seconds) - except queue.Empty: - raise RuntimeFailure("job_timed_out", "The bounded training job reached maxSeconds.") from None - if isinstance(result, Exception): - raise result - _check_deadline(deadline) - return result + result = backend.train( + load_taskset(taskset_directory), + Path(work_directory), + max_updates=max_updates, + deadline=deadline, + ) + send.send(("ok", result)) + except Exception as error: + send.send( + ( + "error", + ( + error.code if isinstance(error, RuntimeFailure) else "backend_failed", + str(error), + error.status if isinstance(error, RuntimeFailure) else 500, + ), + ) + ) + finally: + send.close() + + +def _process_context() -> Any: + """Use spawn when installed; source-only tests bootstrap an alias that spawn cannot import.""" + spec = importlib.util.find_spec("replica_brand_check") + installed = ( + spec is not None + and spec.origin is not None + and Path(spec.origin).parent.name == "replica_brand_check" + ) + return multiprocessing.get_context("spawn" if installed else "fork") diff --git a/packages/learning/environments/brand-check/runtime_server.py b/packages/learning/environments/brand-check/runtime_server.py index 7b5003a1b..5b95f85df 100644 --- a/packages/learning/environments/brand-check/runtime_server.py +++ b/packages/learning/environments/brand-check/runtime_server.py @@ -44,7 +44,7 @@ def runtime_token(environ: dict[str, str]) -> str: class RuntimeHTTPServer(ThreadingHTTPServer): """HTTP server carrying one local runtime and its bearer token.""" - daemon_threads = True + daemon_threads = False def __init__(self, address: tuple[str, int], runtime: LocalRuntime, token: str): if len(token) < MIN_TOKEN_LENGTH or not _SAFE_TOKEN.fullmatch(token): @@ -82,6 +82,24 @@ class RuntimeHandler(BaseHTTPRequestHandler): server: RuntimeHTTPServer protocol_version = "HTTP/1.0" + def setup(self) -> None: + super().setup() + self._header_timer = threading.Timer( + SOCKET_TIMEOUT_SECONDS, self._close_slow_headers + ) + self._header_timer.daemon = True + self._header_timer.start() + + def parse_request(self) -> bool: + try: + return super().parse_request() + finally: + self._header_timer.cancel() + + def finish(self) -> None: + self._header_timer.cancel() + super().finish() + def do_GET(self) -> None: try: path = urlparse(self.path).path @@ -161,6 +179,12 @@ def log_message(self, format: str, *args: Any) -> None: """Keep the default access log, which never includes Authorization header values.""" super().log_message(format, *args) + def _close_slow_headers(self) -> None: + try: + self.connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass + def _authorize(self) -> None: value = self.headers.get("Authorization", "") scheme, _, credentials = value.partition(" ") diff --git a/packages/learning/environments/brand-check/tests/test_runtime.py b/packages/learning/environments/brand-check/tests/test_runtime.py index 2befc7972..831832a6c 100644 --- a/packages/learning/environments/brand-check/tests/test_runtime.py +++ b/packages/learning/environments/brand-check/tests/test_runtime.py @@ -4,7 +4,9 @@ import base64 import json +import multiprocessing import shutil +import socket import threading import time import unittest @@ -39,8 +41,9 @@ class BlockingBackend(FixtureBackend): """A backend whose first job waits so a second job stays queued during close.""" def __init__(self): - self.started = threading.Event() - self.release = threading.Event() + context = multiprocessing.get_context("fork") + self.started = context.Event() + self.release = context.Event() def train(self, *args, **kwargs): self.started.set() @@ -191,6 +194,15 @@ def test_refuses_an_empty_backend_configuration(self): self.assertEqual(missing.exception.code, "backend_missing") + def test_refuses_a_non_string_backend_as_invalid_input(self): + runtime = self._runtime() + + with self.assertRaises(RuntimeFailure) as invalid: + runtime.submit(backend={"fixture": True}, max_updates=1, max_seconds=1) + + self.assertEqual(invalid.exception.code, "invalid_input") + self.assertEqual(invalid.exception.status, 400) + def test_refuses_a_second_runtime_without_changing_the_first_jobs(self): runtime = self._runtime() submitted = runtime.submit(backend="fixture", max_updates=8, max_seconds=5) @@ -216,6 +228,7 @@ def test_marks_a_backend_that_exceeds_the_wall_time_failed(self): self.assertEqual(failed["state"], "failed") self.assertEqual(failed["error"]["code"], "job_timed_out") self.assertEqual(list((self.state_dir / "candidates").iterdir()), []) + self.assertEqual(multiprocessing.active_children(), []) def test_close_cancels_queued_jobs_instead_of_starting_them(self): backend = BlockingBackend() @@ -406,6 +419,31 @@ def test_restart_recovers_a_promotion_generation_written_before_the_pointer(self self.assertEqual(reopened.promotion(), second_promotion) + def test_retry_reconciles_a_generation_written_before_the_pointer(self): + runtime = self._runtime() + finished = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=8, max_seconds=5)["id"], + ) + original = runtime._write_promotion + + def fail_pointer(promotion): + with mock.patch( + "replica_brand_check.runtime._write_json_atomic", + side_effect=OSError("pointer unavailable"), + ): + original(promotion) + + with mock.patch.object(runtime, "_write_promotion", side_effect=fail_pointer): + with self.assertRaises(OSError): + runtime.promote(finished["candidateId"], promoted_by="test-operator") + + recovered = runtime.promote( + finished["candidateId"], promoted_by="retrying-operator" + ) + self.assertEqual(recovered["generation"], 1) + self.assertEqual(recovered["candidateId"], finished["candidateId"]) + def test_production_rejects_fixture_promotion_without_changing_the_current_generation(self): runtime = self._runtime() first = self._wait_for_job( @@ -612,6 +650,46 @@ def test_http_routes_require_the_token_and_track_an_authorized_job(self): self.assertEqual(scored["promotionGeneration"], 3) self.assertEqual(self._http_json(f"{base_url}/ready"), {"status": "ready"}) + def test_header_deadline_releases_a_connection_slot(self): + runtime = self._runtime() + server = RuntimeHTTPServer(("127.0.0.1", 0), runtime, TOKEN) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join, 1) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + with mock.patch( + "replica_brand_check.runtime_server.SOCKET_TIMEOUT_SECONDS", 0.05 + ): + connection = socket.create_connection(server.server_address) + self.addCleanup(connection.close) + connection.sendall(b"GET /health HTTP/1.0\r\nX-Slow: ") + time.sleep(0.1) + + connection.settimeout(1) + self.assertEqual(connection.recv(1), b"") + self.assertEqual( + self._http_json(f"http://127.0.0.1:{server.server_address[1]}/health"), + {"status": "healthy"}, + ) + + def test_unknown_calibration_is_not_reported_as_not_worse(self): + runtime = self._runtime() + finished = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=8, max_seconds=5)["id"], + ) + report = runtime.candidate(finished["candidateId"])["evaluation"] + report["baseline"]["noul"]["ece"] = None + report["candidate"]["noul"]["ece"] = None + from replica_brand_check.runtime import _candidate_evaluation + + compared = _candidate_evaluation( + runtime.taskset_digest, report["baseline"], report["candidate"] + ) + self.assertIsNone(compared["evidence"]["calibrationErrorDelta"]) + self.assertIsNone(compared["evidence"]["calibrationNotWorse"]) + def test_refuses_malformed_questions_before_calling_the_backend(self): runtime = self._runtime() finished = self._wait_for_job( From 6b3e42b61d6071bb53013a6855e3c77af9edda45 Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Tue, 17 Nov 2026 15:57:19 -0800 Subject: [PATCH 7/9] fix(learning): reconcile interrupted rollbacks Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../environments/brand-check/runtime.py | 27 ++++++++++++----- .../brand-check/tests/test_runtime.py | 30 +++++++++++++++++++ 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/packages/learning/environments/brand-check/runtime.py b/packages/learning/environments/brand-check/runtime.py index 1158766c4..073649d79 100644 --- a/packages/learning/environments/brand-check/runtime.py +++ b/packages/learning/environments/brand-check/runtime.py @@ -8,6 +8,7 @@ from __future__ import annotations import hashlib +import importlib.metadata import importlib.util import json import math @@ -320,8 +321,10 @@ def rollback(self, *, rolled_back_by: str) -> dict[str, Any]: if not isinstance(rolled_back_by, str) or not rolled_back_by.strip(): raise RuntimeFailure("invalid_input", "rolledBackBy must name the operator.") with self._lock: - self._recover_promotion() + recovered = self._recover_promotion() current = self.promotion() + if recovered: + return current candidate_id = current.get("previousCandidateId") if not isinstance(candidate_id, str): raise RuntimeFailure( @@ -529,10 +532,10 @@ def _recover_interrupted_jobs(self) -> None: ) _write_json_atomic(path, job) - def _recover_promotion(self) -> None: + def _recover_promotion(self) -> bool: history = sorted((self.state_directory / "promotions").glob("*.json")) if not history: - return + return False latest = self._record(history[-1], "promotion generation", history[-1].stem) latest_generation = latest.get("generation") if ( @@ -550,14 +553,14 @@ def _recover_promotion(self) -> None: ) current_generation = current.get("generation", 0) if current else 0 if latest_generation == current_generation: - return + return False if ( latest_generation == current_generation + 1 and latest.get("previousCandidateId") == (current.get("candidateId") if current else None) ): _write_json_atomic(current_path, latest) - return + return True raise RuntimeFailure( "state_damaged", "Promotion history is not one generation ahead of the current pointer.", @@ -1034,10 +1037,18 @@ def _backend_process( def _process_context() -> Any: """Use spawn when installed; source-only tests bootstrap an alias that spawn cannot import.""" + try: + importlib.metadata.distribution("replica-brand-check") + installed_distribution = True + except importlib.metadata.PackageNotFoundError: + installed_distribution = False spec = importlib.util.find_spec("replica_brand_check") installed = ( - spec is not None - and spec.origin is not None - and Path(spec.origin).parent.name == "replica_brand_check" + installed_distribution + or ( + spec is not None + and spec.origin is not None + and Path(spec.origin).parent.name == "replica_brand_check" + ) ) return multiprocessing.get_context("spawn" if installed else "fork") diff --git a/packages/learning/environments/brand-check/tests/test_runtime.py b/packages/learning/environments/brand-check/tests/test_runtime.py index 831832a6c..7c4d609ab 100644 --- a/packages/learning/environments/brand-check/tests/test_runtime.py +++ b/packages/learning/environments/brand-check/tests/test_runtime.py @@ -444,6 +444,36 @@ def fail_pointer(promotion): self.assertEqual(recovered["generation"], 1) self.assertEqual(recovered["candidateId"], finished["candidateId"]) + def test_rollback_retry_returns_the_recovered_generation(self): + runtime = self._runtime() + first = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=4, max_seconds=5)["id"], + ) + second = self._wait_for_job( + runtime, + runtime.submit(backend="fixture", max_updates=8, max_seconds=5)["id"], + ) + runtime.promote(first["candidateId"], promoted_by="test-operator") + runtime.promote(second["candidateId"], promoted_by="test-operator") + original = runtime._write_promotion + + def fail_pointer(promotion): + with mock.patch( + "replica_brand_check.runtime._write_json_atomic", + side_effect=OSError("pointer unavailable"), + ): + original(promotion) + + with mock.patch.object(runtime, "_write_promotion", side_effect=fail_pointer): + with self.assertRaises(OSError): + runtime.rollback(rolled_back_by="test-operator") + + recovered = runtime.rollback(rolled_back_by="retrying-operator") + self.assertEqual(recovered["generation"], 3) + self.assertEqual(recovered["candidateId"], first["candidateId"]) + self.assertEqual(recovered["operation"], "rollback") + def test_production_rejects_fixture_promotion_without_changing_the_current_generation(self): runtime = self._runtime() first = self._wait_for_job( From 4be58c17d089071cd9217931adee19f276b0713e Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Tue, 17 Nov 2026 16:37:23 -0800 Subject: [PATCH 8/9] test(learning): allow spawned worker startup Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../learning/environments/brand-check/tests/test_runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/learning/environments/brand-check/tests/test_runtime.py b/packages/learning/environments/brand-check/tests/test_runtime.py index 7c4d609ab..664592a36 100644 --- a/packages/learning/environments/brand-check/tests/test_runtime.py +++ b/packages/learning/environments/brand-check/tests/test_runtime.py @@ -239,7 +239,7 @@ def test_close_cancels_queued_jobs_instead_of_starting_them(self): environment="test", ) first = runtime.submit(backend="blocking", max_updates=1, max_seconds=5) - self.assertTrue(backend.started.wait(1)) + self.assertTrue(backend.started.wait(5)) second = runtime.submit(backend="blocking", max_updates=1, max_seconds=5) closing = threading.Thread(target=runtime.close) closing.start() From ca020467fde1654f9477f2fb7211523377cce01f Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Wed, 18 Nov 2026 07:56:02 -0800 Subject: [PATCH 9/9] test(learning): synchronize spawned worker portably Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../environments/brand-check/runtime.py | 24 ++++++++++++------- .../brand-check/tests/test_runtime.py | 24 ++++++++++++------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/packages/learning/environments/brand-check/runtime.py b/packages/learning/environments/brand-check/runtime.py index 073649d79..97c91c6db 100644 --- a/packages/learning/environments/brand-check/runtime.py +++ b/packages/learning/environments/brand-check/runtime.py @@ -1019,18 +1019,24 @@ def _backend_process( max_updates=max_updates, deadline=deadline, ) - send.send(("ok", result)) + try: + send.send(("ok", result)) + except (BrokenPipeError, OSError): + pass except Exception as error: - send.send( - ( - "error", + try: + send.send( ( - error.code if isinstance(error, RuntimeFailure) else "backend_failed", - str(error), - error.status if isinstance(error, RuntimeFailure) else 500, - ), + "error", + ( + error.code if isinstance(error, RuntimeFailure) else "backend_failed", + str(error), + error.status if isinstance(error, RuntimeFailure) else 500, + ), + ) ) - ) + except (BrokenPipeError, OSError): + pass finally: send.close() diff --git a/packages/learning/environments/brand-check/tests/test_runtime.py b/packages/learning/environments/brand-check/tests/test_runtime.py index 664592a36..9a7b1330c 100644 --- a/packages/learning/environments/brand-check/tests/test_runtime.py +++ b/packages/learning/environments/brand-check/tests/test_runtime.py @@ -40,14 +40,15 @@ def train(self, *args, **kwargs): class BlockingBackend(FixtureBackend): """A backend whose first job waits so a second job stays queued during close.""" - def __init__(self): - context = multiprocessing.get_context("fork") - self.started = context.Event() - self.release = context.Event() + def __init__(self, started: Path, release: Path): + self.started = started + self.release = release def train(self, *args, **kwargs): - self.started.set() - self.release.wait(5) + self.started.write_text("started", encoding="utf-8") + deadline = time.monotonic() + 5 + while not self.release.exists() and time.monotonic() < deadline: + time.sleep(0.01) return super().train(*args, **kwargs) @@ -231,7 +232,9 @@ def test_marks_a_backend_that_exceeds_the_wall_time_failed(self): self.assertEqual(multiprocessing.active_children(), []) def test_close_cancels_queued_jobs_instead_of_starting_them(self): - backend = BlockingBackend() + started = self.taskset_dir.parent / "blocking-started" + release = self.taskset_dir.parent / "blocking-release" + backend = BlockingBackend(started, release) runtime = LocalRuntime( self.state_dir, self.taskset_dir, @@ -239,12 +242,15 @@ def test_close_cancels_queued_jobs_instead_of_starting_them(self): environment="test", ) first = runtime.submit(backend="blocking", max_updates=1, max_seconds=5) - self.assertTrue(backend.started.wait(5)) + deadline = time.monotonic() + 5 + while not started.exists() and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue(started.exists()) second = runtime.submit(backend="blocking", max_updates=1, max_seconds=5) closing = threading.Thread(target=runtime.close) closing.start() time.sleep(0.05) - backend.release.set() + release.write_text("release", encoding="utf-8") closing.join(2) self.assertFalse(closing.is_alive()) self.assertEqual(runtime.job(first["id"])["state"], "succeeded")