From de7c82dd3fe5ba3b81ceceb49cab07db206c4bae Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sat, 3 Oct 2026 09:35:26 -0400 Subject: [PATCH] Route native compilation through verified DSPy TypeSafe adapter --- AGENTS.md | 3 + README.md | 10 +- SETUP_PROMPT.md | 6 + docs/ARCHITECTURE.md | 15 ++ docs/INTEGRATIONS.md | 56 ++++- docs/SECURITY.md | 10 + docs/SOURCES.md | 9 + pyproject.toml | 3 +- src/typewright/banking77.py | 11 +- src/typewright/cli.py | 9 +- src/typewright/dspy_typesafe_backend.py | 85 +++++++ src/typewright/hierarchy_study.py | 12 +- src/typewright/research.py | 2 +- tests/test_compile_backend_boundary.py | 95 ++++++++ tests/test_dspy_compat.py | 13 +- tests/test_dspy_typesafe_backend.py | 294 ++++++++++++++++++++++++ tests/test_hierarchy_study.py | 46 ++++ 17 files changed, 658 insertions(+), 21 deletions(-) create mode 100644 src/typewright/dspy_typesafe_backend.py create mode 100644 tests/test_compile_backend_boundary.py create mode 100644 tests/test_dspy_typesafe_backend.py diff --git a/AGENTS.md b/AGENTS.md index 50b5c37..7a3bb25 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,6 +4,9 @@ - This is a compiler/runtime library. DSPy and GEPA belong to compile time. Importing `typewright` or running a frozen program must not import either. + Native compile-time evaluation uses the qualified DSPy TypeSafe wrapper; + frozen execution retains the direct SDK backend. The owner approved this + boundary on 2026-10-03; do not move DSPy into frozen execution. - Target Jev's native typed API. Choice probabilities, Noul P(true), Score expectations, and vendor confidence are distinct values. - Keep the source contract fixed: output names/types, Choice labels, Score scale diff --git a/README.md b/README.md index 7b403b8..c11eaef 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,14 @@ flowchart LR DSPy and GEPA are used only at compile time. Importing `typewright` or running a frozen program does not import either of them. +Native `compile --backend typesafe` and research selection use the experimental +DSPy TypeSafe wrapper qualified with **DSPy 3.4.0 / SDK 0.7.0**. Install +`.[compile]` (and `.[optimize]` for the generative teacher/GEPA). Frozen `run`, +evaluation and held-out research execution retain the direct SDK backend and +need only `.[live]`. This owner-approved boundary keeps DSPy out of runtime; +offline equivalence tests do not establish live provider operation. Programmatic +compilers can explicitly supply `DSPyTypeSafeBackend` to `ManagedBackend`. + ## Where Typewright fits DSPy 3.4 added experimental, native Jev support: a `TypeSafe` client behind the @@ -72,7 +80,7 @@ experimental upstream features that may change. Typewright's structural search a hierarchies are implemented but **not yet shown to improve accuracy**; the checks above are about process and safety, not measured gains. Planned alignment with DSPy: compatibility verification ([#90](https://github.com/Jev-Engineering/TypeWright/issues/90)) -and evaluating DSPy's Jev adapter at compile time ([#91](https://github.com/Jev-Engineering/TypeWright/issues/91)). +and the qualified compile-only DSPy Jev adapter ([#91](https://github.com/Jev-Engineering/TypeWright/issues/91)). ## Quickstart diff --git a/SETUP_PROMPT.md b/SETUP_PROMPT.md index 22be098..afc8342 100644 --- a/SETUP_PROMPT.md +++ b/SETUP_PROMPT.md @@ -98,6 +98,12 @@ the README's one-request draft smoke check with `--backend typesafe --allow-paid model, and usage without exposing credentials. Do not silently substitute mock results when live calls fail. +Native compile/selection uses the experimental DSPy TypeSafe wrapper, qualified +with DSPy 3.4.0 and SDK 0.7.0. Install `.[compile,optimize]` for live optimization; +the generative teacher independently retains its supported 3.3.1 floor. Frozen +runtime remains the direct SDK (`.[live]`) with no DSPy import. Offline transport +equivalence is not authentication or live provider evidence. + Live DSPy/GEPA optimization additionally needs an explicitly selected real teacher model and provider credentials, permission to send the projected training data to that provider, four disjoint splits, and agreed request/signature/token budgets. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a1ecc85..e0b2ef9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -73,6 +73,21 @@ and limits are copied by the compiler, then the full graph is validated. `Revise` updates requested instruction/rubric entries. All use `dspy.Predict` inside an explicit `dspy.context(lm=...)`. A user-selected teacher provider is mandatory. +The owner-approved native evaluation boundary is separate from that generative +teacher. Compile and selection entrypoints use `DSPyTypeSafeBackend`, a lazy +compile-only wrapper over `dspy.experimental.TypeSafe`, qualified against exactly +DSPy 3.4.0 and SDK 0.7.0. It passes raw native question dictionaries and retains +the complete SDK response envelope, including model, usage and Score legend; +it disables retries, upstream caches, history, callbacks and usage tracking. +`ManagedBackend` remains the sole pre-dispatch budget, identity, cache and leaf +receipt owner. Unknown cost remains null. Frozen execution and held-out research +requests use the direct `TypeSafeBackend`; importing or executing a frozen +program never imports DSPy. Both paths have offline native payload/response and +hierarchy reservation, settlement, resume and replay equivalence tests. Private +experimental hooks require the exact version guard; they are not live +qualification. Callers injecting a backend into the Python compiler retain +ownership of that explicit choice. + Standalone GEPA receives a candidate mapping `str -> str`. Each value is the JSON encoding of a TypeSafe entry; the entry itself may be a string, object, list, or null. Keys identify question instructions and indexed criteria/Score levels. diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index 568ccfe..db4e19e 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -7,7 +7,8 @@ Documentation inspection date: **2026-09-19**. These are the implementation targ | Python | >=3.11 | Core library, CLI, tests. | | TypeSafe SDK | `typesafe-sdk==0.7.0` | Native `TypeSafeClient.system_one`. | | Target model | `jev-1.13.0` | Versioned ID, not a moving alias. | -| DSPy | `dspy[litellm]>=3.3.1,<3.5` (verified: 3.3.1 and 3.4.0) | `Signature`, `Predict`, explicit teacher `LM/context`. | +| DSPy teacher | `dspy[litellm]>=3.3.1,<3.5` (verified: 3.3.1 and 3.4.0) | `Signature`, `Predict`, explicit teacher `LM/context`. | +| Native compile client | `dspy[typesafe]==3.4.0`, SDK 0.7.0 (`compile` extra) | Experimental TypeSafe wrapper; absent from frozen runtime. | | GEPA | `gepa==0.1.4` | Standalone adapter and custom component proposer. | These are not claims that all dependencies were installed or live-tested in the @@ -21,20 +22,22 @@ installing an arbitrary newest version. The supported range is `>=3.3.1,<3.5`, with `gepa==0.1.4` pinned (DSPy 3.4.0 resolves the same GEPA). Only 3.3.1 and 3.4.0 were exercised; later 3.4.x patch releases are accepted by the range but unverified. The upper bound is evidence-based: DSPy 3.4 -deprecates overriding `BaseLM.forward()` (which the teacher meter does) and schedules -its removal in 3.5. See `BUILD_REPORT.md` for the executed results. +deprecates overriding `BaseLM.forward()` and schedules its removal in 3.5. +The #93 replacement uses the supported engine on 3.4; only the supported 3.3 +legacy branch still meters forward. See `BUILD_REPORT.md` for the executed results. - Engine: DSPy 3.4 `engine="auto"` prefers native lm15 execution. The native response has no LiteLLM `_hidden_params`, so the teacher's SDK cost estimate would silently - become "unknown". `DSPyTeacher` therefore passes `engine="litellm"` whenever - `dspy.LM` accepts `engine` (3.3.x has no such argument and always uses LiteLLM). + become "unknown". the #90 checkpoint therefore selected `engine="litellm"`. The #93 engine now + observes the raw LiteLLM response before conversion (3.3 retains its legacy path). - Unchanged in 3.4.0 (checked offline against a loopback test double): caches off, `disable_history=True` leaves no history, signature-call and LM-forward ceilings count, `S1_TEACHER_API_BASE`/`S1_TEACHER_API_KEY` reach the request, consent flags are untouched. - Pydantic: `dspy` 3.4.0 itself requires `pydantic>=2.11.0`, so the `optimize` extra resolves to it automatically. The base install stays `pydantic>=2.10,<3`. -- Not covered: `dspy[typesafe]`, `TypeSafe`/`Noul`/`Choice`/`Score` decision types and - `ReAnchor` are not used or tested here. +- The #90 checkpoint did not cover the native DSPy client. The #91 qualification + below covers raw native transport, not DSPy Predict decision-type translation + or ReAnchor. Those remain unused. - The documentation inspection date above was not changed; this verification used the installed packages, not a fresh read of the primary documentation. @@ -121,3 +124,42 @@ the pinned `jev-1.13.0` only under the operator's own equivalence confirmation, not an independent attestation. Unexpected returned model identities fail. None of these routes has been verified live in this repository. See `examples/ai-gateway/README.md` for caps and setup. + +## Compile-only TypeSafe qualification (#91) + +Owner decision on 2026-10-03: use the DSPy client for native compile-time +evaluation and retain the direct SDK for frozen runtime. Install +`python -m pip install -e '.[compile]'`; combine with `optimize` when a generative +teacher/GEPA is required. The teacher's 3.3.1 floor remains supported independently; +the experimental native wrapper refuses all versions except DSPy 3.4.0 and +typesafe-sdk 0.7.0. This deliberately fails closed on unverified patch releases. + +Gap analysis against actual pinned upstream code, rechecked 2026-10-03: + +| Upstream behavior | Qualified wrapper | +| --- | --- | +| Defaults to `jev-latest`; no returned-model equality check | Versioned explicit model; ManagedBackend validates exact returned identity. | +| SDK kwargs omit RetryPolicy (SDK default is two retries) | Explicit RetryPolicy(max_retries=0), fixed timeout. | +| Cache defaults on; history may retain raw state/questions/response | Per-client cache off; history/callbacks/tracker disabled; no upstream response retention. | +| Response mapping drops Score legend; public result drops model/usage envelope | Raw native dictionaries and full SDK JSON envelope preserved. | +| No paid consent or our attempt ledger | Explicit allow_paid; ManagedBackend pre-dispatch budget/cache/identity and per-leaf reservations/settlement. | +| No billed cost evidence | Dollar cost remains null. | + +The wrapper uses exact-version experimental `_sdk_kwargs`, `_response` and +`_finish` hooks. Native client traffic never goes through a generative Predict +program. Compile CLI and flat/hierarchy research selection choose this backend; +frozen run, evaluate and held-out study phases use the direct SDK. Callers of +the Python compiler explicitly supply their backend. The direct SDK backend is +retained for runtime, rather than deprecated away. The live hierarchy manifest +binds separate compile and runtime identities; the code/manifest change needs +a fresh digest-bound review before paid selection. It cannot revive old consent. + +Installed-package socket-forbidden doubles verify payload/envelope equivalence, +real SDK constructor arguments, consent/version/model refusal, zero retries, +cache/attempt accounting, chain/conditional/diamond/nested graph receipts, +interrupted resume and zero-dispatch replay. These are offline mechanics, not +live inference, authenticity, measured gains or production qualification. + +Primary sources: [pinned TypeSafe client](https://github.com/stanfordnlp/dspy/blob/3.4.0/dspy/clients/typesafe.py), +[official Jev tutorial](https://dspy.ai/current/tutorials/jev_decisions/), +[SDK usage](https://docs.typesafe.ai/sdk/python/usage). diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 7cc22d5..599db25 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -3,6 +3,16 @@ ## Data flows A native TypeSafe request contains the projected state and all compiled questions. +Compile/selection uses the qualified DSPy TypeSafe wrapper; frozen and held-out +execution uses the direct SDK. The wrapper preserves native dictionaries and the +full response identity, sets SDK retries to zero and disables upstream cache, +history, callbacks and usage tracking. It does not route native state through a +generative teacher. Entrypoints and provider backends enforce explicit consent; +`ManagedBackend` owns pre-dispatch reservations, cache identity and evidence +settlement on both paths. Experimental DSPy hooks +are guarded to exactly 3.4.0 with SDK 0.7.0; drift fails closed. A reviewed live +hierarchy manifest now binds separate compile and runtime backend identities, +so an old manifest cannot approve the changed selection path by accident. A DSPy teacher call contains the source declaration, current plan/components, and selected projected training examples or training error traces with labels. The teacher may therefore see sensitive task data. Consent to TypeSafe is not assumed diff --git a/docs/SOURCES.md b/docs/SOURCES.md index 00c6842..320c158 100644 --- a/docs/SOURCES.md +++ b/docs/SOURCES.md @@ -63,3 +63,12 @@ The source release intentionally uses a standalone GEPA adapter rather than pretending Jev provides a general text-generation API. No upstream performance claim is asserted for this framework. Metrics from the bundled fixture are synthetic software test output only. + +## Compile-only native client review, 2026-10-03 + +- Exact-version upstream TypeSafe hooks: https://github.com/stanfordnlp/dspy/blob/3.4.0/dspy/clients/typesafe.py +- Native Jev tutorial: https://dspy.ai/current/tutorials/jev_decisions/ + +The wrapper is qualified against installed DSPy 3.4.0 and SDK 0.7.0 with +socket-forbidden native doubles. Direct SDK frozen execution is retained by +owner decision. No live inference, data sharing or measured gain follows. diff --git a/pyproject.toml b/pyproject.toml index 085976a..1b39aed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,10 +14,11 @@ dependencies = ["pydantic>=2.10,<3", "PyYAML>=6.0,<7"] [project.optional-dependencies] live = ["typesafe-sdk==0.7.0"] +compile = ["dspy[typesafe]==3.4.0", "typesafe-sdk==0.7.0"] optimize = ["dspy[litellm]>=3.3.1,<3.5", "gepa==0.1.4"] research = ["numpy>=1.26,<3", "matplotlib>=3.8,<4"] dev = ["pytest>=8,<10", "pytest-cov>=5,<8", "ruff>=0.11,<1", "build>=1,<2"] -all = ["typesafe-sdk==0.7.0", "dspy[litellm]>=3.3.1,<3.5", "gepa==0.1.4", "pytest>=8,<10", "pytest-cov>=5,<8"] +all = ["typesafe-sdk==0.7.0", "dspy[litellm,typesafe]==3.4.0", "gepa==0.1.4", "pytest>=8,<10", "pytest-cov>=5,<8"] [project.scripts] typewright = "typewright.cli:main" diff --git a/src/typewright/banking77.py b/src/typewright/banking77.py index 021b9ab..6a136e7 100644 --- a/src/typewright/banking77.py +++ b/src/typewright/banking77.py @@ -332,8 +332,13 @@ def plan(splits, seeds, search_calls, proposals, max_chars): "matching_note": "Equal budget ceilings, not equal realized tokens or proposal counts. Report actual use."} -def make_backend(live, limit): - return ManagedBackend(TypeSafeBackend(allow_paid=True) if live else MockBackend(), max_calls=limit, cache=None) +def make_backend(live, limit, *, compile_time=False): + if live and compile_time: + from .dspy_typesafe_backend import DSPyTypeSafeBackend + provider = DSPyTypeSafeBackend(allow_paid=True) + else: + provider = TypeSafeBackend(allow_paid=True) if live else MockBackend() + return ManagedBackend(provider, max_calls=limit, cache=None) def select_experiment(source, splits, dataset, out, *, seeds, search_calls, proposals, @@ -539,7 +544,7 @@ def main(argv=None): if args.command == "select" and live and (not args.share_feedback or not args.teacher_model): parser.error("Live selection also requires --share-feedback and an explicit --teacher-model") def backend_factory(limit): - return make_backend(live, limit) + return make_backend(live, limit, compile_time=args.command == "select") if args.command == "select": def teacher_factory(limit): return DSPyTeacher(args.teacher_model, allow_paid=True, share_feedback=True, max_calls=limit, diff --git a/src/typewright/cli.py b/src/typewright/cli.py index 2867b25..5452d1c 100644 --- a/src/typewright/cli.py +++ b/src/typewright/cli.py @@ -54,8 +54,13 @@ def make_backend(args): raise ConfigurationError("Choose --cache or --no-cache, not both.") if args.max_calls < 1 or args.request_timeout <= 0: raise ConfigurationError("Request budget and timeout must be positive.") - provider = MockBackend() if args.backend == "mock" else TypeSafeBackend( - allow_paid=args.allow_paid, timeout=args.request_timeout) + if args.backend == "mock": + provider = MockBackend() + elif getattr(args, "command", None) == "compile": + from .dspy_typesafe_backend import DSPyTypeSafeBackend + provider = DSPyTypeSafeBackend(allow_paid=args.allow_paid, timeout=args.request_timeout) + else: + provider = TypeSafeBackend(allow_paid=args.allow_paid, timeout=args.request_timeout) cache = None if args.no_cache else AnswerCache(args.cache) return ManagedBackend(provider, max_calls=args.max_calls, cache=cache) diff --git a/src/typewright/dspy_typesafe_backend.py b/src/typewright/dspy_typesafe_backend.py new file mode 100644 index 0000000..493aa09 --- /dev/null +++ b/src/typewright/dspy_typesafe_backend.py @@ -0,0 +1,85 @@ +"""Optional compile-time native adapter, qualified against DSPy 3.4.0 only. + +Frozen programs keep using TypeSafeBackend. Importing this module does not import +DSPy or the SDK. ManagedBackend owns all budget, cache and attempt accounting. +""" +from __future__ import annotations + +import importlib.metadata +import os +import re +import time +from typing import Any + +from .backends import Response, is_transient_sdk_error +from .errors import BackendError, ConfigurationError +from .models import Program + + +class DSPyTypeSafeBackend: + identity = "dspy-typesafe/3.4.0+typesafe-sdk/0.7.0" + synthetic = False + + def __init__(self, *, allow_paid: bool = False, timeout: float = 60.0): + if not allow_paid: + raise ConfigurationError("Live calls require explicit allow_paid=True / --allow-paid.") + if not os.getenv("TYPESAFE_API_KEY"): + raise ConfigurationError("Set TYPESAFE_API_KEY in your local environment; never paste it into chat.") + try: + versions = {name: importlib.metadata.version(name) for name in ("dspy", "typesafe-sdk")} + except importlib.metadata.PackageNotFoundError as exc: + raise ConfigurationError("Install DSPy 3.4.0 and typesafe-sdk 0.7.0 for compile-time evaluation.") from exc + if versions != {"dspy": "3.4.0", "typesafe-sdk": "0.7.0"}: + raise ConfigurationError("Experimental TypeSafe adapter requires dspy==3.4.0 and typesafe-sdk==0.7.0.") + import dspy + from dspy.experimental import TypeSafe + from typesafe_sdk import RetryPolicy + + # Upstream private hooks are qualified by the exact version above. Preserve + # raw native dictionaries and the entire SDK JSON envelope: upstream's + # normal decoding drops Score legend and its public result drops identity. + class NativeEnvelope(TypeSafe): + def _sdk_kwargs(self): + return {**super()._sdk_kwargs(), + "retry": RetryPolicy(max_retries=0, timeout=self.timeout)} + + @staticmethod + def _response(response): + data = response.model_dump(mode="json") + if not isinstance(data, dict) or "answers" not in data or "model" not in data: + raise BackendError("TypeSafe response lacks answers or model; refusing it.") + return data + + def _finish(self, request, response, cache_hit): + # No DSPy history, usage tracker or shared cache. The surrounding + # ManagedBackend validates identity and settles native receipts. + if cache_hit: + raise BackendError("Unexpected upstream cache hit; refusing it.") + return response + + self._dspy = dspy + self._client_type = NativeEnvelope + self.timeout = timeout + + def evaluate(self, program: Program, state: dict[str, Any]) -> Response: + if not re.fullmatch(r"jev-\d+\.\d+\.\d+", program.model): + raise ConfigurationError("Use a versioned model ID, not an alias, for measured execution.") + client = self._client_type(model=program.model, timeout=self.timeout, cache=False, callbacks=[]) + start = time.perf_counter() + try: + with self._dspy.context(disable_history=True, usage_tracker=None, callbacks=[]): + data = client(state=state, questions={key: q.wire() for key, q in program.questions.items()}) + except BackendError: + raise + except Exception as exc: + failure = BackendError( + f"TypeSafe request failed ({type(exc).__name__}); check local credentials, quota, and SDK compatibility." + ) + failure.transient = is_transient_sdk_error(exc) + raise failure from exc + return Response(answers=data["answers"], model=data["model"], usage=data.get("usage") or {}, + latency_ms=(time.perf_counter() - start) * 1000) + + def close(self): + # Upstream owns a context-managed SDK client per native attempt. + pass diff --git a/src/typewright/hierarchy_study.py b/src/typewright/hierarchy_study.py index b08093a..cae535c 100644 --- a/src/typewright/hierarchy_study.py +++ b/src/typewright/hierarchy_study.py @@ -378,6 +378,7 @@ def _live_manifest_proposal(protocol: dict[str, Any], source: HierarchySource) - "make no paid calls.") import inspect from .architect import DSPyTeacher + from .dspy_typesafe_backend import DSPyTypeSafeBackend teacher_defaults = inspect.signature(DSPyTeacher.__init__).parameters backend_defaults = inspect.signature(TypeSafeBackend.__init__).parameters optimization = protocol["optimization"] @@ -404,6 +405,10 @@ def _live_manifest_proposal(protocol: dict[str, Any], source: HierarchySource) - "request_timeout_s": backend_defaults["timeout"].default, "credential": {"environment_variable": "TYPESAFE_API_KEY", "recorded_in_manifest": False}}, + "compile_backend": {"identity": DSPyTypeSafeBackend.identity, + "dspy_version": "3.4.0", "sdk_version": "0.7.0", + "sdk_retries": 0, "framework_cache": "disabled", + "request_timeout_s": backend_defaults["timeout"].default}, "teacher": {"model": optimization["teacher_model"], "signature_ceiling": optimization["teacher_max_calls"], "provider_request_ceiling": protocol["budgets"]["teacher_provider_request_ceiling"], @@ -546,7 +551,12 @@ def _study_backend(protocol: dict[str, Any], arm: str, *, phase: str, elif protocol["mode"] == "mock": backend = ManagedBackend(MockBackend(), max_calls=ceiling, cache=None) else: - backend = ManagedBackend(TypeSafeBackend(allow_paid=allow_paid), max_calls=ceiling, cache=None) + if phase == "selection": + from .dspy_typesafe_backend import DSPyTypeSafeBackend + provider = DSPyTypeSafeBackend(allow_paid=allow_paid) + else: + provider = TypeSafeBackend(allow_paid=allow_paid) + backend = ManagedBackend(provider, max_calls=ceiling, cache=None) if backend.synthetic != (protocol["mode"] == "mock") or backend.budget.maximum != ceiling: raise ConfigurationError("Study backend identity or request ceiling differs from the protocol.") return backend diff --git a/src/typewright/research.py b/src/typewright/research.py index f2721e1..466f354 100644 --- a/src/typewright/research.py +++ b/src/typewright/research.py @@ -146,7 +146,7 @@ def select(protocol, out, *, backend_factory=None, teacher_factory=None, data_ro if out.exists(): raise ConfigurationError("Choose a fresh selection directory.") live = protocol["backend"] == "typesafe" - backend_factory = backend_factory or (lambda n: make_backend(live, n)) + backend_factory = backend_factory or (lambda n: make_backend(live, n, compile_time=True)) teacher_factory = teacher_factory or (lambda n: AuditedDSPyTeacher(protocol["teacher_model"], allow_paid=True, share_feedback=True, max_calls=n, max_tokens=protocol["teacher_max_output_tokens"], expected_response_model=protocol["teacher_response_model"], diff --git a/tests/test_compile_backend_boundary.py b/tests/test_compile_backend_boundary.py new file mode 100644 index 0000000..a0fc849 --- /dev/null +++ b/tests/test_compile_backend_boundary.py @@ -0,0 +1,95 @@ +"""Compile/runtime backend selection using authored no-dispatch provider doubles.""" +from types import SimpleNamespace + +import pytest + +from typewright import banking77, cli, hierarchy_study, research +from typewright import dspy_typesafe_backend + + +class ProviderDouble: + synthetic = False + identity = "authored-provider-double" + + def __init__(self, **kwargs): + self.configuration = kwargs + + def close(self): + pass + + +@pytest.fixture +def providers(monkeypatch): + class CompileDouble(ProviderDouble): + pass + + class RuntimeDouble(ProviderDouble): + pass + + monkeypatch.setattr(dspy_typesafe_backend, "DSPyTypeSafeBackend", CompileDouble) + monkeypatch.setattr(cli, "TypeSafeBackend", RuntimeDouble) + monkeypatch.setattr(banking77, "TypeSafeBackend", RuntimeDouble) + monkeypatch.setattr(hierarchy_study, "TypeSafeBackend", RuntimeDouble) + return CompileDouble, RuntimeDouble + + +@pytest.mark.parametrize("command", ["compile", "run", "evaluate"]) +def test_cli_native_backend_preserves_compile_runtime_boundary(command, providers): + compile_type, runtime_type = providers + args = SimpleNamespace(command=command, backend="typesafe", allow_paid=True, + request_timeout=7, max_calls=3, cache=None, no_cache=True) + backend = cli.make_backend(args) + try: + assert type(backend.backend) is (compile_type if command == "compile" else runtime_type) + assert backend.backend.configuration == {"allow_paid": True, "timeout": 7} + assert backend.budget.used == 0 + assert backend.budget.maximum == 3 + assert backend.cache is None + finally: + backend.close() + + +@pytest.mark.parametrize("compile_time", [False, True]) +def test_flat_research_factory_phase_is_explicit(compile_time, providers): + compile_type, runtime_type = providers + backend = banking77.make_backend(True, 2, compile_time=compile_time) + try: + assert type(backend.backend) is (compile_type if compile_time else runtime_type) + assert backend.budget.used == 0 + assert backend.budget.maximum == 2 + assert backend.cache is None + finally: + backend.close() + + +@pytest.mark.parametrize("phase", ["selection", "test"]) +def test_hierarchy_study_backend_phase_and_ceiling(phase, providers): + compile_type, runtime_type = providers + protocol = {"mode": "typesafe", "budgets": {"by_arm": {"arm": { + "allocated_selection_ceiling": 2, "allocated_test_ceiling": 3}}}} + backend = hierarchy_study._study_backend(protocol, "arm", phase=phase, allow_paid=True) + try: + assert type(backend.backend) is (compile_type if phase == "selection" else runtime_type) + assert backend.budget.maximum == (2 if phase == "selection" else 3) + assert backend.budget.used == 0 + assert backend.cache is None + finally: + backend.close() + + +def test_research_selection_default_factory_uses_compile_phase(tmp_path, monkeypatch): + class FactoryObserved(Exception): + pass + + observed = [] + + def factory(live, ceiling, *, compile_time=False): + observed.append((live, ceiling, compile_time)) + raise FactoryObserved + + monkeypatch.setattr(research, "validate_protocol", lambda *_: {"task": (None, {}, {}, {})}) + monkeypatch.setattr(research, "make_backend", factory) + protocol = {"backend": "typesafe", "seeds": [7], "search_calls_per_arm_seed": 3} + with pytest.raises(FactoryObserved): + research.select(protocol, tmp_path / "selection") + assert observed == [(True, 3, True)] diff --git a/tests/test_dspy_compat.py b/tests/test_dspy_compat.py index 4547f14..388d539 100644 --- a/tests/test_dspy_compat.py +++ b/tests/test_dspy_compat.py @@ -37,12 +37,15 @@ def _optimize(): return data["project"]["optional-dependencies"] -def test_optimize_extra_declares_the_verified_dspy_range_and_pinned_gepa(): +def test_extras_declare_verified_teacher_range_and_native_compile_profile(): + extras = _optimize() + assert "dspy[litellm]>=3.3.1,<3.5" in extras["optimize"] + assert "dspy[typesafe]==3.4.0" in extras["compile"] + assert "typesafe-sdk==0.7.0" in extras["compile"] + assert "dspy[litellm,typesafe]==3.4.0" in extras["all"] + assert "typesafe-sdk==0.7.0" in extras["all"] for name in ("optimize", "all"): - extra = _optimize()[name] - assert "dspy[litellm]>=3.3.1,<3.5" in extra, name - assert "gepa==0.1.4" in extra, name - assert not any(r.startswith("dspy[litellm]==") for name in ("optimize", "all") for r in _optimize()[name]) + assert "gepa==0.1.4" in extras[name], name @pytest.mark.optional diff --git a/tests/test_dspy_typesafe_backend.py b/tests/test_dspy_typesafe_backend.py new file mode 100644 index 0000000..48fc051 --- /dev/null +++ b/tests/test_dspy_typesafe_backend.py @@ -0,0 +1,294 @@ +"""Installed DSPy with labeled SDK test doubles; never real Jev results.""" +import copy + +import pytest + +from typewright.backends import AnswerCache, AttemptLedger, ManagedBackend, MockBackend, TypeSafeBackend +from typewright.dspy_typesafe_backend import DSPyTypeSafeBackend +from typewright.errors import BackendError, BudgetExceeded, ConfigurationError +from typewright.runtime import Runtime + +STATE = {"message": "refund", "customer_plan": "team"} + + +@pytest.fixture +def sdk_double(monkeypatch, program): + import socket + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + def forbid_network(sock, address): + raise AssertionError("Native SDK doubles must never dispatch sockets") + + monkeypatch.setattr(socket.socket, "connect", forbid_network) + monkeypatch.setattr(socket.socket, "connect_ex", forbid_network) + pytest.importorskip("dspy") + import importlib.metadata + if importlib.metadata.version("dspy") != "3.4.0": + pytest.skip("Experimental adapter is qualified only against DSPy 3.4.0") + sdk = pytest.importorskip("typesafe_sdk") + monkeypatch.setenv("TYPESAFE_API_KEY", "obviously-fake-offline-fixture") + envelope = MockBackend().evaluate(program, STATE).to_dict() + envelope["usage"] = {"input_tokens": 7, "output_tokens": 3} + calls, constructions = [], [] + control = {"envelope": envelope, "error": None} + + class SDKDouble: + def __init__(self, **kwargs): + constructions.append(kwargs) + self.model = kwargs.get("model") + + def system_one(self, **kwargs): + calls.append({**kwargs, "model": kwargs.get("model", self.model)}) + if control["error"]: + raise control["error"] + + class Envelope: + def model_dump(self, *, mode): + assert mode == "json" + return copy.deepcopy(control["factory"](kwargs) if "factory" in control else control["envelope"]) + return Envelope() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def close(self): + pass + + monkeypatch.setattr(sdk, "TypeSafeClient", SDKDouble) + return calls, constructions, control + + +def test_consent_and_import_boundary(monkeypatch): + monkeypatch.delenv("TYPESAFE_API_KEY", raising=False) + with pytest.raises(ConfigurationError, match="explicit"): + DSPyTypeSafeBackend() + with pytest.raises(ConfigurationError, match="TYPESAFE_API_KEY"): + DSPyTypeSafeBackend(allow_paid=True) + + +def test_native_equivalence(program, sdk_double): + calls, constructors, _ = sdk_double + direct = TypeSafeBackend(allow_paid=True) + wrapped = DSPyTypeSafeBackend(allow_paid=True) + a, b = direct.evaluate(program, STATE), wrapped.evaluate(program, STATE) + assert calls[0] == calls[1] + assert a.answers == b.answers + assert a.model == b.model == program.model + assert a.usage == b.usage == {"input_tokens": 7, "output_tokens": 3} + assert not a.synthetic and not b.synthetic # Transport doubles, no live inference claim. + for config in constructors: + assert config["retry"].max_retries == 0 + + +def test_managed_cache_budget_and_leaf_receipts(program, sdk_double): + calls, _, _ = sdk_double + backend = ManagedBackend(DSPyTypeSafeBackend(allow_paid=True), max_calls=1, cache=AnswerCache()) + ledger = AttemptLedger(graph_sha256="fixture", maximum=1, nodes={"leaf"}) + response = backend.evaluate(program, STATE, ledger=ledger, node_id="leaf") + assert ledger.snapshot()["receipts"][0]["status"] == "response_received" + backend.remember_validated(program, STATE, response, ledger=ledger) + assert ledger.snapshot()["receipts"][0]["status"] == "validated" + assert backend.evaluate(program, STATE, ledger=ledger, node_id="leaf").cached + with pytest.raises(BudgetExceeded): + backend.evaluate(program, {**STATE, "message": "changed"}, ledger=ledger, node_id="leaf") + assert len(calls) == 1 + assert backend.accounting()["dollar_cost"] is None + restored = AttemptLedger.restore(ledger.snapshot(), graph_sha256="fixture", maximum=1, nodes={"leaf"}) + assert restored.snapshot() == ledger.snapshot() + backend.close() + + +@pytest.mark.parametrize("failure", ["connection", "identity", "malformed"]) +def test_failures_charge_and_never_cache(program, sdk_double, failure): + calls, _, control = sdk_double + backend = ManagedBackend(DSPyTypeSafeBackend(allow_paid=True), max_calls=1, cache=AnswerCache()) + if failure == "connection": + control["error"] = TimeoutError("offline transport double") + elif failure == "identity": + control["envelope"]["model"] = "different-model" + else: + control["envelope"]["answers"] = {} + with pytest.raises((BackendError, ConfigurationError)): + Runtime(program, backend).run(STATE) + assert backend.budget.used == 1 + assert len(calls) == 1 + assert backend.cache.db.execute("SELECT COUNT(*) FROM answers").fetchone()[0] == 0 + backend.close() + + +def native_graph_fixture(sdk_double, name): + import json + from pathlib import Path + from types import SimpleNamespace + from typewright.hierarchy import HierarchySource, lower_hierarchy + from typewright.models import Question + root = Path(__file__).resolve().parents[1] / "examples" / "hierarchy_contract" + artifact = lower_hierarchy(HierarchySource.load(root / f"{name}.json")) + state = json.loads((root / "cases.json").read_text())[f"{name}.json"]["state"] + _, _, control = sdk_double + + def factory(payload): + leaf = SimpleNamespace(model=artifact.source.model, questions={ + key: Question.model_validate(value) for key, value in payload["questions"].items()}) + return MockBackend().evaluate(leaf, payload["state"]).to_dict() + control["factory"] = factory + return artifact, state + + +@pytest.mark.parametrize("name", ["chain", "conditional", "diamond", "nested"]) +def test_native_graph_equivalence_replay_resume(name, sdk_double, tmp_path): + from typewright.hierarchy_runtime import HierarchyRuntime + calls, _, _ = sdk_double + artifact, state = native_graph_fixture(sdk_double, name) + direct = HierarchyRuntime(artifact, ManagedBackend(TypeSafeBackend(allow_paid=True))).run(state) + direct_calls = copy.deepcopy(calls) + calls.clear() + directory = tmp_path / "evidence" + compiled = HierarchyRuntime(artifact, ManagedBackend(DSPyTypeSafeBackend(allow_paid=True))).run( + state, evidence_dir=directory) + assert calls == direct_calls + assert compiled["status"] == direct["status"] == "completed" + for field in ("decisions", "executed", "path"): + assert compiled[field] == direct[field] + receipts = compiled["accounting"]["attempt_ledger"]["receipts"] + assert len(receipts) == len(calls) + assert all(receipt["status"] == "validated" for receipt in receipts) + calls.clear() + for mode in ("replay", "resume"): + result = HierarchyRuntime(artifact, ManagedBackend(DSPyTypeSafeBackend(allow_paid=True))).run( + state, evidence_dir=directory, evidence_mode=mode) + assert result == compiled + assert calls == [] + + +def test_interrupted_graph_attempt_stays_charged_on_resume(sdk_double, tmp_path, monkeypatch): + import json + from typewright.hierarchy_evidence import HierarchyEvidence + from typewright.hierarchy_runtime import HierarchyRuntime + calls, _, _ = sdk_double + artifact, state = native_graph_fixture(sdk_double, "chain") + directory = tmp_path / "interrupted" + original = HierarchyEvidence._append + + def interrupt(self, payload): + if payload["kind"] == "stage": + raise RuntimeError("offline simulated interruption") + return original(self, payload) + + with monkeypatch.context() as context: + context.setattr(HierarchyEvidence, "_append", interrupt) + with pytest.raises(RuntimeError, match="interruption"): + HierarchyRuntime(artifact, ManagedBackend(DSPyTypeSafeBackend(allow_paid=True))).run( + state, evidence_dir=directory) + assert len(calls) == 1 + assert json.loads((directory / "attempts.json").read_text())["ledger"]["used"] == 1 + calls.clear() + resumed = HierarchyRuntime(artifact, ManagedBackend(DSPyTypeSafeBackend(allow_paid=True))).run( + state, evidence_dir=directory, evidence_mode="resume") + assert resumed["status"] == "completed" + assert len(calls) == 2 + assert resumed["accounting"]["requests_attempted"] == 3 + assert resumed["accounting"]["usage_unknown_calls"] == 3 + calls.clear() + replayed = HierarchyRuntime(artifact, ManagedBackend(DSPyTypeSafeBackend(allow_paid=True))).run( + state, evidence_dir=directory, evidence_mode="replay") + assert replayed == resumed + assert calls == [] + + +def test_no_dspy_history_cache_callbacks_or_tracking(program, sdk_double): + import dspy + + observed = [] + + class ForbiddenCallback: + def on_lm_start(self, *args, **kwargs): + observed.append("callback") + + class ForbiddenTracker: + def add_usage(self, *args, **kwargs): + observed.append("tracker") + + wrapped = DSPyTypeSafeBackend(allow_paid=True) + clients = [] + original = wrapped._client_type + + def client_factory(**kwargs): + client = original(**kwargs) + clients.append(client) + return client + wrapped._client_type = client_factory + with dspy.context(callbacks=[ForbiddenCallback()], usage_tracker=ForbiddenTracker(), disable_history=False): + wrapped.evaluate(program, STATE) + assert observed == [] + assert clients[0].cache is False + assert clients[0].history == [] + assert clients[0].callbacks == [] + + +def test_unknown_usage_is_unknown(program, sdk_double): + _, _, control = sdk_double + control["envelope"].pop("usage") + backend = ManagedBackend(DSPyTypeSafeBackend(allow_paid=True)) + response = backend.evaluate(program, STATE) + assert response.usage == {} + assert backend.accounting()["usage_unknown_calls"] == 1 + assert backend.accounting()["dollar_cost"] is None + + +def test_version_drift_fails_before_dispatch(monkeypatch, sdk_double): + import importlib.metadata + calls, _, _ = sdk_double + monkeypatch.setattr(importlib.metadata, "version", lambda name: "3.4.1" if name == "dspy" else "0.7.0") + with pytest.raises(ConfigurationError, match="requires"): + DSPyTypeSafeBackend(allow_paid=True) + assert calls == [] + + +@pytest.mark.parametrize("model", ["jev", "jev-latest", "jev-preview", "unexpected-alias"]) +def test_only_versioned_native_model_dispatches(program, sdk_double, model): + calls, _, _ = sdk_double + program = program.model_copy(update={"model": model}) + with pytest.raises(ConfigurationError, match="versioned"): + DSPyTypeSafeBackend(allow_paid=True).evaluate(program, STATE) + assert calls == [] + + +def test_import_compile_module_does_not_import_optional_packages(): + import subprocess + import sys + subprocess.run([sys.executable, "-c", + "import sys; import typewright.dspy_typesafe_backend; " + "assert not {'dspy', 'gepa', 'typesafe_sdk'} & set(sys.modules)"], check=True) + + +def test_actual_sdk_constructor_accepts_upstream_wrapper_arguments(monkeypatch): + import importlib.metadata + import socket + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + def forbid_network(sock, address): + raise AssertionError("SDK constructor contract must stay offline") + + monkeypatch.setattr(socket.socket, "connect", forbid_network) + monkeypatch.setattr(socket.socket, "connect_ex", forbid_network) + pytest.importorskip("dspy") + if importlib.metadata.version("dspy") != "3.4.0": + pytest.skip("Experimental adapter is qualified only against DSPy 3.4.0") + sdk = pytest.importorskip("typesafe_sdk") + monkeypatch.setenv("TYPESAFE_API_KEY", "obviously-fake-offline-fixture") + backend = DSPyTypeSafeBackend(allow_paid=True, timeout=37.0) + upstream = backend._client_type(model="jev-1.13.0", timeout=37.0, cache=False, callbacks=[]) + kwargs = upstream._sdk_kwargs() + assert kwargs["retry"].max_retries == 0 + assert kwargs["retry"].timeout == 37.0 + # Instantiate the actual SDK class, rather than the permissive SDKDouble. + client = sdk.TypeSafeClient(**kwargs) + try: + assert client is not None + finally: + client.close() + backend.close() diff --git a/tests/test_hierarchy_study.py b/tests/test_hierarchy_study.py index 1f03e27..19121bf 100644 --- a/tests/test_hierarchy_study.py +++ b/tests/test_hierarchy_study.py @@ -396,6 +396,11 @@ def test_live_manifest_proposal_pins_every_run_parameter_without_calls(tmp_path, assert proposal["native_backend"]["identity"] == "typesafe-sdk/0.7.0" assert proposal["native_backend"]["sdk_retries"] == 0 assert proposal["native_backend"]["credential"]["recorded_in_manifest"] is False + assert proposal["compile_backend"] == { + "identity": "dspy-typesafe/3.4.0+typesafe-sdk/0.7.0", + "dspy_version": "3.4.0", "sdk_version": "0.7.0", + "sdk_retries": 0, "framework_cache": "disabled", "request_timeout_s": 60.0, + } assert proposal["teacher"]["model"] == "test-only/model" assert proposal["teacher"]["api_base"] == "provider_default" assert proposal["teacher"]["credential"]["recorded_in_manifest"] is False @@ -464,6 +469,47 @@ def attempt(review, name): assert study._envelope_read(tmp_path / "complete-reviewed.json") == reviewed +@pytest.mark.parametrize("change", ["missing", "identity", "dspy_version", "sdk_version", + "sdk_retries", "framework_cache", "request_timeout_s"]) +def test_live_selection_rejects_obsolete_compile_manifest_before_backend_creation(tmp_path, change): + import copy + from typewright.io import fingerprint + + protocol_path, protocol = _live_test_double_protocol(tmp_path) + proposal = study.propose_live_manifest(protocol_path, tmp_path / "manifest.json") + obsolete = copy.deepcopy(proposal) + if change == "missing": + del obsolete["compile_backend"] + else: + obsolete["compile_backend"][change] = { + "identity": "typesafe-sdk/0.7.0", "dspy_version": "3.3.1", + "sdk_version": "0.6.0", "sdk_retries": 2, + "framework_cache": "enabled", "request_timeout_s": 10.0, + }[change] + # The old parameters have an internally consistent review and checksums. + # Rejection must compare current execution parameters, not merely their digest. + reviewed_path = tmp_path / "obsolete-reviewed.json" + study._envelope_write(reviewed_path, { + "format": study.LIVE_MANIFEST_REVIEWED_FORMAT, + "proposal": obsolete, "manifest_sha256": fingerprint(obsolete), + "review": _completed_review(obsolete), + }) + constructions = [] + + def forbidden_backend(*args): + constructions.append(args) + raise AssertionError("Obsolete consent must fail before backend creation") + + output = tmp_path / "selection" + with pytest.raises(ConfigurationError, match="Reviewed live manifest differs"): + select_and_freeze(protocol_path, output, allow_paid=True, share_feedback=True, + approved_protocol_sha256=fingerprint(protocol), + reviewed_manifest=reviewed_path, teacher=object(), + backend_factory=forbidden_backend) + assert constructions == [] + assert not output.exists() + + def test_live_selection_requires_matching_reviewed_manifest_before_any_provider(tmp_path, monkeypatch): from typewright.backends import TypeSafeBackend from typewright.io import fingerprint