diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index f55282c..1f91f26 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -21,7 +21,7 @@ "name": "fxtr", "source": "./plugins/fxtr", "description": "Skills for writing, running, and viewing fxtr experiments, and for calling models from them with behaviors", - "version": "0.1.1", + "version": "0.1.2", "author": { "name": "TransluceAI" }, diff --git a/plugins/fxtr/.claude-plugin/plugin.json b/plugins/fxtr/.claude-plugin/plugin.json index cd446ad..1871c0a 100644 --- a/plugins/fxtr/.claude-plugin/plugin.json +++ b/plugins/fxtr/.claude-plugin/plugin.json @@ -4,5 +4,5 @@ "author": { "name": "TransluceAI" }, - "version": "0.1.1" + "version": "0.1.2" } diff --git a/plugins/fxtr/skills/behaviors/provenance.json b/plugins/fxtr/skills/behaviors/provenance.json index e14415c..4aebdeb 100644 --- a/plugins/fxtr/skills/behaviors/provenance.json +++ b/plugins/fxtr/skills/behaviors/provenance.json @@ -2,10 +2,10 @@ "skill": "behaviors", "documentation": "https://docs.transluce.ai/behaviors", "packages": { - "fxtr": "0.0.1a2", - "behaviors": "0.0.1a2" + "fxtr": "0.0.1a4", + "behaviors": "0.0.1a3" }, - "revision": "81bc041101951f16dd70e69340d64e80abd34f01", + "revision": "051a5d3ab1f6df872092af2365b34c5d0f6671d3", "modified": false, - "content_sha256": "de0ca958d570464776b006b595d0f26b7be280a6c5baf88c46629d2982757eee" + "content_sha256": "9ca8c278e92e77f29f281313b17717930e6ecc67e13027fedde3152fc37ea5b9" } diff --git a/plugins/fxtr/skills/behaviors/references/guides/conversation-records.mdx b/plugins/fxtr/skills/behaviors/references/guides/conversation-records.mdx index cb317c2..b5dd298 100644 --- a/plugins/fxtr/skills/behaviors/references/guides/conversation-records.mdx +++ b/plugins/fxtr/skills/behaviors/references/guides/conversation-records.mdx @@ -16,23 +16,89 @@ answer's text alone, keep the `ContentText` blocks of the turn's `AssistantMessa [judge step's](language-models.mdx#a-judge-step) `answer_text` does, and keep the structured rollout for later inspection. -A model turn made by an API-backed session records the provider's response in -`ModelTurn.metadata["response"]`, as JSON, including its stop reason and usage. Metadata is defined -by whatever produced the turn, so validate it before scoring with it. +## Metadata + +Turns (`ContextTurn`, `ModelTurn`, `UserTurn`, `ToolTurn`), `EndRollout`, and `ChatRollout` carry +`metadata`: a tuple of `DynamicData` entries, each holding one +[tagged dataclass](../../../fxtr/references/concepts/entities-and-custom-types.mdx#inline-tagged-unions) in its stored +form. Whatever produced the turn decides which types it records, so a reader names the type it +expects. `find_metadata(metadata, SomeType)` returns the first entry of that type, decoded, or +`None` if there is none; `entry.try_as_type(SomeType)` does the same for a single entry. An entry +tagged with the type but holding fields that don't match it raises `ConversionError`. + +A model turn made by an API-backed session records the provider's response as a `ModelResponse`, +including its stop reason and usage: + +```python +from behaviors.types import ModelResponse, find_metadata + +response = find_metadata(model_turn.metadata, ModelResponse) +if response is not None: + print(response.stop_reason.type, response.usage.output_tokens) +``` + +To record something of your own, such as which question a context turn asked, define a frozen +tagged dataclass with a type ID in a domain you control, and wrap each value with +`DynamicData.from_tagged`: + +```python +from dataclasses import dataclass + +from behaviors.types import ContentText, ContextTurn, DynamicData, UserMessage, find_metadata +from fxtr.entity_defns.dataclass_bases import TaggedDataclass + + +@dataclass(frozen=True) +class QuestionIndex(TaggedDataclass, fxtr_type="org.example.my_experiment.QuestionIndex.v1"): + index: int + + +turn = ContextTurn( + messages=(UserMessage(content=(ContentText(text="What is 2 + 2?"),)),), + available_tools=None, + metadata=(DynamicData.from_tagged(QuestionIndex(index=0)),), +) +assert find_metadata(turn.metadata, QuestionIndex) == QuestionIndex(index=0) +``` + +Add an entry to existing metadata with `turn.metadata + (DynamicData.from_tagged(value),)`. +`Transcript`, `TranscriptGroup`, and `AgentRun` metadata is different: it is a JSON object with +string keys, matching [Docent](docent-readings.mdx)'s metadata. ## Storing records `ChatRollout`, `Transcript`, `TranscriptGroup`, `AgentRun`, and the concrete session initializations, such as `SetSystemMessage`, are fxtr entities, with type IDs of the form -`org.transluce.behaviors..v1`. A step that returns a `ChatRollout` has it stored like any -other entity (see [Entities and custom types](../../../fxtr/references/concepts/entities-and-custom-types.mdx)), and +`org.transluce.behaviors..v`, such as `org.transluce.behaviors.ChatRollout.v2`. A +version is bumped when a type's stored shape changes, and a record stored under an earlier version +doesn't load as the current type. Versions are per type: `ChatRollout` and its turns use `.v2`, +while messages and `ModelResponse` use `.v1`. Only the current version of each type is supported. + +A step that returns a `ChatRollout` has it stored like any other entity +(see [Entities and custom types](../../../fxtr/references/concepts/entities-and-custom-types.mdx)), and `await storage.store(entity)` and `await storage.load(ref)` work on them too. To load a bare ID, pass the concrete type, as in `as_type=ChatRollout`, not an abstract base such as `SessionInitialization`. A tree of transcripts and groups is stored as one entity, so there is no need to store each child separately. +A stored rollout includes every field, including `initialization`. It keeps its initialization +inline, as `DynamicData` (or an explicit null if there was none), since initialization types are +open-ended. In Python, read it by naming the type you handle, which gives `None` for +any other type: + +```python +from behaviors.types import SetSystemMessage + +init = rollout.initialization +set_system = init.try_as_type(SetSystemMessage) if init is not None else None +``` + fxtr checks stored records strictly: give fields their declared types (`temperature=0.0`, not -`0`), and keep metadata to JSON values with string keys. +`0`), and keep transcript metadata to JSON values with string keys. + +The step cache can still hold results written under another type version. When changing the +rollout version a project uses, launch under a fresh root or clear the affected steps' cache +entries before rerunning (see [Managing the step cache](../../../fxtr/references/concepts/managing-the-step-cache.mdx)). ## Transcripts @@ -69,7 +135,11 @@ A new project's renderer package doesn't depend on it yet: add ## Checkpointing sessions A serializable session can save its state and be restored from it. `await session.get_state()` -returns a JSON object, and `await factory.restore_session(state)` starts a session in that state. +returns the state as `DynamicData`, tagged with the implementation's own state type, and +`await factory.restore_session(state)` starts a session in that state; it raises `ConversionError` +for a state that isn't its own. To keep a state as JSON, write +`json.dumps(thaw_fxtr_data(state.data))` (`thaw_fxtr_data` is in `fxtr.core.entity_data`), and +read it back with `DynamicData.from_data(json.loads(text))`. The sessions of `chat_session_model_from_api` and of `ScriptedUserSimulator` support this, and the `serializable_context_policy_from_*` combinators build checkpointable policies when every component they combine is checkpointable. Generator-based policies can't be checkpointed, because a diff --git a/plugins/fxtr/skills/behaviors/references/guides/policies-and-tools.mdx b/plugins/fxtr/skills/behaviors/references/guides/policies-and-tools.mdx index 923316d..7749d91 100644 --- a/plugins/fxtr/skills/behaviors/references/guides/policies-and-tools.mdx +++ b/plugins/fxtr/skills/behaviors/references/guides/policies-and-tools.mdx @@ -55,10 +55,19 @@ tools offered only at some turns), write the policy as an async generator: each next context turn, and the model's reply comes back as the value of the `yield`: ```python +from dataclasses import dataclass + from behaviors.implementations.context_policies import ( ContextPolicyGenerator, context_policy_from_generator, ) -from behaviors.types import ContentText, ContextTurn, EndRollout, UserMessage +from behaviors.types import ContentText, ContextTurn, DynamicData, EndRollout, UserMessage +from fxtr.entity_defns.dataclass_bases import TaggedDataclass + + +@dataclass(frozen=True) +class ReplyReceived(TaggedDataclass, fxtr_type="org.example.my_experiment.ReplyReceived.v1"): + received: bool + async def turns() -> ContextPolicyGenerator: reply = yield ContextTurn( @@ -66,13 +75,16 @@ async def turns() -> ContextPolicyGenerator: available_tools=None, ) # Inspect reply.messages here to choose a data-dependent next turn if needed. - yield EndRollout(metadata={"received_reply": reply is not None}) + outcome = ReplyReceived(received=reply is not None) + yield EndRollout(metadata=(DynamicData.from_tagged(outcome),)) policy = context_policy_from_generator(turns, initialization=None) ``` Pass the generator function (`turns`), not a running generator: each rollout starts its own. Yield `EndRollout` explicitly to finish, since a generator that simply returns raises an error. +Its `metadata`, like a turn's, holds tagged records (see +[Metadata](conversation-records.mdx#metadata)). For a reusable policy with settings of its own, subclass `GeneratorContextPolicy`, overriding `generate_turns`, and `generate_initialization` if its sessions need one. diff --git a/plugins/fxtr/skills/first-experiment/provenance.json b/plugins/fxtr/skills/first-experiment/provenance.json index b924a56..a43d0a9 100644 --- a/plugins/fxtr/skills/first-experiment/provenance.json +++ b/plugins/fxtr/skills/first-experiment/provenance.json @@ -2,10 +2,10 @@ "skill": "first-experiment", "documentation": null, "packages": { - "fxtr": "0.0.1a2", - "behaviors": "0.0.1a2" + "fxtr": "0.0.1a4", + "behaviors": "0.0.1a3" }, - "revision": "81bc041101951f16dd70e69340d64e80abd34f01", + "revision": "051a5d3ab1f6df872092af2365b34c5d0f6671d3", "modified": false, "content_sha256": "1b39e083738e6d10b515568bf7912241a1874cda150eb40908b92e7927d38b5f" } diff --git a/plugins/fxtr/skills/fxtr/SKILL.md b/plugins/fxtr/skills/fxtr/SKILL.md index b2208e5..e109e08 100644 --- a/plugins/fxtr/skills/fxtr/SKILL.md +++ b/plugins/fxtr/skills/fxtr/SKILL.md @@ -57,11 +57,12 @@ views before drawing anything in a renderer. Launching and following jobs, and the step cache that decides what a launch recomputes. Read Running and viewing jobs before the first launch. -| Task | Read | -| ------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------- | -| Launch a job, watch it in the viewer, return a report and read results, resume, inspect, or share jobs | [Running and viewing jobs](references/guides/running-and-viewing-jobs.mdx) | -| Reuse or recompute cached results, resolve cache conflicts, change a step's logic | [Managing the step cache](references/concepts/managing-the-step-cache.mdx) | -| Look up a command's options | [CLI reference](references/reference/cli.mdx) | +| Task | Read | +| ------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------- | +| Launch a job, watch it in the viewer, return a report and read results, resume, inspect, or share jobs | [Running and viewing jobs](references/guides/running-and-viewing-jobs.mdx) | +| Choose how many steps run at once, or speed up a job whose steps call APIs | [Choose how many steps run at once](references/guides/running-and-viewing-jobs.mdx#choose-how-many-steps-run-at-once) | +| Reuse or recompute cached results, resolve cache conflicts, change a step's logic | [Managing the step cache](references/concepts/managing-the-step-cache.mdx) | +| Look up a command's options | [CLI reference](references/reference/cli.mdx) | ## Best practices diff --git a/plugins/fxtr/skills/fxtr/provenance.json b/plugins/fxtr/skills/fxtr/provenance.json index 676077f..5c5ec05 100644 --- a/plugins/fxtr/skills/fxtr/provenance.json +++ b/plugins/fxtr/skills/fxtr/provenance.json @@ -2,10 +2,10 @@ "skill": "fxtr", "documentation": "https://docs.transluce.ai/fxtr", "packages": { - "fxtr": "0.0.1a2", - "behaviors": "0.0.1a2" + "fxtr": "0.0.1a4", + "behaviors": "0.0.1a3" }, - "revision": "81bc041101951f16dd70e69340d64e80abd34f01", + "revision": "051a5d3ab1f6df872092af2365b34c5d0f6671d3", "modified": false, - "content_sha256": "84eda286f7e7179dc378d160b5fba2e343752cda4a7f6afc1a7c629aa71c330c" + "content_sha256": "51ba3d88cfa95b80fee856d1d9a8672f7df95e54d5ae269b9ceee5fcee63bbe9" } diff --git a/plugins/fxtr/skills/fxtr/references/guides/running-and-viewing-jobs.mdx b/plugins/fxtr/skills/fxtr/references/guides/running-and-viewing-jobs.mdx index 90cd865..99277ee 100644 --- a/plugins/fxtr/skills/fxtr/references/guides/running-and-viewing-jobs.mdx +++ b/plugins/fxtr/skills/fxtr/references/guides/running-and-viewing-jobs.mdx @@ -42,12 +42,12 @@ if __name__ == "__main__": [(("greeting",), "Hello there"), (("question",), "Why?")], ("[prompt: str]", str), ) - launch_cli(experiment, {"prompts": prompts}, anchor=__file__, max_concurrency=8) + launch_cli(experiment, {"prompts": prompts}, anchor=__file__, max_concurrency=64) ``` `anchor=__file__` tells fxtr which project the launcher belongs to. `max_concurrency` caps how -many steps run at once; workflow functions don't count against it. If the job doesn't succeed, -`launch_cli` exits with status 1. +many steps run at once (see [Choose how many steps run at once](#choose-how-many-steps-run-at-once)). +If the job doesn't succeed, `launch_cli` exits with status 1. For more control, such as choosing the `root`, loading datasets, or reading results, write an async launcher with `open_local_client` and `run_job`: @@ -65,7 +65,7 @@ async def main() -> None: [(("greeting",), "Hello there"), (("question",), "Why?")], ("[prompt: str]", str), ) - async with open_local_client(__file__, max_concurrency=8) as client: + async with open_local_client(__file__, max_concurrency=64) as client: result = await run_job(client, experiment, {"prompts": prompts}, root="word-count-v1") print(list(result.items())) @@ -87,6 +87,11 @@ if __name__ == "__main__": `run_job`. * In a notebook, call `await main()` instead of `anyio.run(main)`. +## Choose how many steps run at once + +`max_concurrency`, on `launch_cli` and `open_local_client`, caps how many of a job's steps run at +once. Note that all jobs run on a single event loop, so CPU-bound work will probably need a lower concurrency limit. + ## Watch jobs in the viewer Start the viewer from the project, in a terminal of its own, and leave it running: