diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 753f1e7..f55282c 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -21,7 +21,7 @@ "name": "fxtr", "source": "./plugins/fxtr", "description": "Skills for writing, running, and viewing fxtr experiments, and for calling models from them with behaviors", - "version": "0.1.0", + "version": "0.1.1", "author": { "name": "TransluceAI" }, diff --git a/.github/scripts/plugin_sanity.py b/.github/scripts/plugin_sanity.py index 09a09d6..44ed98e 100644 --- a/.github/scripts/plugin_sanity.py +++ b/.github/scripts/plugin_sanity.py @@ -99,12 +99,17 @@ def check_docent(plugin_dir: Path) -> None: fail(f"local credential/config file must not be published: {path}") +FXTR_SKILLS = ["behaviors", "first-experiment", "fxtr"] +SKILLS_WITH_REFERENCES = {"behaviors", "fxtr"} + + def check_fxtr(plugin_dir: Path) -> None: - """The fxtr and behaviors skills, as fxtr3's `pnpm sync:plugin` prepares them: both, from - one clean revision of fxtr3, each with its references and links that resolve.""" + """The behaviors, first-experiment, and fxtr skills, as fxtr3's `pnpm sync:plugin` prepares + them: all from one clean revision of fxtr3, with links that resolve. The behaviors and fxtr + skills each hold a copy of the pages they link; first-experiment links into those copies.""" skills = sorted(path.name for path in (plugin_dir / "skills").iterdir() if path.is_dir()) - if skills != ["behaviors", "fxtr"]: - fail(f"fxtr plugin must hold exactly the behaviors and fxtr skills, not {skills}") + if skills != FXTR_SKILLS: + fail(f"fxtr plugin must hold exactly the skills {FXTR_SKILLS}, not {skills}") revisions = set() for name in skills: skill_dir = plugin_dir / "skills" / name @@ -112,7 +117,7 @@ def check_fxtr(plugin_dir: Path) -> None: if not skill_md.is_file() or not skill_md.read_text(encoding="utf-8").strip(): fail(f"{name} skill needs a non-empty SKILL.md") references = [p for p in (skill_dir / "references").rglob("*") if p.is_file()] - if not references: + if not references and name in SKILLS_WITH_REFERENCES: fail(f"{name} skill has no references") for page in references: if page.suffix in {".md", ".mdx"} and not page.read_text(encoding="utf-8").strip(): @@ -135,7 +140,7 @@ def check_fxtr(plugin_dir: Path) -> None: if not path.is_file(): fail(f"{name}/SKILL.md links to a missing file: {target}") if len(revisions) != 1: - fail("the fxtr and behaviors skills must be prepared from the same fxtr3 revision") + fail("the fxtr plugin's skills must be prepared from the same fxtr3 revision") PLUGIN_CHECKS: dict[str, Callable[[Path], None]] = { diff --git a/README.md b/README.md index 351b108..e77765d 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,11 @@ The docent plugin includes two skills: - **analysis** - Analyzing agent behavior with Docent - **ingestion** - Structured workflow for ingesting agent run data into Docent -The fxtr plugin includes two skills, each with a copy of the documentation it links from +The fxtr plugin includes three skills, with a copy of the documentation they link from [docs.transluce.ai](https://docs.transluce.ai): - **fxtr** - Setting up, writing, running, and viewing fxtr experiments - **behaviors** - Calling language models from fxtr experiments with the behaviors library +- **first-experiment** - A guided walkthrough of a first fxtr experiment, from a question to results ## Installation diff --git a/plugins/fxtr/.claude-plugin/plugin.json b/plugins/fxtr/.claude-plugin/plugin.json index 4df0a4e..cd446ad 100644 --- a/plugins/fxtr/.claude-plugin/plugin.json +++ b/plugins/fxtr/.claude-plugin/plugin.json @@ -4,5 +4,5 @@ "author": { "name": "TransluceAI" }, - "version": "0.1.0" + "version": "0.1.1" } diff --git a/plugins/fxtr/skills/behaviors/provenance.json b/plugins/fxtr/skills/behaviors/provenance.json index 1da6c97..e14415c 100644 --- a/plugins/fxtr/skills/behaviors/provenance.json +++ b/plugins/fxtr/skills/behaviors/provenance.json @@ -2,10 +2,10 @@ "skill": "behaviors", "documentation": "https://docs.transluce.ai/behaviors", "packages": { - "fxtr": "0.0.1a0", - "behaviors": "0.0.1a0" + "fxtr": "0.0.1a2", + "behaviors": "0.0.1a2" }, - "revision": "8d820d77c1292b4792602cbbe5a9876693de3d99", + "revision": "81bc041101951f16dd70e69340d64e80abd34f01", "modified": false, - "content_sha256": "1c5471096fa21b00e363ac7f69f33bb54d19ade3998658a937328b980645485b" + "content_sha256": "de0ca958d570464776b006b595d0f26b7be280a6c5baf88c46629d2982757eee" } diff --git a/plugins/fxtr/skills/behaviors/references/INDEX.md b/plugins/fxtr/skills/behaviors/references/INDEX.md index 0e8a13c..e5f4a36 100644 --- a/plugins/fxtr/skills/behaviors/references/INDEX.md +++ b/plugins/fxtr/skills/behaviors/references/INDEX.md @@ -6,7 +6,7 @@ in the order of the site's navigation. ## Get started -- [Introduction](index.mdx): Primitives for evaluating AI model behavior. +- [What is the behaviors library?](index.mdx): Primitives for evaluating AI model behavior. ## Other pages diff --git a/plugins/fxtr/skills/behaviors/references/guides/language-models.mdx b/plugins/fxtr/skills/behaviors/references/guides/language-models.mdx index 7c0c4f0..2e9af1e 100644 --- a/plugins/fxtr/skills/behaviors/references/guides/language-models.mdx +++ b/plugins/fxtr/skills/behaviors/references/guides/language-models.mdx @@ -22,8 +22,8 @@ uv add "behaviors[models]" Then commit the updated `pyproject.toml` and `uv.lock`. -Set your provider's API key in the environment: `OPENAI_API_KEY` or `ANTHROPIC_API_KEY`. Never -store credentials in experiment inputs or entities. +Set your provider's API key, `OPENAI_API_KEY` or `ANTHROPIC_API_KEY`, in the project's `.env` +(see [Project configuration](../../../fxtr/references/reference/project-configuration.mdx#env)) or in the environment. | Provider | Adapter | | ---------------------------------- | ------------------------------------------------------------------------------------ | diff --git a/plugins/fxtr/skills/behaviors/references/index.mdx b/plugins/fxtr/skills/behaviors/references/index.mdx index 34e656c..0108759 100644 --- a/plugins/fxtr/skills/behaviors/references/index.mdx +++ b/plugins/fxtr/skills/behaviors/references/index.mdx @@ -1,5 +1,5 @@ --- -title: "Introduction" +title: "What is the behaviors library?" description: "Primitives for evaluating AI model behavior." --- diff --git a/plugins/fxtr/skills/first-experiment/SKILL.md b/plugins/fxtr/skills/first-experiment/SKILL.md new file mode 100644 index 0000000..7682774 --- /dev/null +++ b/plugins/fxtr/skills/first-experiment/SKILL.md @@ -0,0 +1,307 @@ +--- +name: first-experiment +description: Walk someone new to fxtr through their first experiment with language models, from their own question to results in the viewer. The user proposes a question about model behavior; you help scope it, design it with them, set up the project, write the experiment with behaviors, run a small pilot and then the full run, and change it with their input. Use when the user asks for a walkthrough, tutorial, or guided start with fxtr, asks for help with their first fxtr experiment, or invokes this skill. For later work in a project they already know, use the fxtr and behaviors skills instead. +--- + +# Your first experiment + +You are guiding someone through their first fxtr experiment. They bring the question; you help +them turn it into an experiment that calls language models, build it, run it, and read what it +found, explaining the fxtr ideas as each one comes up. By the end they should have results they +care about and know where each part of the experiment lives, so they can change it themselves. + +This skill sets the order of the work and when to check in with the user. How to write each part +is in the adjacent [fxtr skill](../fxtr/SKILL.md) and [behaviors skill](../behaviors/SKILL.md): +read the pages they point to for each task, and follow their best practices. +[Your first experiment](../fxtr/references/first-experiment.mdx) shows an example of a first +experiment. By default you should follow along with this example, but if the user has a different +question they want to study or decides to deviate from the example in some other way, you should +follow their lead and just use this example as a guide for the pace and size of a good first +experiment. Even if you do follow this example, you should guide the user to customize the experiment +design as you go; don't just copy the code exactly! + +Before you start reading the docs and testing out their workflow, please send the user a short +message like this: + +> Welcome to fxtr's "Your first experiment" guide! I'll start by reading the documentation +> and checking your environment. Then we'll decide what question to study, and I'll guide you +> through how to set up your first experiment with fxtr. + +## How to guide + +* **Guide the user through each choice and unit of work.** The point of this guided walkthrough + is not to just write an experiment, it is to use this experiment as a way to teach the user + about the pieces of fxtr and how they work. By default, you should pause often, explaining + what you are about to do and pointing out the relevant concepts and guides that the user can + read. +* **Ask a few questions at a time**, and offer concrete options with a recommended default. + Someone new to fxtr can choose between options more easily than they can fill in a blank. +* **Teach in passing.** When a fxtr idea first matters (a step, a dimension, the cache), explain + it in a sentence or two in terms of their experiment, and link the page that covers it. Don't + front-load concepts they haven't needed yet. +* **Keep the first version small.** Aim for an experiment of about one screen of design that + answers one question: a few conditions, ten to fifty items, and one measurement. Record bigger + ideas as next steps rather than building them now. +* **Ask before spending money.** Before any launch that calls a paid API, say how many model + calls it makes (conversations times turns, plus judge calls) and get the user's go-ahead. + Repeat this for the full run. +* **Report plainly.** Say what ran, what was reused, and what failed, with the job ID. Make sure + to ground your summaries in the actual results of the experiment (e.g. the report entity) + rather than by manually adding up printouts. + +## 1. Check the setup + +Before talking design, check that the user can run a job, and tell them what's missing. If a +check fails with a permission error, the cause may be your sandbox rather than their machine +(a sandbox can block Docker's socket, local ports such as the database's, and caches outside +the project): ask for the permission and rerun the check before reporting anything missing. + +* The tools in [Installation](../fxtr/references/installation.mdx#prerequisites): uv, Docker (or + another Postgres server), Node, pnpm, and git. Check their versions. +* A Postgres server that fxtr can reach. If they have none, offer to start the container from + [Start Postgres](../fxtr/references/installation.mdx#start-postgres). +* An API key for at least one provider: `OPENAI_API_KEY` or `ANTHROPIC_API_KEY` in the + environment or in a project's `.env`, or another provider's key for + [OpenRouter](../behaviors/references/guides/choosing-models.mdx#openrouter). Check whether a key + is set without printing it, in a way every shell understands: `printenv NAME > /dev/null` for + the environment, `grep -qE '^NAME=.+' .env` for a `.env`. If none is set yet, ask which + provider they will use along with their question in step 2, since it decides which models + and which judge the design can use. They can add the key while you write the code; nothing + needs it before the pilot. +* Whether the working directory is already in a fxtr project (a `pyproject.toml` with a + `[tool.fxtr]` table). If it is, offer to add the experiment there; otherwise you will create a + project in step 3. + +## 2. Find the question and design the experiment + +**Decide on a question to answer.** Ask the user whether they'd like to follow along with the +experiment in the "first-experiment" guide, or if they want to choose a different idea +to study. Please suggest these five other ideas for inspiration, and note that they can also +pick their own question to answer if they have something else in mind. + +* **Self portraits**: What kinds of art (text or visual) do models make if asked to make art + about themselves, or to illustrate themselves? +* **Responding to a strange statement**: How do different models respond when you send them a + short message out of context (like "I killed an ant" from this post + )? +* **Prompt sensitivity**: How do differences in a user's message or the system prompt affect + how the model behaves? +* **Consistency under pressure**: Does a model keep their original answer when the user pushes back? +* **Responding to repetition**: What happens when the user keeps sending the same message + to the model over and over? + +A good first experiment: + +* is self-contained (doesn't require external data) +* compares a few models, with multiple samples from each +* involves generating some transcripts, measuring something about those transcripts, and then + either summarizing or visualizing the results + +If their idea needs no model calls, or needs things a first experiment shouldn't take on +(fine-tuning, long agentic tasks, large external datasets), say so and help them find the part +that fits: one question, answered with model calls. + +**Pin down the design.** Settle these with them, proposing defaults: + +1. **What variables are we measuring differences between?** Often, the best variable to study + is the model. However, the user might instead want to study differences in system prompt, or + other differences about the scenario. In the initial pilot of the experiment, you should include + at least two (and usually at least three) conditions that you expect to produce different + results (e.g. a small model and a larger one, or two system prompts on opposite extremes). +2. **What other variables are we sweeping across, if any?** To learn something about how a model + behaves in general, it's usually a good idea to measure that behavior across multiple conditions. + For instance, when evaluating humor, we might want to sweep across joke topics or joke formats. When + evaluating sensitivity to a system prompt, we might want to aggregate over multiple user prompts. + When generating art, we might want to sweep over suggested styles for the art. However, if the task + is simple, it might also be OK to not sweep over anything to start (e.g. just try with one prompt, or + don't give any suggested styles for some art). In this case, you should still propose a possible + dimension to sweep across, and just populate it with a single entry; this makes it possible to extend + it with new values later. +3. **How do we generate the transcript?** For some tasks, it might be enough to just run a single turn + of a conversation with a fixed prompt. More advanced tasks might require multiple turns or tools. This + will become the logic in the step function that generates the transcripts. +4. **What are we measuring?** To make a quantitative comparison between the variables we are studying, + we need some way to measure differences in behavior. In some cases, this can just be a pure function + (e.g. response length or accuracy), but usually this will involve running a judge or grader model with + a rubric that returns a verdict. Judge models should be asked for a rationale or explanation before + giving a final answer, which should be either a numeric score or a set of named categories. XML + format for structured outputs is a good default. Note that for some exploratory tasks (like seeing + how models respond to an open-ended prompt), it might make sense to + first generate some initial data and then look at it manually before proposing what the actual + rubric or set of outcomes should be! +5. **What data types will we use to represent the experiment?** It usually makes sense to define some + new entity types to hold the results of various parts of your experiment. For some parts of the experiment, you might be able to use the types from the `behaviors` library; in this case you should + just use these. However, other things such as experiment configuration, judge results, and the final + report object should usually be experiment-specific types. By default you should use + "local.{projectname}.{TypeName}" for experiment-specific types (since there is not yet a global + registry of types). + +Show the design in one short block before writing code, along with +what each part becomes in fxtr. For example: + +```text +Question: which models write the funniest jokes? +Main variable of interest: model: haiku-4.5, sonnet-5.5 +Variables to sweep over: topic: 10 everyday topics (coffee, dentist, printer, ...) +Conversation: one user message asking for a joke about the topic +Measurement: a judge (opus-5.5) rates each joke from 1 to 10, with a rationale first +Samples: sample: 3 per model and topic +Summary: per model, the mean score +Size: pilot 2 x 3 x 1 = 6 conversations (6 model calls) + 6 judge calls + full 2 x 10 x 3 = 60 conversations (60 model calls) + 60 judge calls + +New entity types to define: +- GenerationConfig: provider, model_name, sampling, prompt_template +- JudgeConfig: provider, model_name, sampling, rubric +- JokeGrade: topic, joke, score, rationale, judge_reply +- JokeReport: references to the jokes, grades, and summary arrays + +Structure of the experiment workflow: +- a joke step mapped over [model, topic] replicated over [sample], returning a ChatRollout +- a judge step mapped over [model, topic, sample] +- a summary step mapped over [model] +``` + +Iterate with the user on this until they are satisfied with the design. + +Once you have a design, you can ask the user how they want to proceed with the rest of the steps. +By default you can recommend a step-by-step walkthrough where you explain each unit of work before +you do it, but the user might ask for you to implement everything at once and just explain it at +the end. + +## 3. Set up the project + +If they have no project, ask where to create it (possibly current working dir or `~/fxtr/`), +then create it, install its dependencies, build its renderers, and connect it to the database as +[Run an example](../fxtr/references/installation.mdx#run-an-example) does, without `--example`. +Then add behaviors as +[Calling language models](../behaviors/references/guides/language-models.mdx#set-up) shows. Have +the user put their keys in the project's `.env` themselves, or confirm that the environment +already has them; never write a key into a file, an input, or an entity. Commit. + +`fxtr init` uses the schema `fxtr_`, and if an earlier project of the same name +used it, its jobs and cache are still there. Run `uv run fxtr jobs list` after `fxtr init`. If it +lists anything, tell the user, and offer a fresh schema +(`uv run fxtr init --database-url URL --schema NEW_NAME --force`) rather than deleting anything. +If they ask you to delete data anyway, name exactly what would go, confirm it, and never touch +another project's schema. + +Tell them in two or three sentences what the project holds, using +[Project configuration](../fxtr/references/reference/project-configuration.mdx#what-fxtr-new-writes): +the package for experiment code, the renderer package `views/`, `.env`, and `fxtr.local.toml`. + +## 4. Write the experiment + +Read [Workflows and steps](../fxtr/references/concepts/workflows-and-steps.mdx), +[Arrays and parallel computations](../fxtr/references/concepts/arrays-and-parallel-computations.mdx), +and [Calling language models](../behaviors/references/guides/language-models.mdx). + +Then tell the user what you are about to do, and then write the following: + +* **One module** for the experiment, registered in `[tool.fxtr] modules`, holding its + configuration entities, its verdict and report entities, its steps, and its workflow, shaped as + the behaviors skill's best practices say. Pass each step only the fields it uses: a + conversation step that receives the question's text but not its reference answer doesn't rerun + when a reference answer is corrected, and only the judge does. +* **A launcher** at the project root that builds the items, stores a configuration per condition, + runs the workflow, and prints the summary from the report in an `on_result` callback + ([Read results](../fxtr/references/guides/running-and-viewing-jobs.mdx#read-results)), so its + output ends with the answer rather than the report's ID. Define the pilot and full sizes side + by side and choose one with an argument (`uv run python launch.py pilot`), so the full run is + the same launcher with another word. + +Check that the module imports cleanly, and type-checks if the project has a type checker. + +Unless the user has said to do everything at once, this is a good point to pause and explain what you've +implemented so far. Walk the user through the files in a few lines each: which function is +which step, which dimension each one maps over, where the prompts and the rubric live, and where +the sizes are set. Point out what parts live in the steps, what parts live in the workflow, and what +parts live in the launcher. + +Next, you can offer to test out the experiment logic by running it with a fake model, and ask the +user if they want you to do this. If so, you can point the project at a scratch schema +(set `schema` in `fxtr.local.toml`, as +[Before you launch](../fxtr/references/guides/running-and-viewing-jobs.mdx#before-you-launch) +says), launch the pilot with `make_api` returning a fake model, and set the schema back. This +catches mistakes in the steps, the workflow, the launcher, and the summary before any of them +costs money, and nothing from it lands in the real results. + +Finally, you can recommend writing some custom renderers to help visualize the data in the experiment. +You can use the existing `behaviors` renderers for conversations, and make a proposal for how +to write renderers of + +* the entities you introduced +* step invocations (if there is something more to show than just the array contents) +* a result overview that answers the user's question. + +You can follow the fxtr skill's best practices to implement these views. By default, you should write +all of the renderers before launching the real pilot experiment, so that the user has something to +look at while the job runs. + +## 5. Run the pilot + +Start the viewer in the background as the fxtr skill says, then confirm the pilot's size and +number of model calls with the user, commit, and launch it under a `root` named for the experiment +(such as `jokes-v1`). The full run uses the same `root`, so it reuses the pilot's results. + +While it runs, tell the user what they're looking at: their viewer has switched to the job, with +a card per step that fills in as conversations finish. + +When it finishes, report the job's ID and status and the summary the +launcher printed. You can guide the user to open a few of the results in the viewer to see what +happened. + +You can also spot-check the results to check if they match the initial hypothesis about what +would happen, or if there are any problems with the experiment. Any possible problems or refinements +you notice can be suggestions for the next step. + +## 6. Change it with the user + +Ask what they'd like to change. If the pilot has surfaced problems or improvements, you can suggest +making those improvements. If it looks good, you can also suggest expanding the pilot to a full +run over all of the conditions. + +Before relaunching, tell them which results the next launch will reuse and which it will compute, following +[Managing the step cache](../fxtr/references/concepts/managing-the-step-cache.mdx). The common +changes on a first experiment: + +* **Scaling up** (more items or samples): every pilot conversation and verdict is reused, and only + the new ones run. Steps that reduce over a dimension that grew, such as the summary, are at the + same address with new inputs, so the job stops on a cache conflict there. That is expected; + explain it, then resume with `--overwrite-cache-conflicts`, since the pilot's summaries aren't + wanted. A resume doesn't run the launcher's printing, so launch once more afterwards: every + step is cached, so it makes no model calls and prints the summary at once. +* **Adding a condition** (a model, a phrasing): add it as a new key of its dimension. The existing + conditions' results are reused, and the new key is computed beside them, so the two can be + compared; a step that reduces over that dimension conflicts, as in scaling up. (If the condition + needs a dimension the experiment doesn't have yet, tell the user that adding that dimension will + recompute every step mapped over it, and ask the user if they prefer to just clear the old + results or if you should use a new `root` name to keep both versions in the cache.) +* **Changing an input in place** (editing a prompt, the rubric, a sampling setting): the steps + that receive it get new inputs at the same addresses and stop on cache conflicts. If the old + results should be kept for comparison, make the change a new condition instead; otherwise + explain the choices in + [Resolving a job's conflicts](../fxtr/references/concepts/managing-the-step-cache.mdx#resolving-a-jobs-conflicts) + and let the user pick. +* **Adding a new step at the end** (e.g. for a new analysis): all previous steps will hit the cache, + so you can just insert the new step into the workflow in place. +* **Changing a step's code** (how answers are parsed or scored): the cache won't notice. Clear + that step's entries, or add a configuration field, as + [Handling changes to step logic](../fxtr/references/concepts/managing-the-step-cache.mdx#handling-changes-to-step-logic) + says, and tell the user which results that recomputes. + +Summarize for the user what each change will mean about the number of new model calls before each +launch, based on the current design and the state of the cache. Report each run as in step 5. + +## 7. Figure out next steps + +From here, you should ask the user what they want to do next, and help them do it. Some possible +next steps include: + +* Update the overview renderer to go into more detail about what was found, or make it easier for + people to explore the results. +* Discuss and explain some of fxtr's core concepts in more detail, building on the understanding + from this experiment. +* Read over the generated data to refine the measurement rubrics or propose new things to measure. +* Study a follow-up question based on something you or the user noticed. +* Start a new experiment to answer a different question, based on what the user is interested in. diff --git a/plugins/fxtr/skills/first-experiment/provenance.json b/plugins/fxtr/skills/first-experiment/provenance.json new file mode 100644 index 0000000..b924a56 --- /dev/null +++ b/plugins/fxtr/skills/first-experiment/provenance.json @@ -0,0 +1,11 @@ +{ + "skill": "first-experiment", + "documentation": null, + "packages": { + "fxtr": "0.0.1a2", + "behaviors": "0.0.1a2" + }, + "revision": "81bc041101951f16dd70e69340d64e80abd34f01", + "modified": false, + "content_sha256": "1b39e083738e6d10b515568bf7912241a1874cda150eb40908b92e7927d38b5f" +} diff --git a/plugins/fxtr/skills/fxtr/SKILL.md b/plugins/fxtr/skills/fxtr/SKILL.md index d530db4..b2208e5 100644 --- a/plugins/fxtr/skills/fxtr/SKILL.md +++ b/plugins/fxtr/skills/fxtr/SKILL.md @@ -33,7 +33,7 @@ usually means reading Workflows and steps and Arrays and parallel computations. | Task | Read | | ------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------- | -| See a small experiment written, launched, and rerun end to end | [Your first experiment](references/first-experiment.mdx) | +| Guide someone new to fxtr through designing and running their first experiment | the adjacent [first-experiment skill](../first-experiment/SKILL.md) | | Write or change steps and workflows: signatures, determinism, hermeticity, failures | [Workflows and steps](references/concepts/workflows-and-steps.mdx) | | Shape data and sweeps: arrays, handles, mapping, alignment, reductions, replicas | [Arrays and parallel computations](references/concepts/arrays-and-parallel-computations.mdx) | | Filter, select, aggregate, or join arrays, or write SQL over them, in a workflow or a step | [Array queries](references/reference/array-queries.mdx) | @@ -97,37 +97,42 @@ before [adding one](references/reference/renderers.mdx#adding-a-package-to-a-pro many fields, add an `EntityHoverPreview` showing the few that identify it. Records from behaviors already have renderers; the [behaviors skill](../behaviors/SKILL.md) says how to include them. -* **Give every step you write an `InvocationPanel`**, registered under the step's name. It - becomes the step's default view, in its pane and in its card's preview, and it is handed every - invocation the viewer's selection picks out, up to all of a mapped step's. Show one in full and - several together (a table with a row per key, a chart, or a summary), loading their outputs - with `LoadEach` ([Step panels](references/reference/renderers.mdx#step-panels)). +* **Give each step an `InvocationPanel`**, registered + under the step's name, whenever there is something useful to show beyond the output array itself. The step's pane already shows its arrays as tables, so don't redraw them + ([Where a view appears](references/guides/designing-views.mdx#where-a-view-appears)). + The panel becomes the step's default view, in its pane and in its card's preview, and it is + handed every invocation the viewer's selection picks out, up to all of a mapped step's. Show + one in full and several together (a chart or a summary), loading their outputs with `LoadEach` + ([Step panels](references/reference/renderers.mdx#step-panels)). Wrapping a view of one invocation in `singleInvocation` asks the reader to select one when there are several, so keep it for the root workflow and for steps that are neither mapped nor replicated. * **Give the root workflow a results overview**: an `InvocationPanel` that reads the workflow's report and presents the experiment's findings ([Results overviews](references/concepts/visualization-and-custom-renderers.mdx#results-overviews)). -* **Check a view after changing it**, as [Check it](references/guides/designing-views.mdx#check-it) - describes: both themes, every width, the keyboard, its links, and the search for literal colors - and sizes. +* **Check a view without opening it.** After changing a view, confirm it builds and run the + searches in [Check it](references/guides/designing-views.mdx#check-it), then tell the + user it is ready to look at in their viewer. Don't open the viewer in a browser yourself, or set + up Playwright or a headless browser to screenshot it, unless the user asks you to. ### Running jobs -* **Start the viewer before a launch.** When the user asks for a job to run, first start - `uv run fxtr view` in the background, as a long-running process of its own that you don't wait - on. When the project's viewer already runs, this only opens it, so it is safe every time. Each - launch then opens its job in a new tab of the viewer: tell the user it is there, and report the - job's ID and status from the launch's output. A launch that finds no viewer prints - `no viewer is running for this project` with the command to start one: start it then. +* **Start the viewer once.** The first time the user asks for a job to run, start + `uv run fxtr view` in the background, as a long-running process of its own (don't wait for it to + exit), and before the launch wait until it prints `the viewer is at `. It opens the viewer + in the user's browser. Don't run `fxtr view` again for later launches: each run opens another + tab. While it runs, each launch shows its job in the viewer the user has open, or in a new tab + when none is open: tell the user it is there, and report the job's ID and status from the + launch's output. A launch that finds no viewer prints `no viewer is running for this project` + with the command to start one: start it then. * **Keep the viewer running** across launches and renderer edits. It rebuilds the renderers when their sources change, and the open viewer switches to them in place, so never ask the user to reload or restart it. ### Reading the documentation -* **Know which documentation you have.** Distributed in the Claude plugin, this skill carries - a copy of the pages, and `provenance.json` beside this file records the fxtr and behaviors - versions they describe. If the project depends on another fxtr version (its `uv.lock` says - which), tell the user. The plugin has its own version and update cycle; docs.transluce.ai has +* **Know which documentation you have.** Distributed in the fxtr plugin for Claude Code or + Codex, this skill carries a copy of the pages, and `provenance.json` beside this file records + the fxtr and behaviors versions they describe. If the project depends on another fxtr version + (its `uv.lock` says which), tell the user. The plugin has its own version and update cycle; docs.transluce.ai has the newest pages. diff --git a/plugins/fxtr/skills/fxtr/provenance.json b/plugins/fxtr/skills/fxtr/provenance.json index 99205b7..676077f 100644 --- a/plugins/fxtr/skills/fxtr/provenance.json +++ b/plugins/fxtr/skills/fxtr/provenance.json @@ -2,10 +2,10 @@ "skill": "fxtr", "documentation": "https://docs.transluce.ai/fxtr", "packages": { - "fxtr": "0.0.1a0", - "behaviors": "0.0.1a0" + "fxtr": "0.0.1a2", + "behaviors": "0.0.1a2" }, - "revision": "8d820d77c1292b4792602cbbe5a9876693de3d99", + "revision": "81bc041101951f16dd70e69340d64e80abd34f01", "modified": false, - "content_sha256": "ddeb8de1de321e0f5513f8fc621795b6176f5e95b29a3cac6f0725840784b7ab" + "content_sha256": "84eda286f7e7179dc378d160b5fba2e343752cda4a7f6afc1a7c629aa71c330c" } diff --git a/plugins/fxtr/skills/fxtr/references/INDEX.md b/plugins/fxtr/skills/fxtr/references/INDEX.md index d5302ea..a330ac6 100644 --- a/plugins/fxtr/skills/fxtr/references/INDEX.md +++ b/plugins/fxtr/skills/fxtr/references/INDEX.md @@ -6,9 +6,9 @@ in the order of the site's navigation. ## Get started -- [Introduction](index.mdx): fxtr is an experiment orchestration system for verifiable AI-powered analyses -- [Installation](installation.mdx): Install fxtr from PyPI and verify the setup with an example project. -- [Your first experiment](first-experiment.mdx): Write a small experiment with a mapped step and a reduction, launch it, and see how caching works. +- [What is fxtr?](index.mdx): fxtr is an experiment orchestration system for verifiable AI-powered analyses +- [Installation](installation.mdx): Install fxtr and get started with an example project +- [Your first experiment](first-experiment.mdx): Design, build, run, and change your first fxtr experiment: a small evaluation of how language models behave. ## Concepts diff --git a/plugins/fxtr/skills/fxtr/references/first-experiment.mdx b/plugins/fxtr/skills/fxtr/references/first-experiment.mdx index 36048c5..edd7804 100644 --- a/plugins/fxtr/skills/fxtr/references/first-experiment.mdx +++ b/plugins/fxtr/skills/fxtr/references/first-experiment.mdx @@ -1,250 +1,1026 @@ --- title: "Your first experiment" -description: "Write a small experiment with a mapped step and a reduction, launch it, and see how caching works." +description: "Design, build, run, and change your first fxtr experiment: a small evaluation of how language models behave." --- -This tutorial starts with an empty project and builds an experiment that counts the words in -each prompt in a dataset, then averages the counts. There are no model calls. It has the same -shape as many experiments: a step mapped over a dataset, followed by a step that reduces the -results. +This guide walks you through a first experiment in fxtr: a small evaluation of how language +models behave. You'll design the experiment, write its code, run a small pilot, look at the +results in the viewer, and then grow it. Along the way you'll meet the main ideas in fxtr: +entities, steps, workflows, arrays, and the step cache. -This assumes you have completed [Installation](installation.mdx) and have a database running. +The example experiment in this guide evaluates language models writing jokes, but you can study anything you'd like! -## Create a project and connect it to the database + + We strongly recommend walking through this experience interactively with an AI agent. To try it out, invoke the `first-experiment` skill: -Create a new project in a new or empty directory. Use your existing database's URL; the command -below uses the local database from the installation guide: + + + From a terminal, if you haven't already: + + ```bash + claude plugin marketplace add TransluceAI/claude-code-plugins + claude plugin install fxtr@transluce-plugins + ``` + + Then, in Claude Code: + + ```text + /fxtr:first-experiment + ``` + + + + From a terminal, if you haven't already: + + ```bash + codex plugin marketplace add TransluceAI/codex-plugins + codex plugin add fxtr@transluce-plugins + ``` + + Then, in Codex: + + ```text + $fxtr:first-experiment + ``` + + + + +## 1. Check your setup + +Before you start, complete [Installation](installation.mdx), and check that you have: + +* **The tools**: uv, Docker (or another Postgres server), Node, pnpm, and git, at the versions + [Installation](installation.mdx#prerequisites) lists. Check with `uv --version`, + `node --version`, and `pnpm --version`. +* **A running Postgres server**: `docker ps` lists the `fxtr-postgres` container from + [Start Postgres](installation.mdx#start-postgres). If it's stopped, run + `docker start fxtr-postgres`. +* **An API key** for each model provider you'll call: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or + both. The running example uses Anthropic models, but its code pairs every model with its + provider, so switching to OpenAI or mixing the two is a change to a few lines (see + [Write the launcher](#write-the-launcher)). For other providers, see + [OpenRouter](../../behaviors/references/guides/choosing-models.mdx#openrouter). + +## 2. Design your experiment + +For your first experiment in fxtr, you can start with a question about how models behave, such as: + +* Do models keep a correct answer when the user pushes back? +* How does the wording of a system prompt change how a model responds? +* What do models make when asked to draw or describe themselves? + +A good first question needs no external data, compares a few conditions (often a few models), +and can be measured from the conversations themselves. Once you have one, decide: + +1. **What you're comparing.** Usually this is models, but it could be system prompts or other + settings. Pick at least two conditions you expect to differ. +2. **What you're sweeping over.** One prompt tells you little about a model in general, so + measure the behavior across several variations, such as topics, phrasings, or tasks, and take + several samples of each. +3. **How you generate a transcript.** Often one turn of conversation with a fixed prompt is + enough. More involved experiments use several turns or tools. +4. **What you measure.** A pure function of the transcript, such as its length or whether an + answer is correct, or more often a judge model that follows a rubric. Have the judge explain + its reasoning before it gives a score or a label. +5. **How you summarize.** For example, a mean score or the share of each label per condition. + +You should generally start small: a few conditions, a handful of items, and one measurement. Once you have something simple working, you can scale it out. + +### The running example + +The rest of this guide follows one question: *which language models write the funniest jokes?* +Its design: + +```text +Question: which models write the funniest jokes? +Compare: model: Haiku 4.5, Sonnet 5.5 +Sweep over: topic: 10 everyday topics (coffee, dentists, printers, ...) + sample: several jokes per model and topic +Conversation: one user message asking for a joke about the topic +Measurement: a judge model (Opus 5.5) rates each joke from 1 to 10, with a rationale +Summary: the mean score per model +Size: pilot 2 models x 3 topics x 1 sample = 6 jokes, 6 judge calls + full 2 models x 10 topics x 3 samples = 60 jokes, 60 judge calls +``` + +### From design to code + +Each part of a design becomes a piece of fxtr code: + +* **Entities**: Data records that represent objects in your program, such as configuration + objects, measurement results, or a report that collects the final results. Each entity + type has a unique type ID, which fxtr uses to visualize it. +* **Steps**: The functions that do the work. For our joke evaluation: `tell_joke` asks one model for one joke, + `grade_joke` grades one joke, and `summarize` averages one model's grades. +* **Arrays** hold many values indexed by named **dimensions**, including the variables you are comparing and those you are sweeping over. Each step is *mapped* over some of them to run computations in parallel. For instance, in the joke evaluation, `model`, + `topic`, and `sample` are dimensions, and fxtr calls `tell_joke` once per model, topic, and + sample. +* A **workflow** connects the steps, and a **launcher** script supplies its inputs and runs it + as a **job**. + +You can read more about these pieces in [Core concepts](concepts/overview.mdx). + +## 3. Set up the project + +Create a project and install its dependencies. The commands below use the name +`joke-eval` (feel free to substitute your own). ```bash -uvx fxtr new ~/code/my-project -cd ~/code/my-project +uvx fxtr new ~/fxtr/joke-eval +cd ~/fxtr/joke-eval uv sync +uv add "behaviors[models]" (cd views && pnpm install) +``` + +`fxtr new` writes an empty project and makes it a git repository. +[behaviors](../../behaviors/references/index.mdx) is our library for calling language models in fxtr, which your steps can use. + +Next, connect the project to your database: + +```bash uv run fxtr init --database-url postgresql+asyncpg://fxtr:fxtr@localhost:55433/fxtr ``` -`fxtr new` creates an empty Python package under `src/my_project/`, project configuration, and -an empty renderer bundle, and initializes git. `uv sync` installs the project's Python -dependencies. There is no experiment or launcher yet; you will write both below. -The `pnpm install` command installs dependencies for the project's renderer bundle; commit its -lockfile with your experiment. +Then add your keys to the project's `.env`, which is gitignored: -`fxtr init` writes the database connection to `fxtr.local.toml` and creates the project's -tables. That file is gitignored because the connection is specific to this machine. Run the -remaining commands from the new project directory. +```bash .env +ANTHROPIC_API_KEY=sk-ant-... +``` -## Write the experiment +The project now holds: -Create `src/my_project/word_count.py`: +```text +pyproject.toml the project, and [tool.fxtr]: the modules that define its steps and workflows +.env API keys, read into the environment of each launch; never committed +fxtr.local.toml where this machine reaches the database; never committed +src/joke_eval/ the Python package your experiment goes in (named after the project) +views/ renderers that customize how the viewer shows your data +``` -```python src/my_project/word_count.py -from typing import Annotated +See [Project configuration](reference/project-configuration.mdx) for more details on the project structure. -from fxtr.entity_defns.array import Array -from fxtr.experiment.handles import ArrayHandle -from fxtr.experiment.steps import StepContext, step -from fxtr.experiment.workflows import WorkflowContext, workflow +Next, make an initial commit: +```bash +git add . && git commit -m "A new fxtr project" +``` + +## 4. Write the experiment + +Experiments in fxtr generally consist of at least two files: + +* the *experiment module*, a library that defines the main experiment workflow and other core components, +* the *launcher*, which specifies what the experiment runs on and is responsible for launching it in fxtr. + +If you want to get started immediately, you can skip ahead to the complete +[experiment module](#put-it-all-together-into-an-experiment-module) and +[launcher](#write-the-launcher) for the joke eval. Below we'll walk you through the pieces and +what they mean. -@step(name="word_count.count") -async def count_words(context: StepContext, prompt: str) -> int: - """Count the whitespace-separated words in a prompt.""" - return len(prompt.split()) +### Define the entities +An entity is a frozen dataclass that fxtr can store and show in the viewer. Each gets a type name +of its own. You'll usually want: -@step(name="word_count.mean") -async def mean_length( - context: StepContext, - counts: Annotated[Array[int], "[prompt: str]"], -) -> float: - """Average the word counts across prompts.""" - values = list(counts.values()) - return sum(values) / len(values) +* a **configuration** for each kind of model call, holding every setting that affects its output, + so the results always point at the exact settings that produced them; +* a **measurement** entity, such as a grade or a verdict, with the judge's rationale and its full + reply; +* a **report** that references everything the experiment produced, so the launcher, and later a + results view, can find it all from one place. + +You can also use existing entities like `ChatRollout` from `behaviors`. + +Here's an example configuration entity for how to generate a joke from a topic: + +```python +@dataclass(frozen=True) +class GenerationConfig(DataclassEntity, fxtr_type="local.joke_eval.GenerationConfig.v1"): + """A model asked for jokes, and the prompt it is asked with.""" + + provider: Provider + model_name: str + sampling: SamplingParams + prompt_template: str # formatted with {topic} +``` +You can expand below to see the other types we'll need: -@workflow(name="word_count.experiment") -async def experiment( + + ```python + @dataclass(frozen=True) + class JudgeConfig(DataclassEntity, fxtr_type="local.joke_eval.JudgeConfig.v1"): + """A judge model and the rubric it grades jokes with.""" + + provider: Provider + model_name: str + sampling: SamplingParams + rubric: str # formatted with {topic} and {joke} + + + @dataclass(frozen=True) + class JokeGrade(DataclassEntity, fxtr_type="local.joke_eval.JokeGrade.v1"): + """The judge's grade of one joke.""" + + topic: str + joke: str + score: int | None # 1 to 10; None when there was no joke or the reply had no valid score + rationale: str + judge_reply: str + + + class ModelSummary(TypedDict): + """How one model's jokes were graded. A TypedDict isn't an entity: it's stored inline, as a + value in an array.""" + + jokes: int + graded: int + mean_score: float | None + + + @dataclass(frozen=True) + class JokeReport(DataclassEntity, fxtr_type="local.joke_eval.JokeReport.v1"): + """Everything the evaluation produced. Each field references a stored array; the comments + give each array's dimensions.""" + + jokes: BoundID[Array[BoundID[ChatRollout]]] # [model, topic, sample] + grades: BoundID[Array[BoundID[JokeGrade]]] # [model, topic, sample] + summary: BoundID[Array[ModelSummary]] # [model] + ``` + + +See [Entities and custom types](concepts/entities-and-custom-types.mdx) for info on how to +define entities. + +### Write the steps + +A step is an async function decorated with `@step`. fxtr stores each call's result in its +**cache**, and reuses it when the same call comes up again. Write one step per call you'd make by +hand: one conversation, one judgment, one summary. fxtr handles running them many times. + +For our joke evaluation, we can use three steps: + +* `tell_joke(config, topic)` formats the prompt and holds a one-turn conversation with + `run_chat_rollout`. It returns the `ChatRollout`, a behaviors entity holding the whole + conversation. +* `grade_joke(judge, topic, rollout)` takes the joke's text from the rollout, sends the rubric to + the judge, and reads the `` and `` tags from its reply. A missing joke or an + unreadable score is recorded as a grade with no score. A failed model call raises, so that + fxtr retries it when you resume the job. +* `summarize(grades)` receives one model's grades, as an array over `topic` and `sample`, and + averages their scores. + +Here's the step that generates a joke: + + + ```python + @step(name="joke_eval.tell_joke") + async def tell_joke(context: StepContext, config: GenerationConfig, topic: str) -> ChatRollout: + """Ask a model for a joke about the topic.""" + prompt = config.prompt_template.format(topic=topic) + api = make_api(config.provider, config.model_name) + try: + # The model to talk to, which retries rate limits and server errors. + model = chat_session_model_from_api(api, config.sampling, retry=RetryConfig()) + # The user's side of the conversation: one scripted message. + user = ScriptedUserSimulator(messages=(UserMessage(content=(ContentText(text=prompt),)),)) + policy = context_policy_from_user_simulator(user, initialization=None) + # Run the conversation. One more turn than there are scripted messages, so the rollout + # can end as completed. The returned ChatRollout is an entity, so fxtr stores it. + return await run_chat_rollout(model, policy, max_turns=2) + finally: + with anyio.CancelScope(shield=True): + await api.aclose() + ``` + + +The behaviors guide [Calling language models](../../behaviors/references/guides/language-models.mdx) explains some +common conversation and judge patterns. + +### Connect them in a workflow + +The workflow function defines the overall structure of your experiment. Instead of running steps +directly, it schedules them using `run_step`, and defines both the inputs that each step should +receive and also the dimensions that the steps should be mapped over. + +Our joke evaluation can be defined using the following workflow: + +```python +@workflow(name="joke_eval.evaluate") +async def evaluate( context: WorkflowContext, - prompts: Annotated[ArrayHandle[str], "[prompt: str]"], -) -> Annotated[ArrayHandle[float], "[]"]: - """Measure the length of each prompt, then average the lengths.""" - counts = context.run_step("count_words", count_words, {"prompt": prompts}, map_over=["prompt"]) - return context.run_step("mean_length", mean_length, {"counts": counts}) + # The inputs the launcher supplies, with their dimensions: a configuration per model, the + # topics, and one judge configuration with no dimensions. + models: Annotated[ArrayHandle[BoundID[GenerationConfig]], "[model: str]"], + topics: Annotated[ArrayHandle[str], "[topic: str]"], + judge: ScalarHandle[BoundID[JudgeConfig]], + samples: int, +) -> JokeReport: + """Have each model write jokes about each topic, grade them, and summarize each model.""" + # One call per model and topic, repeated `samples` times along a new `sample` dimension. + # The result is indexed by [model, topic, sample]. + jokes = context.run_step( + "tell_jokes", + tell_joke, + {"config": models, "topic": topics}, + map_over=["model", "topic"], + replicas={"sample": range(samples)}, + ) + # One call per joke. The judge goes to every call, and each call gets its joke's topic. + grades = context.run_step( + "grade_jokes", + grade_joke, + {"judge": judge, "topic": topics, "rollout": jokes}, + map_over=["model", "topic", "sample"], + ) + # One call per model, which receives that model's grades over [topic, sample]. + summary = context.run_step("summarize", summarize, {"grades": grades}, map_over=["model"]) + # Wait for the results (`observe`), store each array, and return a report that references + # them. + return JokeReport( + jokes=await context.store(await jokes.observe()), + grades=await context.store(await grades.observe()), + summary=await context.store(await summary.observe()), + ) ``` -In this code: +* `tell_jokes` is mapped over `model` and `topic`, and `replicas` adds a `sample` dimension, so + its result is an array of rollouts indexed by `[model, topic, sample]`. +* `grade_jokes` is mapped over all three. The judge, which has no dimensions, goes to every call, + and each call gets the topic matching its joke. +* `summarize` is mapped over `model` only, so each call receives one model's grades over the + remaining dimensions, `[topic, sample]`. + +[Arrays and parallel computations](concepts/arrays-and-parallel-computations.mdx) explains +mapping in depth, and [Workflows and steps](concepts/workflows-and-steps.mdx) explains how +workflows run. + +### Put it all together into an experiment module + +For this experiment, we can put the entity types, steps, and workflow function into a single +experiment module. Here's what it looks like for the joke evaluation: + + + ```python src/joke_eval/experiment.py + """A joke evaluation: models write jokes about topics, and a judge model grades each joke. + + This module defines the experiment's entity types, its steps, and the workflow that connects them. + It is listed in `[tool.fxtr] modules` in pyproject.toml, so fxtr imports it to find the steps and + the workflow when it resumes a job or shows one in the viewer. + """ + + import re + from dataclasses import dataclass + from typing import Annotated, Literal, TypedDict + + import anyio + + from behaviors.implementations.context_policies import ( + ScriptedUserSimulator, + context_policy_from_user_simulator, + ) + from behaviors.implementations.models import ( + DEFAULT_FAILURE_CATEGORIES_TO_RETRY, + chat_session_model_from_api, + generate_with_retries, + ) + from behaviors.implementations.models.anthropic import AnthropicMessagesAPI + from behaviors.implementations.models.openai_responses import OpenAIResponsesAPI + from behaviors.rollouts.chat import run_chat_rollout + from behaviors.types import ( + AssistantMessage, + ChatMessage, + ChatRollout, + ContentText, + ModelCallFailure, + ModelRequest, + RetryConfig, + RolloutCompleted, + SamplingParams, + UserMessage, + ) + from fxtr.core.entities import BoundID + from fxtr.entity_defns.array import Array + from fxtr.entity_defns.dataclass_entity import DataclassEntity + from fxtr.experiment.handles import ArrayHandle, ScalarHandle + from fxtr.experiment.steps import StepContext, step + from fxtr.experiment.workflows import WorkflowContext, workflow + + Provider = Literal["anthropic", "openai"] + + + def make_api(provider: Provider, model_name: str) -> AnthropicMessagesAPI | OpenAIResponsesAPI: + """A client for the provider's API. Both steps build their clients here, so a test can + replace this one function with a fake model.""" + if provider == "anthropic": + return AnthropicMessagesAPI(model_name) + return OpenAIResponsesAPI(model_name) + + + def answer_text(messages: tuple[ChatMessage, ...]) -> str: + """The assistant's text, without reasoning or tool-use blocks.""" + return "".join( + block.text + for message in messages + if isinstance(message, AssistantMessage) + for block in message.content + if isinstance(block, ContentText) + ) + + + # --- Entities ------------------------------------------------------------------------------------ + # + # An entity is a record that fxtr stores in its database and can show in the viewer. A frozen + # dataclass becomes one by subclassing DataclassEntity, with a type ID (`fxtr_type`) that names it + # uniquely. A `BoundID[T]` field is a reference to another stored entity of type T. + + + @dataclass(frozen=True) + class GenerationConfig(DataclassEntity, fxtr_type="local.joke_eval.GenerationConfig.v1"): + """A model asked for jokes, and the prompt it is asked with.""" + + provider: Provider + model_name: str + sampling: SamplingParams + prompt_template: str # formatted with {topic} + + + @dataclass(frozen=True) + class JudgeConfig(DataclassEntity, fxtr_type="local.joke_eval.JudgeConfig.v1"): + """A judge model and the rubric it grades jokes with.""" + + provider: Provider + model_name: str + sampling: SamplingParams + rubric: str # formatted with {topic} and {joke} -* `count_words` takes one prompt and returns one number. `mean_length` takes a whole array of - counts, indexed by the `prompt` dimension, and returns one number. -* `run_step` schedules a step and returns a *handle* to its future result without waiting for - it. Passing `counts` to the second step makes it depend on the first. -* `map_over=["prompt"]` calls `count_words` once per prompt and stacks the results into an array - with the same `prompt` dimension. The second `run_step` has no `map_over`, so `mean_length` - runs once on the whole array. -* A plain `str` or `int` annotation means a single value. `Annotated[..., "[prompt: str]"]` also - names the array's dimensions, which a Python type can't express. -* Each docstring appears on its step's or workflow's card in the viewer. -Then register the module in `pyproject.toml`, so that fxtr can find the experiment by name when -it resumes a job or shows it in the viewer: + @dataclass(frozen=True) + class JokeGrade(DataclassEntity, fxtr_type="local.joke_eval.JokeGrade.v1"): + """The judge's grade of one joke.""" + + topic: str + joke: str + score: int | None # 1 to 10; None when there was no joke or the reply had no valid score + rationale: str + judge_reply: str + + + class ModelSummary(TypedDict): + """How one model's jokes were graded. A TypedDict isn't an entity: it's stored inline, as a + value in an array.""" + + jokes: int + graded: int + mean_score: float | None + + + @dataclass(frozen=True) + class JokeReport(DataclassEntity, fxtr_type="local.joke_eval.JokeReport.v1"): + """Everything the evaluation produced. Each field references a stored array; the comments + give each array's dimensions.""" + + jokes: BoundID[Array[BoundID[ChatRollout]]] # [model, topic, sample] + grades: BoundID[Array[BoundID[JokeGrade]]] # [model, topic, sample] + summary: BoundID[Array[ModelSummary]] # [model] + + + # --- Steps --------------------------------------------------------------------------------------- + # + # A step is an async function that does one unit of work. `@step` registers it under a name. fxtr + # caches each call's result, and reuses it when the same step is called with the same inputs. + # An argument annotated with an entity type, such as `config: GenerationConfig`, receives the + # loaded entity; the `context` gives access to fxtr, such as storing and loading entities. + + + @step(name="joke_eval.tell_joke") + async def tell_joke(context: StepContext, config: GenerationConfig, topic: str) -> ChatRollout: + """Ask a model for a joke about the topic.""" + prompt = config.prompt_template.format(topic=topic) + api = make_api(config.provider, config.model_name) + try: + # The model to talk to, which retries rate limits and server errors. + model = chat_session_model_from_api(api, config.sampling, retry=RetryConfig()) + # The user's side of the conversation: one scripted message. + user = ScriptedUserSimulator(messages=(UserMessage(content=(ContentText(text=prompt),)),)) + policy = context_policy_from_user_simulator(user, initialization=None) + # Run the conversation. One more turn than there are scripted messages, so the rollout + # can end as completed. The returned ChatRollout is an entity, so fxtr stores it. + return await run_chat_rollout(model, policy, max_turns=2) + finally: + with anyio.CancelScope(shield=True): + await api.aclose() + + + @step(name="joke_eval.grade_joke") + async def grade_joke( + context: StepContext, judge: JudgeConfig, topic: str, rollout: ChatRollout + ) -> JokeGrade: + """Grade a joke from 1 to 10.""" + joke = "" + if isinstance(rollout.status, RolloutCompleted) and rollout.model_turns: + joke = answer_text(rollout.model_turns[-1].messages).strip() + if not joke: + # Return, rather than raise, for outcomes that a retry would reproduce: the result is + # cached as a grade with no score. + return JokeGrade( + topic=topic, joke="", score=None, rationale="", + judge_reply=f"not graded: no joke ({rollout.status.kind})", + ) + + # A judge needs no conversation, so it makes a single request. + request = ModelRequest( + context=(UserMessage(content=(ContentText(text=judge.rubric.format(topic=topic, joke=joke)),)),), + tools=(), + tool_choice=None, + parallel_tool_calls=None, + sampling=judge.sampling, + ) + api = make_api(judge.provider, judge.model_name) + try: + response = await generate_with_retries( + api, + request, + retry=RetryConfig(), + failure_categories_to_retry=DEFAULT_FAILURE_CATEGORIES_TO_RETRY, + ) + finally: + with anyio.CancelScope(shield=True): + await api.aclose() + if isinstance(response, ModelCallFailure): + # Raise rather than return: a returned failure would be cached as the result, while a + # raised one makes the job stop, and resuming it calls the judge again. + raise RuntimeError(f"judge call failed ({response.category}): {response.description}") + + # Read the score and rationale from the reply's XML tags. + reply = answer_text(response.output) + score_match = re.search(r"\s*(\d+)\s*", reply) + score = int(score_match.group(1)) if score_match else None + rationale_match = re.search(r"(.*?)", reply, re.DOTALL) + # Log lines show on the step's card in the viewer. + context.log(f"{score}: {joke}") + return JokeGrade( + topic=topic, + joke=joke, + score=score if score is not None and 1 <= score <= 10 else None, + rationale=rationale_match.group(1).strip() if rationale_match else "", + judge_reply=reply, + ) + + + @step(name="joke_eval.summarize") + async def summarize( + context: StepContext, + # An array, and the dimensions each call receives: all of one model's grades. Its values are + # references to the grades, which the step loads itself. + grades: Annotated[Array[BoundID[JokeGrade]], "[topic: str, sample: int]"], + ) -> ModelSummary: + """Average one model's scores.""" + scores = [] + for ref in grades.values(): + grade = await context.load(ref) + if grade.score is not None: + scores.append(grade.score) + return ModelSummary( + jokes=len(grades), + graded=len(scores), + mean_score=sum(scores) / len(scores) if scores else None, + ) + + + # --- Workflow ------------------------------------------------------------------------------------ + # + # The workflow connects the steps. It doesn't call them directly: `context.run_step` schedules a + # step and returns a handle to its future results, which can be passed on to later steps. fxtr + # runs each step's calls in parallel, as soon as their inputs are ready. + + + @workflow(name="joke_eval.evaluate") + async def evaluate( + context: WorkflowContext, + # The inputs the launcher supplies, with their dimensions: a configuration per model, the + # topics, and one judge configuration with no dimensions. + models: Annotated[ArrayHandle[BoundID[GenerationConfig]], "[model: str]"], + topics: Annotated[ArrayHandle[str], "[topic: str]"], + judge: ScalarHandle[BoundID[JudgeConfig]], + samples: int, + ) -> JokeReport: + """Have each model write jokes about each topic, grade them, and summarize each model.""" + # One call per model and topic, repeated `samples` times along a new `sample` dimension. + # The result is indexed by [model, topic, sample]. + jokes = context.run_step( + "tell_jokes", + tell_joke, + {"config": models, "topic": topics}, + map_over=["model", "topic"], + replicas={"sample": range(samples)}, + ) + # One call per joke. The judge goes to every call, and each call gets its joke's topic. + grades = context.run_step( + "grade_jokes", + grade_joke, + {"judge": judge, "topic": topics, "rollout": jokes}, + map_over=["model", "topic", "sample"], + ) + # One call per model, which receives that model's grades over [topic, sample]. + summary = context.run_step("summarize", summarize, {"grades": grades}, map_over=["model"]) + # Wait for the results (`observe`), store each array, and return a report that references + # them. + return JokeReport( + jokes=await context.store(await jokes.observe()), + grades=await context.store(await grades.observe()), + summary=await context.store(await summary.observe()), + ) + ``` + + +Register your module in `pyproject.toml`, so that fxtr can find its steps and workflow when it +resumes a job or shows one in the viewer: ```toml pyproject.toml [tool.fxtr] -modules = ["my_project.word_count"] +modules = ["joke_eval.experiment"] ``` -## Launch it +### Write the launcher + +The launcher is a script which is responsible for configuring and launching the experiment. +This is usually a good place to specify global configuration arguments or inputs that you want to +run your experiment over. -Create a launcher at the project root. It builds the input dataset and runs the workflow as a -job: +The simplest possible launcher is just a `main` script which connects to fxtr, stores the +configurations, builds the workflow's inputs, and runs it. This one asks one model for one joke +about one topic: -```python launch_word_count.py +```python launch.py import anyio +from behaviors.types import SamplingParams +from fxtr.core.entities import BoundID from fxtr.entity_defns.array import Array from fxtr.project.running import open_local_client, run_job -from my_project.word_count import experiment +from joke_eval.experiment import GenerationConfig, JudgeConfig, evaluate async def main() -> None: - prompts = Array.from_items( - [ - (("greeting",), "Hello there"), - (("question",), "Why is the sky blue?"), - (("request",), "Please summarize this article in three sentences."), - ], - ("[prompt: str]", str), - ) async with open_local_client(__file__) as client: - result = await run_job(client, experiment, {"prompts": prompts}, root="word-count-v1") - print(f"mean length: {result.item():.2f} words") + # Store the configurations, getting back references to them. + haiku = await client.store(GenerationConfig( + provider="anthropic", + model_name="claude-haiku-4-5-20251001", + sampling=SamplingParams(max_tokens=1000), + prompt_template='Write one original joke about "{topic}". Reply with the joke only.', + )) + judge = await client.store(JudgeConfig( + provider="anthropic", + model_name="claude-opus-5-5", + sampling=SamplingParams(max_tokens=1000), + rubric=( + 'Rate this joke about "{topic}" from 1 to 10.\n\n{joke}\n\n' + "Explain your rating, then give it as ...N." + ), + )) + # Build the workflow's inputs, as arrays with named dimensions, and run it as a job. + inputs = { + "models": Array.from_items([(("haiku-4.5",), haiku)], ("[model: str]", BoundID[GenerationConfig])), + "topics": Array.from_items([(("coffee",), "coffee")], ("[topic: str]", str)), + "judge": Array.scalar(judge, BoundID[JudgeConfig]), + "samples": Array.scalar(1, int), + } + await run_job(client, evaluate, inputs, root="jokes-minimal") if __name__ == "__main__": anyio.run(main) ``` -`Array.from_items` takes `(key, value)` pairs and a type: here, one `prompt` dimension with string -keys, holding string values. `root` names the job's root workflow, and, as you'll see below, -decides which earlier results the job can reuse. +However, you might want to expand this to include helper functions and command line flags for +configuring the shape of your experiment. For instance, here's a more fully-featured launcher +for the joke evaluation: + + + ```python launch.py + """Launch the joke evaluation: `uv run python launch.py pilot` or `uv run python launch.py full`.""" + + import sys + from typing import Any + + import anyio + + from behaviors.types import SamplingParams + from fxtr.core.entities import BoundID, FxtrEntityStorage + from fxtr.entity_defns.array import Array + from fxtr.project.client import JobNotFinishedError, ProjectClient + from fxtr.project.running import open_local_client, run_job + from joke_eval.experiment import GenerationConfig, JokeReport, JudgeConfig, Provider, evaluate + + # The name of this run. Every cached result's address starts with it, so launching again under the + # same root reuses earlier results, and a new root starts fresh. + ROOT = "jokes-v1" + + # How many topics (from the start of TOPICS) and samples per model and topic each size uses. + SIZES = {"pilot": (3, 1), "full": (10, 3)} + + # Each model's short name (its key in the `model` dimension), provider, and model ID. + MODELS: dict[str, tuple[Provider, str]] = { + "haiku-4.5": ("anthropic", "claude-haiku-4-5-20251001"), + "sonnet-5.5": ("anthropic", "claude-sonnet-5-5"), + } + JUDGE: tuple[Provider, str] = ("anthropic", "claude-opus-5-5") + + TOPICS = [ + "coffee", "dentist", "printer", "penguin", "Monday", + "airport", "houseplant", "spreadsheet", "cat", "elevator", + ] + + JOKE_PROMPT = ( + 'Write one original joke about "{topic}". Reply with the joke only, ' + "with no preamble or explanation." + ) + + RUBRIC = """\ + You are judging a joke about "{topic}". Rate how funny it is, from 1 (not funny at all) to 10 \ + (a joke a professional comedian would be proud of). A typical competent joke is around 5. \ + Score a well-known joke, or a stock format with the topic swapped in, low. + + + {joke} + + + First explain your rating in two or three sentences, then give the score, in this form: + ... + N""" + + + async def build_inputs(client: ProjectClient, size: str) -> dict[str, Array[Any]]: + """The workflow's inputs, as arrays keyed by the names of its arguments.""" + topic_count, samples = SIZES[size] + + # Store a configuration entity per model. `client.store` returns a reference (a BoundID) to + # the stored entity, which is what the workflow takes. + models = [] + for name, (provider, model_name) in MODELS.items(): + config = GenerationConfig( + provider=provider, + model_name=model_name, + sampling=SamplingParams(max_tokens=1000), + prompt_template=JOKE_PROMPT, + ) + models.append(((name,), await client.store(config))) + provider, model_name = JUDGE + judge = JudgeConfig( + provider=provider, + model_name=model_name, + sampling=SamplingParams(max_tokens=1000), + rubric=RUBRIC, + ) + + # `Array.from_items` builds an array from (key, value) pairs, given its dimensions and value + # type; `Array.scalar` builds one with no dimensions. + return { + "models": Array.from_items(models, ("[model: str]", BoundID[GenerationConfig])), + "topics": Array.from_items( + [((topic,), topic) for topic in TOPICS[:topic_count]], ("[topic: str]", str) + ), + "judge": Array.scalar(await client.store(judge), BoundID[JudgeConfig]), + "samples": Array.scalar(samples, int), + } + + + async def print_summary(storage: FxtrEntityStorage, result: Array[Any]) -> None: + """Print each model's mean score from the report the workflow returned.""" + report = await storage.load(result.item(), as_type=JokeReport) + summary = await storage.load(report.summary) + for (model,), row in summary.items(): + mean = "-" if row["mean_score"] is None else f"{row['mean_score']:.2f}" + print(f"{model}: mean score {mean} ({row['graded']} of {row['jokes']} jokes graded)") + + + async def main(size: str) -> None: + # Connect to the project's database. `max_concurrency` caps how many steps run at once. + async with open_local_client(__file__, max_concurrency=8) as client: + inputs = await build_inputs(client, size) + # Submit the workflow as a job, show it in the viewer, wait for it to finish, and then + # print the summary. + await run_job(client, evaluate, inputs, root=ROOT, on_result=print_summary) + + + if __name__ == "__main__": + if len(sys.argv) != 2 or sys.argv[1] not in SIZES: + sys.exit(f"usage: python launch.py {{{'|'.join(SIZES)}}}") + try: + anyio.run(main, sys.argv[1]) + except JobNotFinishedError: + sys.exit(1) # the job's status and how to resume it are already printed + ``` + + +### Show conversations in the viewer -Commit, then launch, with `uv run fxtr view` running in another terminal. A launch records the -commit it runs, so it refuses uncommitted changes: +The viewer can show your conversations using the renderers from behaviors. Add +`fxtr-view-behaviors` to the dependencies in `views/package.json`: + +```json views/package.json +"fxtr-view-behaviors": "link:./vendor/fxtr/packages/fxtr-view-behaviors" +``` + +Register its renderers in `views/src/bundle.ts`: + +```ts views/src/bundle.ts +import type { ViewBundle } from "fxtr-view"; +import { behaviorsRenderers } from "fxtr-view-behaviors"; + +const bundle: ViewBundle = { + renderers: [...behaviorsRenderers], +}; + +export default bundle; +``` + +Then install it, rebuild, and commit everything: ```bash -git add . && git commit -m "Add the word count experiment" -uv run python launch_word_count.py +(cd views && pnpm install) +uv run fxtr views build +git add . && git commit -m "First experiment" +``` + + + Before you spend money on model calls, you can ask a coding agent to check the experiment: + have it run the pilot on a scratch database schema, with the function that builds model + clients (`make_api` in the example) replaced by a fake model. This catches mistakes in the steps, + the workflow, and the launcher. + + +## 5. Run the pilot + +In a second terminal, start the viewer from the project, and leave it running: + +```bash +uv run fxtr view +``` + +Next you can launch the evaluation: + +```bash +uv run python launch.py pilot ``` ```text -job 3eb75a59-911e-44af-a148-411ac5dcd8d1: running -watch it at http://127.0.0.1:8000/#job=3eb75a59-911e-44af-a148-411ac5dcd8d1 -job 3eb75a59-911e-44af-a148-411ac5dcd8d1: succeeded -... -mean length: 4.67 words +job 06bb0ae8-f73c-4995-acf3-2ab0b299b259: running +watch it at http://127.0.0.1:8000/#job=06bb0ae8-f73c-4995-acf3-2ab0b299b259 +sent it to the viewer's open tab +job 06bb0ae8-f73c-4995-acf3-2ab0b299b259: succeeded + result: bafyreicl5xf4h3itncd34nnehfpoae6hnqpcumrty7o4ffjpirgidfaray +haiku-4.5: mean score 2.67 (3 of 3 jokes graded) +sonnet-5.5: mean score 3.67 (3 of 3 jokes graded) ``` -The launch opens the job in a new tab of the viewer: the `prompts` input feeding three -`count_words` calls, one per prompt, feeding `mean_length`. +The launcher prints the job's ID and a link, and the viewer switches to the job. It shows the +workflow with a card for each step, which fills in as the step's calls finish. + +Try clicking through the results: + +* Open the card of the step that generates conversations (`tell_jokes`) to see its result array. + Each value links to a conversation. +* Open a measurement from the judge step (`grade_jokes`) to read the judge's rationale. +* Open the result of the summary step (`summarize`) to see the result per condition. + +You can use the viewer to read a few conversations and their measurements, and figure out if there is anything that you want to change in the experiment before you scale it up. + +## 6. Change it and run it again -## Rerun it +Whenever you launch the experiment again, fxtr reuses every cached result whose step gets the same inputs, so only +new work costs model calls. The two most common changes to a first experiment are scaling it up +and adding a condition. -Launch it again, unchanged. The job succeeds straight away, because every step finds its result -in the cache. You can see the entries with `fxtr cache list`: +### Scale up to the full run + +Once the pilot looks right, you can launch the full run. For the joke evaluation, that's all 10 topics with 3 +samples each: + +```bash +uv run python launch.py full +``` + +It reuses all of the pilot's results and computes only the new ones: in the example, the other 54 +jokes and 54 grades. Then it stops: ```text -completed word-count-v1/count_words[prompt:"greeting"] (word_count.count) -completed word-count-v1/count_words[prompt:"question"] (word_count.count) -completed word-count-v1/count_words[prompt:"request"] (word_count.count) -completed word-count-v1/mean_length (word_count.mean) +job 4c713924-2fc5-4d9c-a55d-0ae1bdca398f: stopped + reason: cache conflicts at 2 addresses, such as 'jokes-v1/summarize[model:"haiku-4.5"]': different requests own them; clear them to run these requests there + cache conflicts at 2 addresses: + jokes-v1/summarize[model:*] 2 of 2 entries joke_eval.summarize + to list them: fxtr cache list --conflicts-in 4c713924-2fc5-4d9c-a55d-0ae1bdca398f + to clear them: fxtr cache clear --conflicts-in 4c713924-2fc5-4d9c-a55d-0ae1bdca398f, then fxtr resume 4c713924-2fc5-4d9c-a55d-0ae1bdca398f + to rerun overwriting: fxtr resume 4c713924-2fc5-4d9c-a55d-0ae1bdca398f --overwrite-cache-conflicts ``` -Each entry is at a **cache address** built from the job's `root`, the step's name in the -workflow, and, for a mapped step, the key it was called with. A step reuses a cached result when -its address holds a result computed from the same step and the same inputs. +A step that reduces over a dimension that grew, here `summarize`, now gets more inputs than in +the pilot, at the same cache address (`jokes-v1/summarize[model:"haiku-4.5"]`). fxtr won't +silently replace a cached result with one computed from different inputs, so it stops and reports +a **cache conflict**. You don't need the pilot's summaries any more, so you can resume the job and +overwrite them, using the job ID it printed: + +```bash +uv run fxtr resume --overwrite-cache-conflicts +``` -## Change an input +This reruns only the summaries. Resuming doesn't run the launcher, so launch once more to +print the summary. Every step is now cached, so it finishes at once, with no model calls: -Now edit the `question` prompt, commit, and launch again under the same `root`. The job stops: +```bash +uv run python launch.py full +``` ```text -job 3fa3df92-2edb-4088-9e1f-a99d28b7ec77: stopped - reason: cache conflict at 'word-count-v1/count_words[prompt:"question"]': a different request owns the address; clear it to run this request there - cache conflicts at 1 address: - word-count-v1/count_words[prompt:"question"] 1 address word_count.count - to list them: fxtr cache list --conflicts-in 3fa3df92-2edb-4088-9e1f-a99d28b7ec77 - to clear them: fxtr cache clear --conflicts-in 3fa3df92-2edb-4088-9e1f-a99d28b7ec77, then fxtr resume 3fa3df92-2edb-4088-9e1f-a99d28b7ec77 - to rerun overwriting: fxtr resume 3fa3df92-2edb-4088-9e1f-a99d28b7ec77 --overwrite-cache-conflicts -Traceback (most recent call last): - ... -fxtr.project.client.JobNotFinishedError: Job 3fa3df92-2edb-4088-9e1f-a99d28b7ec77 is stopped -``` - -The address `word-count-v1/count_words[prompt:"question"]` already holds a result computed from -the old prompt. fxtr won't reuse that result for the new prompt, and doesn't replace a completed -result unless you ask, so this is a **cache conflict**: the step doesn't run, and the job stops -once nothing else can. The unchanged prompts still reuse their results. `run_job` raises -`JobNotFinishedError` for any job that doesn't succeed, so your launcher can tell. - -The simplest way forward is to clear exactly the conflicting entries and resume the job: +haiku-4.5: mean score 2.37 (30 of 30 jokes graded) +sonnet-5.5: mean score 3.57 (30 of 30 jokes graded) +``` + +### Add a condition + +Next, you can add a condition as a new key of the dimension you compare. In the example, add Opus 5.5 to +`MODELS` in `launch.py`: + +```python +MODELS: dict[str, tuple[Provider, str]] = { + "haiku-4.5": ("anthropic", "claude-haiku-4-5-20251001"), + "sonnet-5.5": ("anthropic", "claude-sonnet-5-5"), + "opus-5.5": ("anthropic", "claude-opus-5-5"), +} +``` + +Commit and launch the full run again. fxtr computes the new key's conversations, measurements, +and summary at new addresses, and reuses everything else. This time nothing conflicts, because no +step reduces over `model`: ```bash -uv run fxtr cache clear --conflicts-in 3fa3df92-2edb-4088-9e1f-a99d28b7ec77 -uv run fxtr resume 3fa3df92-2edb-4088-9e1f-a99d28b7ec77 +git commit -am "Add Opus 5.5" +uv run python launch.py full ``` -`count_words` runs again for the new prompt. If its new count changes the mean, the resume stops -again, this time at `mean_length`, whose input only changed once `count_words` had rerun. Clear -and resume once more and the job succeeds. Clearing affects later requests only: jobs that -already finished keep the results they used. +```text +haiku-4.5: mean score 2.37 (30 of 30 jokes graded) +opus-5.5: mean score 4.63 (30 of 30 jokes graded) +sonnet-5.5: mean score 3.57 (30 of 30 jokes graded) +``` -When you are changing data on purpose and don't need to look at each conflict first, -`uv run fxtr resume JOB --overwrite-cache-conflicts` does the same in one go, rerunning every -conflicting step and replacing its old result, downstream steps included. A third option is to -launch under a new `root`, which leaves every earlier result intact. -[Resolving a job's conflicts](concepts/managing-the-step-cache.mdx#resolving-a-jobs-conflicts) -compares the three. +### What a change reuses - - Changing a step's **code** does not invalidate its cached results: the cache matches a step by - its name and inputs, not its implementation. After changing what a step computes, launch under - a new `root` or clear its entries. See - [Handling changes to step logic](concepts/managing-the-step-cache.mdx#handling-changes-to-step-logic). - +| Change | What happens on the next launch | +| ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Add a condition, such as a model | Computes only the new condition's results. A step that reduces over that dimension, if any, conflicts. | +| Add items or samples | Computes only the new conversations and measurements. A step that reduces over them, like `summarize`, conflicts. | +| Edit a prompt, the rubric, or a sampling setting | The steps that get it have new inputs at the same addresses, and conflict. To keep the old results for comparison, add the change as a new condition, such as a second model entry with another prompt, or launch under a new `ROOT`. | +| Add a step at the end of the workflow | Reuses every earlier step, and runs only the new one. | +| Change a step's code | Reuses the old results: the cache only sees a step's inputs, not its code. Clear that step's results to recompute them. | -## Working with a coding agent +[Managing the step cache](concepts/managing-the-step-cache.mdx) explains how each of these +works, and the ways to resolve a conflict. -fxtr has two skills for coding agents, `fxtr` and `behaviors`, which send an agent to the pages -of this documentation a task needs: writing steps and workflows, launching jobs, caching, calling -models with behaviors, and building viewer renderers. So you can ask an agent for an experiment in -plain language and review the result in the viewer. The skills carry a copy of the pages they use, -so an agent reads them without going online. +### Build a results view -Install the fxtr plugin in Claude Code: +The viewer's default views show every array and entity, but you can customize the view to show a more informative view of the data. With the fxtr skills +[installed](installation.mdx#install-the-agent-skills), try asking your coding agent for one, +describing what you want to see. For the example: ```text -/plugin marketplace add TransluceAI/claude-code-plugins -/plugin install fxtr@transluce-plugins +Use the /fxtr skill to add renderers for this project's entities and step panels, and a +results overview for joke_eval.evaluate that compares the models' mean scores and lets me browse +the best and worst jokes for each model. ``` -The plugin has a version of its own, and Claude Code offers each new one as an update; it isn't -tied to the fxtr your project uses. To see which documentation a copy of the skills carries, read -the `provenance.json` beside each skill's `SKILL.md`: it names the fxtr and behaviors versions, -the commit of the fxtr repository the copy was made from (and whether it had changes not yet -committed), and a hash of the skill's files. The pages on this site are always the newest. +[Visualization and custom renderers](concepts/visualization-and-custom-renderers.mdx) explains +how views work. ## Next steps - - - The rules workflows and steps follow, and why they are separate. +You've built, run, and changed your first experiment. From here, you could: + +* Refine the rubric, or add a second judge to see whether the judges agree (in the example, + whether Opus favors its own jokes). +* Study a follow-up question about something you noticed in the conversations. +* Start an experiment on another question, by hand or with the `first-experiment` skill. + +To learn more about the ideas in this guide: + + + + How projects, steps, workflows, jobs, and arrays fit together. + + + + Dimensions, mapping steps over arrays, and replicas. - - Sweep over models, prompts, and samples, and reduce over any dimension. + + What a launch reuses, when it stops on a conflict, and how to resolve one. - Use behaviors to hold conversations and judge outputs in your steps. + Conversations, judges, and summaries in fxtr steps. diff --git a/plugins/fxtr/skills/fxtr/references/guides/designing-views.mdx b/plugins/fxtr/skills/fxtr/references/guides/designing-views.mdx index 01b764f..b9ac98d 100644 --- a/plugins/fxtr/skills/fxtr/references/guides/designing-views.mdx +++ b/plugins/fxtr/skills/fxtr/references/guides/designing-views.mdx @@ -37,6 +37,10 @@ The viewer draws the frame. Leave out what it already gives: (`Transcript`); an invocation view starts with `InvocationHead`. * The top right corner of an entity's panel holds the viewer's view switcher and info button, faint until hovered. Keep controls out of that corner. +* A step's pane already shows each of its input and output arrays as a table you can page + through and slice, with links to each entity. Don't redraw those arrays as a table. A step's + view earns its place by adding what that table can't: a chart, a summary across invocations, + or one invocation's content laid out to read. ## The look @@ -304,18 +308,16 @@ export function ReportPanel({ id, storage }: EntityPanelProps) { ## Check it -Save, let the running `uv run fxtr view` rebuild the bundle (the open viewer updates in place; see -[Building and reloading](../reference/renderers.mdx#building-and-reloading)), and look at the view: - -* in both themes (the theme toggle at the foot of the viewer's left rail); -* as the results overview at full width, in a pane dragged narrow, and in a card's preview (close - every pane to see the previews); -* with the keyboard: Tab reaches every mark and control, Enter presses, the tooltip follows focus; -* pressing a mark opens its entity in a pane, and the mark then wears the ink ring. - -And search the package for what the viewer never uses; each of these should find nothing: +Save, and let the running `uv run fxtr view` rebuild the bundle (the open viewer updates in place), +or build it with `uv run fxtr views build`, which prints why a build failed +([Building and reloading](../reference/renderers.mdx#building-and-reloading)). Then search the +package for what the viewer never uses; each of these should find nothing: ```bash grep -rnE '#[0-9a-fA-F]{3,8}\b|rgba?\(|hsla?\(' views/src grep -rnE 'font-family: *[^v ]|font-size: *[0-9]|color-scheme|!important|' views/src ``` + +The rest is a look at the view in the viewer. The usual things to catch are a theme it reads badly +in (the toggle is at the foot of the viewer's left rail) and a width it breaks at: drag a pane +narrow, or close every pane to see the card previews. diff --git a/plugins/fxtr/skills/fxtr/references/guides/running-and-viewing-jobs.mdx b/plugins/fxtr/skills/fxtr/references/guides/running-and-viewing-jobs.mdx index 17ea76e..90cd865 100644 --- a/plugins/fxtr/skills/fxtr/references/guides/running-and-viewing-jobs.mdx +++ b/plugins/fxtr/skills/fxtr/references/guides/running-and-viewing-jobs.mdx @@ -24,7 +24,7 @@ its jobs as they run and after they finish. results, set `schema` in `fxtr.local.toml` to another name while experimenting (or run `fxtr init` again with the same `--database-url`, another `--schema`, and `--force`), and set it back for real runs. -* Start the viewer and leave it running, so each launch opens its job in it (see +* Start the viewer and leave it running, so each launch shows its job in it (see [Watch jobs in the viewer](#watch-jobs-in-the-viewer)). ## Launch from Python @@ -74,10 +74,12 @@ if __name__ == "__main__": anyio.run(main) ``` -* `run_job` opens the job in a new tab of the project's running viewer (`uv run fxtr view`) and - prints its link before waiting, then the job's status and result; with no viewer running, it - prints how to start one. `open_browser=False`, or `FXTR_NO_BROWSER=1` in the environment, only - prints the link. It raises `JobNotFinishedError` if the job doesn't succeed. +* `run_job` shows the job in the project's running viewer (`uv run fxtr view`) and prints its + link before waiting, then the job's status and result; with no viewer running, it prints how to + start one. The viewer's open tabs switch to the job, or a new tab opens when none is open (see + [Watch jobs in the viewer](#watch-jobs-in-the-viewer)). `open_browser=False`, or + `FXTR_NO_BROWSER=1` in the environment, only prints the link. It raises `JobNotFinishedError` + if the job doesn't succeed. * `root` is the root workflow's pathname, and so the start of every cache address in the job. See [Managing the step cache](../concepts/managing-the-step-cache.mdx). * Use `handle = await client.submit(...)` when you need the job's ID or want to control the @@ -94,17 +96,22 @@ uv run fxtr view ``` It selects a local port and opens the viewer in your browser; running it again while the -project's viewer runs opens that one. The [CLI reference](../reference/cli.mdx#viewer) has its -options. Each job you launch opens in a new tab of it, at the link the launcher prints, such as -`http://127.0.0.1:8000/#job=<job id>`, and you can pick any of the project's jobs from the menu -above the minimap. A job that's still running updates live. +project's viewer runs opens another tab of that one. The [CLI reference](../reference/cli.mdx#viewer) +has its options. Each job you launch shows up in the viewer's open tabs, at the link the launcher +prints, such as `http://127.0.0.1:8000/#job=<job id>`; with no tab open, the launch opens one. A +tab switches to each job you launch, and a note in its corner says why, with a button to go back. +Once you pick a job in a tab yourself, from the menu above the minimap or by changing the link's +`#job=`, the tab stays on it, and the note offers the new job instead. A job that's still running +updates live. For one job, the viewer shows: * **The tree of workflow invocations.** The root workflow at the top, with each child workflow one level down. A mapped child appears once per key. * **A card for every step and child workflow** a workflow scheduled, showing its docstring, how it - was mapped, its progress, and its log lines. A card links to the function's code in your editor. + was mapped, its progress, and its log lines. A card's code chip, which names the commit the job + ran, opens the function's file as the job ran it in a pane, whose **Open** button opens it in your editor (choose which from its + menu). * **The arrays each operation received and produced**, as tables you can page through and slice by dimension. A value that references an entity is a link. * **Entity details**, opened from any link, in a pane you can keep alongside the trace. Hovering a diff --git a/plugins/fxtr/skills/fxtr/references/index.mdx b/plugins/fxtr/skills/fxtr/references/index.mdx index 68dde61..b16cbdc 100644 --- a/plugins/fxtr/skills/fxtr/references/index.mdx +++ b/plugins/fxtr/skills/fxtr/references/index.mdx @@ -1,5 +1,5 @@ --- -title: "Introduction" +title: "What is fxtr?" description: "fxtr is an experiment orchestration system for verifiable AI-powered analyses" --- @@ -7,6 +7,22 @@ fxtr (pronounced "fixture") is a framework for running experiments and data anal In short: fxtr tries to preserve the full power of coding agents and avoid restricting your experiment design, but provide enough scaffolding so that your analyses are systematic, organized, and reproducible and do not devolve into slop. +<iframe className="w-full aspect-video rounded-xl" src="https://www.youtube.com/embed/AhgRj3EP6EE" title="Quickly making a bespoke "humor evaluation" with fxtr" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowFullScreen /> + +<CardGroup cols={3}> + <Card title="Installation" icon="download" href="installation.mdx"> + Install fxtr and set up your project and database. + </Card> + + <Card title="Your first experiment" icon="flask" href="first-experiment.mdx"> + Design and run an experiment of your own, with your coding agent as a guide. + </Card> + + <Card title="Core concepts" icon="book" href="concepts/overview.mdx"> + Projects, steps, workflows, jobs, arrays, and entities. + </Card> +</CardGroup> + ## How it Works 1. **Define your experiment as simple, readable, environment-independent code.** Each experiment or analysis is expressed as a Python package containing one or more *workflows*, which schedule expensive or nondeterministic *steps* into a graph of dependent computations. This experiment definition serves as an ideal "spec" for what you want to compute, without worrying about data storage, concurrency, or looking up previously-computed results. The dataflow and data types belong to you and are highly customizable. @@ -35,20 +51,6 @@ fxtr pairs with [**behaviors**](../../behaviors/references/index.mdx), a library provides model adapters for OpenAI and Anthropic, multi-turn conversations, retries, and storable conversation records. -fxtr has skills for coding agents, which carry this documentation, so an agent such as Claude Code -or Codex knows how to write steps and workflows, launch jobs, and build viewer renderers in your -project. See [Working with a coding agent](first-experiment.mdx#working-with-a-coding-agent). - -<CardGroup cols={3}> - <Card title="Installation" icon="download" href="installation.mdx"> - Install fxtr and set up your project and database. - </Card> - - <Card title="Your first experiment" icon="flask" href="first-experiment.mdx"> - Write, launch, and rerun a small experiment. - </Card> - - <Card title="Core concepts" icon="book" href="concepts/overview.mdx"> - Projects, steps, workflows, jobs, arrays, and entities. - </Card> -</CardGroup> +To get started, follow the [installation guide](installation.mdx) to install fxtr and set up your project and database. +The guide also covers fxtr's [agent skills](installation.mdx#install-the-agent-skills), which let +Claude Code set up projects and write and run experiments for you. diff --git a/plugins/fxtr/skills/fxtr/references/installation.mdx b/plugins/fxtr/skills/fxtr/references/installation.mdx index 6ca6dfc..3915fb4 100644 --- a/plugins/fxtr/skills/fxtr/references/installation.mdx +++ b/plugins/fxtr/skills/fxtr/references/installation.mdx @@ -1,57 +1,93 @@ --- title: "Installation" -description: "Install fxtr from PyPI and verify the setup with an example project." +description: "Install fxtr and get started with an example project" --- -fxtr is on PyPI. You don't install it on its own: each project depends on it, and `fxtr new` -writes a project that does. - -## Prerequisites - -* [uv](https://docs.astral.sh/uv/) 0.12.2 or newer, and git (or jj). -* A Postgres server. The steps below run one locally with Docker; any Postgres server works. -* For the viewer renderers a project builds: [Node](https://nodejs.org/) 22 or newer and - [pnpm](https://pnpm.io/) 10. The renderer build runs with the `node` and `npm` on your `PATH`. +<Steps> + <Step title="Install the agent skills" id="install-the-agent-skills"> + These skills teach your agent how to write experiment logic, import data, operate the runner and viewer, manage their state, and much more. Just ask for what you want! + + <Tabs> + <Tab title="Claude Code"> + From a terminal: + + ```bash + claude plugin marketplace add TransluceAI/claude-code-plugins + claude plugin install fxtr@transluce-plugins + ``` + + To update the skills later, refresh the marketplace, update the plugin, and then restart + Claude Code: + + ```bash + claude plugin marketplace update transluce-plugins + claude plugin update fxtr@transluce-plugins + ``` + </Tab> + + <Tab title="Codex"> + From a terminal: + + ```bash + codex plugin marketplace add TransluceAI/codex-plugins + codex plugin add fxtr@transluce-plugins + ``` + + To update the skills later, upgrade the marketplace. This also updates the installed plugin: + + ```bash + codex plugin marketplace upgrade transluce-plugins + ``` + </Tab> + </Tabs> + </Step> -## Verify the installation with an example + <Step title="System prerequisites" id="prerequisites"> + Before continuing, make sure you have the following tools installed on your machine: -The following steps create and run a complete greeting example. The `--example` flag includes -the experiment code and its launcher, so you can check that your installation works before -writing an experiment of your own. + | Tool | Install | + | ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | + | [uv](https://docs.astral.sh/uv/) 0.12.2 or newer | [uv installation](https://docs.astral.sh/uv/getting-started/installation/). | + | [Docker](https://www.docker.com/) | On macOS, [Docker Desktop](https://docs.docker.com/desktop/setup/install/mac-install/). On Linux, [Docker Engine](https://docs.docker.com/engine/install/) for your distribution. | + | [Node](https://nodejs.org/) 22 or newer | The [Node download page](https://nodejs.org/en/download) gives commands for each system and version manager. | + | [pnpm](https://pnpm.io/) 10 | `npm install -g pnpm@10`, once Node is installed. Other methods: [pnpm installation](https://pnpm.io/installation). | + | git | Usually preinstalled. Otherwise see [Installing Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git). jj also works. | + </Step> -Without `--example`, `fxtr new` creates an empty project with no experiment or launcher. -[Your first experiment](first-experiment.mdx) walks through that path from project creation -to writing and running your own code. - -<Steps> - <Step title="Start Postgres"> - fxtr keeps every job, result, and cache entry in Postgres. To run a local server in a - container: + <Step title="Start Postgres" id="start-postgres"> + fxtr keeps every job, result, and cache entry in Postgres. To run a local server in a container: ```bash docker run -d --name fxtr-postgres -p 55433:5432 \ -e POSTGRES_USER=fxtr -e POSTGRES_PASSWORD=fxtr -e POSTGRES_DB=fxtr postgres:16 ``` - It listens on port `55433`, with user, password, and database all named `fxtr`. + It listens on port `55433`, with user, password, and database all named `fxtr`, so its URL is + `postgresql+asyncpg://fxtr:fxtr@localhost:55433/fxtr`. </Step> +</Steps> + +## Run an example +To check your installation, you can create and run a complete greeting example: + +<Steps> <Step title="Create the example project"> ```bash - uvx fxtr new ~/code/fxtr-example --example + mkdir ~/fxtr + uvx fxtr new ~/fxtr/fxtr-example --example ``` This writes a uv project that depends on fxtr, with a complete greeting experiment in `src/fxtr_example/experiment.py`, a `launch.py` that runs it, and greeting renderers for the viewer. It also runs `git init` and prints the next steps. - Install the example's Python and renderer dependencies, then build its renderers: + Install the example's Python and renderer dependencies: ```bash - cd ~/code/fxtr-example + cd ~/fxtr/fxtr-example uv sync (cd views && pnpm install) - uv run fxtr views build ``` </Step> @@ -74,7 +110,6 @@ to writing and running your own code. ``` This serves the experiment viewer at `http://127.0.0.1:8000/` and opens it in your browser. - Leave it running: each job you launch opens in it. </Step> <Step title="Commit and launch the example"> @@ -85,8 +120,7 @@ to writing and running your own code. uv run python launch.py ``` - The included code greets three guests; you do not need to write any experiment code for - this check: + The included code greets three guests: ```text job f0d6b1ac-a781-4b93-a571-edef402ad131: running @@ -98,9 +132,8 @@ to writing and running your own code. Hello, Grace! ``` - The job opens in a new tab of the viewer, showing its graph: the guest list, the `greet` step - mapped over it, and each greeting it produced. Without a viewer running, the launch prints how - to start one instead of the link. + The viewer switches to the job, showing its graph: the guest list, the `greet` step mapped + over it, and each greeting it produced. </Step> </Steps> @@ -129,7 +162,7 @@ run `pnpm install` in `views/` and rebuild. <CardGroup cols={2}> <Card title="Your first experiment" icon="flask" href="first-experiment.mdx"> - Write an experiment of your own, step by step. + Bring a question about language models, and run an experiment that answers it. </Card> <Card title="Core concepts" icon="book" href="concepts/overview.mdx"> diff --git a/plugins/fxtr/skills/fxtr/references/reference/cli.mdx b/plugins/fxtr/skills/fxtr/references/reference/cli.mdx index 8cd69b5..7debef1 100644 --- a/plugins/fxtr/skills/fxtr/references/reference/cli.mdx +++ b/plugins/fxtr/skills/fxtr/references/reference/cli.mdx @@ -39,8 +39,8 @@ describes each command's options. `--root` sets the root pathname. `--allow-staged` includes staged changes in the pinned code, `--no-verify` skips checking the environment against `uv.lock`, and `--overwrite-cache-conflicts` reruns steps in cache conflict and replaces the old results. - When the project's viewer is running, the job opens in a new tab of it; `--no-open` only - prints the link. + When the project's viewer is running, its open tabs switch to the job, or it opens in a new + tab when none is open; `--no-open` only prints the link. </ResponseField> <ResponseField name="fxtr resume JOB"> diff --git a/plugins/fxtr/skills/fxtr/references/reference/project-configuration.mdx b/plugins/fxtr/skills/fxtr/references/reference/project-configuration.mdx index caafc5a..b660a2f 100644 --- a/plugins/fxtr/skills/fxtr/references/reference/project-configuration.mdx +++ b/plugins/fxtr/skills/fxtr/references/reference/project-configuration.mdx @@ -21,8 +21,9 @@ already depends on fxtr, `uv run fxtr new` does the same): ```text pyproject.toml the project, its dependency on fxtr, and [tool.fxtr] README.md how to run it -.gitignore fxtr.local.toml, .venv/, __pycache__/, .fxtr/ (where `fxtr view` - records its address), node_modules/, and dist/ +.gitignore fxtr.local.toml, .env, .venv/, __pycache__/, .fxtr/ (where + `fxtr view` records its address), node_modules/, and dist/ +.env API keys for model providers src/my_project/ __init__.py views/ the renderer package, with an empty bundle @@ -52,7 +53,7 @@ The file `fxtr new` writes, without its comments: name = "my-project" version = "0.1.0" description = "A fxtr project" -requires-python = ">=3.13" +requires-python = ">=3.12.4" dependencies = ["fxtr[runner,debug-web]>=0.1.0"] [tool.uv] @@ -134,3 +135,16 @@ point the project at a scratch schema while experimenting (see fxtr reads both files strictly: a setting it doesn't know, or a value of the wrong type, is an error, so a misspelled setting is never silently ignored. + +## `.env` + +`.env` holds the API keys of the model providers your experiments +call. `fxtr new` writes one with empty placeholders: + +```bash .env +ANTHROPIC_API_KEY= +OPENAI_API_KEY= +``` + +Fill in the keys you use and add any other variable your experiments read. +Keys in `.env` override variables set in the environment.