From 9d19cedb9c4e7e4b9ac5afec3eca10341f69a3a8 Mon Sep 17 00:00:00 2001 From: Christos Gkoros Date: Wed, 9 Sep 2026 10:44:55 +0300 Subject: [PATCH] Measure the agent claim, and report that it does not hold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repository asserts that a model handed this schema writes better filters than it would against the alternatives, that its failures are loud rather than silent, and that a rejection is enough to repair in one round trip. None of it had been measured; nothing here had ever called a model. experiments/intent-to-filter puts sixty-three natural-language questions to six interfaces and grades by executed row sets. Sixty of the questions reuse the answer key in experiments/filter-to-sql/cases.mjs, hand-derived from SPEC.md months ago for an unrelated purpose, so it cannot have been chosen to flatter the language; three more are written here to tempt the mistakes README catalogues, and are never averaged in with the rest. Every arm is executed by the same compiler, so no arm can win on its own semantics, and every arm has a hand-written gold query per case that gold.test.mjs runs before anything is paid for. Seven predictions and their refutation conditions were committed before the run. On GPT-5.5, over 1,890 calls: a Lucene-style search string documented in six hundred tokens of prose answered 89% correctly, a prose-documented bespoke JSON dialect 86%, and the generated schema 79% at fifteen times the tokens per correct answer. The mechanism the repository argues for is absent — the generated schema had the highest silent-failure rate of the three working arms and a 4% repair yield, and the prose JSON arm scored the worst possible loud:silent ratio while still winning. Four of the seven predictions are refuted, including the headline. Two of the three that held were written against the repository's interest. Three findings needed no run at all. Gemini cannot receive the tool definition: no $ref, and the grammar is recursive so it cannot be flattened. OpenAI accepts it and silently discards $defs, which is 94% of the file, so the schema examples/mcp-server ships bills 704 tokens instead of 14,089 and the grammar never reaches the model. And the generated schema is not the narrowing the README claims: it accepts a lone $unknownAs that the published grammar rejects, which the compiler turns into invalid SQL. That last one is issue #8, left unfixed here on purpose. What survives is the internal argument. Generation is worth 35 points over the bare narrowed grammar, and the closed-domain case fails silently against the published grammar exactly as predicted. It is the comparative claim, and the agent-first framing built on it, that the evidence does not support. Co-Authored-By: Claude --- .gitignore | 1 + CHANGELOG.md | 35 + README.md | 1 + experiments/intent-to-filter/README.md | 474 +++ .../intent-to-filter/arms/bespoke-json.mjs | 295 ++ .../arms/docs/bespoke-json.md | 23 + .../arms/docs/search-string.md | 23 + experiments/intent-to-filter/arms/execute.mjs | 63 + .../intent-to-filter/arms/fixed-params.mjs | 122 + experiments/intent-to-filter/arms/index.mjs | 54 + experiments/intent-to-filter/arms/jql.mjs | 192 + .../intent-to-filter/arms/search-string.mjs | 409 ++ experiments/intent-to-filter/corpus.mjs | 44 + experiments/intent-to-filter/deref.mjs | 154 + experiments/intent-to-filter/differential.mjs | 207 + experiments/intent-to-filter/gold.test.mjs | 203 + experiments/intent-to-filter/grade.mjs | 208 + experiments/intent-to-filter/pet.filter.json | 2326 +++++++++++ .../intent-to-filter/pet.resource.json | 111 + experiments/intent-to-filter/providers.mjs | 247 ++ experiments/intent-to-filter/questions.mjs | 215 ++ experiments/intent-to-filter/report.mjs | 280 ++ .../intent-to-filter/results/full.jsonl | 3419 +++++++++++++++++ .../intent-to-filter/results/pilot.jsonl | 90 + experiments/intent-to-filter/runner.mjs | 209 + package-lock.json | 7 +- package.json | 5 +- 27 files changed, 9411 insertions(+), 6 deletions(-) create mode 100644 experiments/intent-to-filter/README.md create mode 100644 experiments/intent-to-filter/arms/bespoke-json.mjs create mode 100644 experiments/intent-to-filter/arms/docs/bespoke-json.md create mode 100644 experiments/intent-to-filter/arms/docs/search-string.md create mode 100644 experiments/intent-to-filter/arms/execute.mjs create mode 100644 experiments/intent-to-filter/arms/fixed-params.mjs create mode 100644 experiments/intent-to-filter/arms/index.mjs create mode 100644 experiments/intent-to-filter/arms/jql.mjs create mode 100644 experiments/intent-to-filter/arms/search-string.mjs create mode 100644 experiments/intent-to-filter/corpus.mjs create mode 100644 experiments/intent-to-filter/deref.mjs create mode 100644 experiments/intent-to-filter/differential.mjs create mode 100644 experiments/intent-to-filter/gold.test.mjs create mode 100644 experiments/intent-to-filter/grade.mjs create mode 100644 experiments/intent-to-filter/pet.filter.json create mode 100644 experiments/intent-to-filter/pet.resource.json create mode 100644 experiments/intent-to-filter/providers.mjs create mode 100644 experiments/intent-to-filter/questions.mjs create mode 100644 experiments/intent-to-filter/report.mjs create mode 100644 experiments/intent-to-filter/results/full.jsonl create mode 100644 experiments/intent-to-filter/results/pilot.jsonl create mode 100644 experiments/intent-to-filter/runner.mjs diff --git a/.gitignore b/.gitignore index aafcb34..1c5c1ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ .DS_Store *.log +.claude/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 1efdb68..142857e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,41 @@ No change to the grammar or to the semantics of evaluation. The only edit to under `properties.filter`, a bundled schema's self-references resolve against its own `$id`, so removing the `$id` breaks it — ajv fails to compile it at all. +- **`experiments/intent-to-filter/`** — a second exercise, pre-registered and not yet run. It + asks whether a model actually writes better queries against a JSON-Schema-described predicate + language than against a documented search string, a prose-documented bespoke JSON query + language, or fixed scalar parameters — the comparison README §Why asserts and the repository + has never measured. Sixty questions are inherited from `experiments/filter-to-sql/cases.mjs`, + whose answer key was hand-derived from SPEC.md months earlier and so cannot have been chosen to + flatter the language; three more are authored to tempt the three catalogued mistakes. Five arms + share one evaluator, so no arm can win on its own semantics. Seven predictions and their + refutation conditions are committed before the run; two of the seven predict against the + repository's interest. +- **The generated schema is not the narrowing the README says it is.** `examples/pet.filter.json` + accepts `{"microchip": {"$unknownAs": false}}`; the published grammar rejects it, via + `$defs/ConstraintObject.dependentSchemas.$unknownAs` (`minProperties: 2`), and + `tests/fixtures/invalid/24-unknownas-alone.json` pins that. The generator drops the rule. + `../filter-to-sql` then compiles the filter to `coalesce((), FALSE)`, which SQLite rejects — so a + server built as the README recommends answers 500 to a filter its own schema called valid. + README claims "Every filter the generated schema accepts is also valid against the published + grammar… `tests/generator.test.mjs` asserts this"; that test enumerates fifteen hand-written + filters, and a model found a sixteenth on its first pass. Recorded, not yet fixed: the repair + belongs in its own change and should replace the enumeration with a property check. +- **Two of three providers cannot carry the recommended tool definition.** Gemini rejects it with + a 400 — its function-declaration schema has no `$ref`, and the grammar is recursive, so it + cannot be flattened either. OpenAI accepts it and silently discards `$defs`: measured, the + inlined schema bills 704 input tokens against 14,089 for the same schema dereferenced first, so + 94% of the file — every operator description and every value domain — never reaches the model. + README §"Exposing search to an agent" says to inline the schema and mentions neither. +- Three further findings, all about the *recommended* setup: + a generated schema cannot express a `$field` cross-field comparison at all — the generator + narrows an operand to the field's own type, leaving the `refs` profile unreachable — emits no + indexed paths, and withholds pattern matching from enumerated domains. The published grammar + expresses all four cases the generated one cannot, so the recommended setup is *less* expressive + than the cheap one. None of this is currently documented. +- The experiment adds **no dependency**. It calls three providers over `fetch` because it has to + reach OpenAI, Google and Groq, and a single-vendor SDK would not have covered them. + ### Changed - **Repositioned around the agent case.** README now leads with the language as a search diff --git a/README.md b/README.md index b215ed7..5995631 100644 --- a/README.md +++ b/README.md @@ -427,6 +427,7 @@ tests/generator.test.mjs the generator: narrowing, soundness, recursion tests/fixtures/valid/ one per operator; also the docs' example set tests/fixtures/invalid/ every defect this version fixes, pinned experiments/filter-to-sql/ an exercise: compile a filter to SQL, then judge the design by it +experiments/intent-to-filter/ an exercise: put the same question to five interfaces, count what a model gets right .github/workflows/ci.yml tests on Node 20/22/24 + OpenAPI lint .github/workflows/release.yml verifies a GitHub Release; publishes nothing ``` diff --git a/experiments/intent-to-filter/README.md b/experiments/intent-to-filter/README.md new file mode 100644 index 0000000..c37901b --- /dev/null +++ b/experiments/intent-to-filter/README.md @@ -0,0 +1,474 @@ +# Experiment: from intent to filter + +**This is an exercise, not a deliverable.** Nothing here is published, nothing +here is versioned, and nothing in `../../query-language-schema.json`, +`../../SPEC.md` or `../../tools/` depends on it. It exists to answer one +question the repository has been asserting the answer to: + +> Given the same information need and the same documentation budget, does a +> model produce a *correct* query more often against a JSON-Schema-described +> predicate language than against a documented search string, a prose-documented +> bespoke JSON query language, or fixed scalar parameters — and, separately, +> does it *know* when it has failed? + +`README.md:3` says the model "needs nothing else to use it". `:37` says "no +prompt teaching the syntax, no few-shot examples, no steering". `:210` says a +rejection is "enough for an agent to repair its own request in one round trip", +and that every other approach fails as "an empty result set, which an agent +cannot distinguish from 'no such records'". `COMPARISON.md:107-110` says the +same to GraphQL and cites the README, which cites nothing. None of it has been +measured; nothing in this repository had ever called a model before this +directory existed. + +**The predictions are committed before the run.** The seven hypotheses below, +and the conditions that would refute each, are in this file's first commit; the +results go in a second one, reported against each numbered prediction including +the ones that miss. Two of the seven predict *against* the repository's +interest. They are the reason the other five are worth believing. + +**Status: run, and the headline prediction is refuted.** On GPT-5.5, across +1,890 calls, the two prose-documented arms beat both schema arms on accuracy +*and* on the silent-failure rate that this repository argues is the thing that +matters. Details under Results, reported against each numbered prediction. + +**The plan changed once, under duress.** It was designed for Claude Opus 5 and +Haiku 4.5. No working Anthropic key was available, and the three keys that did +work — OpenAI, Google, Groq — turned out to make a better experiment: the claim +under test is about "a model", not about one vendor's models, so three +independent families answer an objection two models from one family could not. + +--- + +## What is here + +| File | | +| --- | --- | +| `questions.mjs` | one natural-language question per case, the authoring rule, and the excluded set with reasons | +| `corpus.mjs` | the 63 graded questions, assembled | +| `arms/` | five interfaces: tool definition, validation, translation | +| `arms/docs/` | the two prose-documented arms' tool descriptions, verbatim | +| `differential.mjs` | 200 generated records, used only to tell two filters apart | +| `grade.mjs` | outcome classification and the mistake detector | +| `runner.mjs` | batch submission and collection | +| `report.mjs` | every number that appears below | +| `gold.test.mjs` | the thirteen checks that must pass before anything is paid for | + +```bash +node --test experiments/intent-to-filter/gold.test.mjs # 13 pre-flight checks +node experiments/intent-to-filter/report.mjs # the corpus, before any run +node experiments/intent-to-filter/runner.mjs --dry-run # cost estimate, sends nothing +node experiments/intent-to-filter/runner.mjs --pilot # 15 cases, one model, ~$1 +node experiments/intent-to-filter/runner.mjs --run # the full run +``` + +## Method + +### The answer key was written before the question + +`../filter-to-sql/cases.mjs` holds 72 cases, each with a gold filter and the row +set SPEC.md says it returns — *"derived by hand from the fixture records rather +than from any implementation"*, and cross-validated by compiling every case two +different ways. It was written months ago to measure compiler complexity. That +it long predates this experiment is the single best property here: the answer +key cannot have been chosen to flatter the language. + +What it lacks is an information need. Its `title` names the mechanism under test +— "The §4.1 surprise: `$not` over a nullable field" — which, handed to a model, +is the answer. So `questions.mjs` adds one question per case under a rule that +`gold.test.mjs` enforces: + +> A question may name fields and values. It may not contain an operator, a +> syntax token, or a structural concept belonging to any arm. + +Twelve cases cannot be phrased under that rule and are excluded, each with a +published reason. Two kinds recur, and they say different things: *no user +intent* (nobody asks for "a weight greater than the word heavy" — but a +specification still has to pin the behaviour down) and *not separable* (c06 has +a real intent, identical in both words and rows to c05; grading both would +double-count one question). Sixty survive, and three more are written here to +tempt the three mistakes the README catalogues. Those three are tagged +`authored` and never averaged into the inherited numbers. + +### Every arm is evaluated by the same evaluator + +Each arm translates whatever the model produced into a filter, and every filter +is then compiled and run by the same compiler from `../filter-to-sql`, against +the same ten records. No arm has its own semantics, so none can win or lose +because its executor treated a null differently. What an arm owns is its +translation and its rejections — which is the thing under test. + +### Right rows is not the same as right filter + +Across the 62 corpus cases with a row-set answer there are only **44 distinct +row sets** over ten records: `a01`, `a17` and `a20` all return the same six ids. +Grading on rows alone would pass a wrong filter that lands on the right rows, +and with ten records that is not long odds. + +So `differential.mjs` generates two hundred more records from a fixed seed, +dense where filters are easy to confuse — present-nulls beside absent keys, +empty arrays, values exactly on the range boundaries the corpus uses, the +occasional status outside the documented domain. A candidate counts as correct +only if it returns the gold rows on the ten authoritative records *and* agrees +with the gold filter on the two hundred. The ten keep their authority; the two +hundred only compare candidate to gold, so nothing here needs hand-checking. + +It works: the gold filters produce 44 distinct row sets over ten records and +**58 of 58** over two hundred, leaving no pair the grader cannot separate. + +### The arms + +| Arm | What the model sees | Tool tokens | Ceiling | +| --- | --- | --- | --- | +| `jql-generated` | a schema generated from the resource, per-field operators and domains | 15,407 | 4 | +| `jql-published` | the published grammar, `FieldPath` narrowed. No domains | 3,656 | 0 | +| `search-string` | `{q: string}` and 647 tokens of syntax | 741 | 10 | +| `bespoke-json` | `{where: object}` and 747 tokens of prose | 873 | 0 | +| `fixed-params` | ten typed scalar parameters | 311 | 54 | + +**`search-string` is not a strawman.** Every construct is lifted from Lucene, +Elasticsearch query-string or GitHub search: wildcards, `[lo TO hi]` ranges, +`/regex/i`, `_exists_`, `_missing_`, quoted field names, boolean grouping, +`count()` and `first()`. Its parser is deliberately generous, accepting every +variant spelling it can — a stingy parser would manufacture rejections and hand +the comparison to the schema arms, which is the easiest way to fake this whole +experiment. On one axis the string syntax is frankly *better* than the filter +grammar: `tags:indoor` means any-element without naming a quantifier, which is +exactly the mistake `$in` invites. + +**`bespoke-json` is the real control.** It is near-isomorphic to the filter +grammar, resolution of unknowns included, so its ceiling is empty by design. The +claim under test is that describing a language *with a schema* beats describing +it *with prose*; holding the language constant is the only way to isolate that. +It is deliberately not Mongo-shaped — the filter grammar is Mongo-adjacent +(`COMPARISON.md:174`), and a `$eq`/`$in` competitor would measure familiarity. + +**The documentation budget binds the two prose arms only**, at 700 tokens ±10%. +It does not bind the schema arms, whose whole claim is that they need no prose, +and `gold.test.mjs` asserts their descriptions stay under 100 tokens and never +mention `$unknownAs`, `$some`, three-valued logic or nulls. The budget was set +*after* both prose documents were drafted, at the length the longer needed, and +the shorter was left as written rather than padded — fixing the number first and +cutting to hit it would mean deciding how good a competitor's documentation is +allowed to be. + +### A ceiling is not a failure + +Every arm has a hand-written gold query for every case, and `gold.test.mjs` +executes all of them: 63 cases × 5 arms, each asserted to return the gold answer +through that arm's own executor. Where an interface provably cannot put a +question, the case is counted as that arm's **expressivity ceiling** and dropped +from its accuracy denominator, not marked wrong. Without this, `fixed-params` +would score near zero for reasons that have nothing to do with how well a model +uses it. + +Four of `jql-generated`'s ceiling entries are findings in themselves, and the +README does not currently mention any of them. A generated schema cannot express +a `$field` cross-field comparison (the generator narrows `$gt`'s operand to the +field's own type, leaving no room for a reference operand, so the whole `refs` +profile is unreachable); it withholds pattern matching from enumerated domains, +which is correct and still costs the case; and it emits no indexed paths. +`jql-published` expresses all four. The recommended setup is *less* expressive +than the cheap one, and that trade-off is currently undocumented. + +### Outcomes + +| Class | | +| --- | --- | +| `correct` | gold rows on the ten, and equivalent to the gold filter on the two hundred | +| `loud-failure` | rejected by validation or parse — **the agent is told** | +| `silent-failure` | accepted, executed, wrong rows — **the agent is not told** | +| `fabrication` | an unanswerable question answered with an invented field | +| `correct-abstention` | an unanswerable question reported as unaskable | +| `no-call` / `refusal` | no tool call, or the model declined | + +**The headline is the silent-failure rate and the loud:silent ratio, not raw +accuracy.** For an agent, an interface that is wrong often but always says so +beats one that is wrong half as often and never does. That is the repository's +actual argument, and it is the one worth testing. + +Five trials per cell, with Wilson intervals: sampling parameters are rejected on +Opus 5, so runs are non-deterministic and a single-shot number would be +dishonest. Repair is measured as `acc@2 − acc@1` over turn-1 loud failures, with +the rejection fed back as a tool result flagged `is_error`, exactly as +`examples/mcp-server` returns it. + +## Predictions, committed before the run + +| # | Prediction | Refuted if | +| --- | --- | --- | +| H1 | `jql-generated` has the lowest silent-failure rate: ≤10%, against ≥25% for `search-string` | the gap is under 10 points — the central README claim is then unsupported and has to be softened | +| H2 | `jql-generated` has **more** turn-1 loud failures than `search-string` — a bigger surface to get wrong — but they convert | it has fewer of both, which is a stronger result than predicted and should be said so | +| H3 | Repair yield ≥15 points for the schema arms, ≤5 for `search-string` | the schema arms yield ≤5 — "one round trip" is rhetoric, and five places in the repository need rewriting | +| H4 | `jql-published`'s silent rate is 2–3× `jql-generated`'s: same syntax, no domains | they are equal — then the generator is not earning its 4.2× token cost | +| H5 | The `$ne`-drops-nulls mistake is **not** reduced by the generated schema, because it is prose-only; the ablation shows `SILENT_RULES` is what moves it | the generated schema reduces it anyway — then the validation/prose split is wrong | +| H6 | `fixed-params` cannot express ≥40% of the corpus, but is the **most accurate** arm on what it can | it is not most accurate on its answerable subset — the ceiling argument is weaker than assumed | +| H7 | `jql-generated` costs ~20× `search-string`'s input tokens; at low volume the string arm wins on accuracy-per-dollar | the schema arm is cost-competitive per correct answer — a stronger claim than the repository currently makes | + +H6's ceiling is already known, before any model is called: `fixed-params` can +put **9 of 63** questions, an 86% ceiling against the 40% predicted. The +prediction is recorded as written and scored as a miss. + +### What each outcome would change + +- **H1, H3, H4 holding** does not license "agents are better at this language". + It licenses a narrower and more defensible claim — *fails loudly instead of + silently, and recovers* — with a number attached. `README.md:37` and `:210` + get rewritten to that, and `COMPARISON.md:107-110` gets a citation instead of + a cross-reference to itself. +- **H5 holding** is actionable: add `$ne` and `$nbetween` to the generator's + `SURPRISING` set (`tools/generate-filter-schema.mjs:65-70`). `$not` keeps its + prose under the default `brief`; `$ne`, where the canonical mistake actually + lives, does not. +- **H6 and H7** give the boundary — when *not* to reach for this. A repository + that publishes its own boundary is why the rest of it is believable. + +## Findings that did not need the run + +Three, all about the path README §"Exposing search to an agent" tells the reader +to take. Each is reproducible from this directory. + +### 1. Gemini cannot host the recommended setup at all + +Its function-declaration schema is a restricted subset with **no `$ref` field**. +Inlining `pet.filter.json` into a tool declaration returns HTTP 400. The schema +cannot be flattened into a finite tree either, because the grammar is recursive +(`$and`, `$or`, `$nor` and `$not` all take `{"$ref": "#"}`), so there is no +version of the recommended setup that Gemini will accept. Both schema arms are +unavailable there, and on that provider the string syntax wins by walkover. + +### 2. OpenAI accepts the schema and silently discards 94% of it + +The generated schema is 61,354 characters, of which the `$defs` block is 57,756 +— every operator description and every per-field value domain. OpenAI's function +parameters do not carry `$defs`. The request succeeds, with no warning: + +| tool as sent | input tokens billed | +| --- | --- | +| `pet.filter.json` inlined, as the README describes | **704** | +| the same schema with every `$ref` resolved first | **14,089** | + +At 704 tokens the model is receiving a list of field names and a set of dangling +references. The grammar never arrives. `examples/mcp-server` ships the schema +this way, and `README.md:232` prices it at "about 42 KB… paid once per session". + +Flattening is the fix, and it is not free: it has to cut the recursion at a fixed +depth, so the model is told nesting stops sooner than the server will accept. +`deref.mjs` measures the trade — depth 1 is 65,775 characters, depth 2 is +245,904, depth 3 is 280,526. The experiment runs `jql-raw` and `jql-generated` +side by side so the gap between the documented setup and a working one is a row +in the results rather than a claim in this paragraph. + +### 3. The generated schema is not a narrowing, and the soundness test cannot see it + +Found by gpt-5.4-mini during the pilot, not by hand. Answering c05 it emitted: + +```json +{"microchip": {"$unknownAs": false}} +``` + +The published grammar **rejects** this. `$defs/ConstraintObject` carries + +```json +"dependentSchemas": { "$unknownAs": { "minProperties": 2 } } +``` + +because `$unknownAs` is a modifier and needs an operator to modify — pinned by +`tests/fixtures/invalid/24-unknownas-alone.json`. The generator does not emit +that rule, so every generated per-field constraint object accepts a lone +`$unknownAs`. The committed `examples/pet.filter.json` accepts it too. + +Downstream, `../filter-to-sql` compiles it to `coalesce((), FALSE)` and SQLite +rejects the statement. A server built exactly as the README recommends answers +500 to a filter its own published schema called valid. + +This contradicts `README.md`: + +> **Narrowing only.** Every filter the generated schema accepts is also valid +> against the published grammar… `tests/generator.test.mjs` asserts this. + +It does not assert it. It iterates a hand-written list of fifteen filters +somebody thought of. The property is universally quantified and the test is an +enumeration, so it can only ever find the counterexamples already known. A model +found a new one on its first pass. + +Recorded here and deliberately left unfixed: the repair belongs in its own +change, against `tools/generate-filter-schema.mjs`, and it should replace the +enumeration with a property check that fuzzes generated schemas against the +published grammar. + +## Results + +**GPT-5.5, 63 questions x 6 arms x 5 trials, 1,890 calls, no failures.** This is +the substantive result. The other two providers could not complete: Groq +rate-limited and Gemini exhausted its free-tier daily quota, so their accuracy +columns are too thin to read and are reported only for the infrastructure +findings above. + +``` + arm n correct loud silent loud:silent + jql-raw 295 66% ± 5 24% 4% 6.0 + jql-generated 295 79% ± 5 9% 9% 1.0 + jql-published 315 44% ± 5 39% 6% 6.2 + search-string 265 89% ± 4 2% 5% 0.4 + bespoke-json 315 86% ± 4 0% 7% 0.0 + fixed-params 45 100% ± 4 0% 0% — +``` + +A documented search string beat a generated JSON Schema by ten points, and a +prose-documented bespoke JSON language beat it by seven — at a fifteenth of the +token cost. The schema arms were not close. + +### Against the predictions + +| # | Predicted | Measured | | +| --- | --- | --- | --- | +| H1 | `jql-generated` lowest silent rate: ≤10% against ≥25% for `search-string` | 9% against **5%**. `jql-generated` has the *highest* silent rate of the three working arms | **refuted** | +| H2 | more loud failures than `search-string`, but they convert | more loud (9% vs 2%), and they do not convert — 4% repair yield | **half refuted** | +| H3 | repair yield ≥15% for schema arms, ≤5% for `search-string` | `jql-generated` 4%, `jql-published` 2%. The only arm that repairs well is `jql-raw` at 36%, because its errors are trivial ones caused by the model never seeing the schema | **refuted** | +| H4 | `jql-published` silent rate 2–3× `jql-generated`'s | 6% against 9% — the *published* grammar is lower. It fails loudly instead, at 39% | **refuted** | +| H5 | the `$ne`/nulls mistake is not reduced by the generated schema | 5% against 8% published and 6% raw. Within noise; the prose arms are 6% and 10% | **confirmed** | +| H6 | `fixed-params` unanswerable for ≥40%, most accurate on the rest | 86% unanswerable, and 100% correct on the nine it can put | **confirmed** (the 40% was already a recorded miss) | +| H7 | ~10–20× the input tokens of `search-string`; the string wins per-dollar | 14,086 against 938 tokens a call, and **19,044 tokens per correct answer against 1,243** — 15× the cost *and* lower accuracy | **confirmed, and worse than predicted** | + +### What did work + +Within the JQL family the design decisions pay off in the order the repository +claims, and by large margins: `jql-published` 44% → `jql-raw` 66% → generated +and flattened 79%. Generation is worth 35 points over the bare grammar. So the +generator earns its place — the argument it does not survive is the one against +*other interfaces*. + +The clearest single win is the authored probe m03, "Which pets have the status +Available?", where the closed domain is stated in the schema and nowhere else: + +``` + m03 — value-outside-domain + jql-generated correct 5/5 loud 0 silent 0 + jql-raw correct 0/5 loud 5 silent 0 + jql-published correct 0/5 loud 0 silent 5 <- the predicted failure + search-string correct 5/5 loud 0 silent 0 + bespoke-json correct 5/5 loud 0 silent 0 +``` + +`jql-published` fails exactly as README §"Exposing search to an agent" says it +will — five silent empty result sets. The generated schema fixes it. But so do +both prose arms, for a reason the README does not consider: the model simply +normalised the case on its own. The domain in the schema is doing work that the +model was going to do anyway. + +The other two catalogued mistakes barely appeared. `$in`-as-membership: zero +occurrences on any arm. `$ne`-drops-nulls was the only common mistake, at 5–10% +everywhere, and no arm's design reduced it much. + +### What this costs the repository's claims + +`README.md:3` — "the model needs nothing else to use it" — survives in the +narrow sense that no prompt was needed. It does not survive as an argument for +*choosing* this interface: on this corpus, 600 tokens of prose describing a +Lucene-style string outperformed 14,000 tokens of generated schema. + +`README.md:210`, that other approaches fail as silent empty result sets while +this one fails loudly, is contradicted. `bespoke-json` had a **0.0** loud:silent +ratio and still beat the schema arms on accuracy; `jql-generated` had the worst +silent rate of the three. + +`COMPARISON.md:107-110` — "a great deal from a grammar it can be handed, +validated against, and corrected by in one round trip" — has now been measured +in all three parts. Handed: only one of three providers can receive it. +Validated against: yes, and it rejects 9% of what the model writes. Corrected in +one round trip: 4%. + +## Verdict + +**The agent-first positioning is not supported by this run and should be withdrawn +or heavily qualified.** The language is fine. The comparative claim made about it +is what fails, and it fails precisely where it is loudest. + +**1. The claim is specifically about agents, and agents is where it loses.** +`README.md:3` calls this "a search interface for agents". Six hundred tokens of +prose describing a Lucene-style string produced correct answers 89% of the time; +fourteen thousand tokens of generated JSON Schema produced 79%. Per correct +answer that is 1,243 tokens against 19,044 — fifteen times the cost for ten +points less accuracy. A bespoke JSON language documented only in prose, with no +schema at all, also beat it, at 86%. + +**2. The mechanism the repository actually argues for did not appear.** The case +in `README.md:210` is not that this is more accurate; it is that other approaches +"fail as an empty result set, which an agent cannot distinguish from 'no such +records'" while this one fails loudly and recovers in one round trip. Measured, +`jql-generated` had the **highest** silent-failure rate of the three working arms +— 9%, against 5% for the search string and 7% for the prose JSON — and a repair +yield of **4%**, not one round trip. `bespoke-json` scored a loud:silent ratio of +0.0, the worst possible on this metric, and still beat both schema arms. + +**3. It cannot be delivered to two of the three providers tested.** Gemini's +function declarations have no `$ref`, and the grammar is recursive so it cannot +be flattened; after flattening anyway, `dependentRequired` and `patternProperties` +are missing too. Groq rejects the flattened schema as too large and fails +tool-call validation on the raw one. OpenAI accepts the raw schema and silently +discards the 94% of it that lives in `$defs`, so it only works if you dereference +it yourself first — which the README does not mention and `examples/mcp-server` +does not do. An interface that reaches one of three major providers intact is not +yet a search interface for agents. + +**4. The mistakes it prevents are largely mistakes a strong model no longer +makes.** This is the uncomfortable one. `$in`-as-array-membership, one of the +three canonical failures the README is built around, occurred **zero times on +every arm**. On the closed-domain probe the generated schema scored 5/5 — and so +did the search string and the prose JSON, because the model normalised "Available" +to "available" without being told the domain. The domain in the schema was doing +work the model was going to do anyway. Only `$ne`-drops-nulls was common, at 5–10% +across every arm, and no arm's design reduced it meaningfully. + +### What survives + +Within the family, the design decisions pay off in the claimed order and by large +margins: the bare narrowed grammar 44%, shipped raw 66%, generated and flattened +79%. **Generation is worth 35 points**, so the generator earns its place. And the +one case where the published grammar fails exactly as predicted — five silent +empty result sets on `{"status": "Available"}` — is exactly the case generation +fixes. The internal argument of this repository is sound. The external one is not. + +### What would change the verdict + +One model completed. Groq and Gemini ran out of rate limit and quota, so the +cross-provider accuracy comparison the design was built for does not exist. Sixty +three questions, one resource, ten records. And the comparison is against a search +syntax and a bespoke JSON dialect *that this experiment wrote*, carefully but +self-interestedly. + +Most importantly: nothing here contradicts the narrower claim that one grammar +across every search endpoint in an API, validated in CI and typed in generated +clients, is worth having. That claim was the repository's original one, before +the agent framing was put in front of it. The evidence supports going back to it. + +### Recommended changes + +- Withdraw "A search interface for agents" as the lead. The measured comparison + does not support it. +- Replace `README.md:210`'s loud-versus-silent argument with the numbers, or drop + it. As written it is contradicted. +- Document the deliverability problem in §"Exposing search to an agent": the + schema must be dereferenced before it reaches any provider, recursion must be + cut at a depth, and Gemini cannot take it at all. +- Give `COMPARISON.md:107-110` a citation or delete the paragraph. +- Fix the generator bug in issue #8 — separate change. + +## What this does not show + +- **Two models, one provider.** Nothing here separates a property of this + language from a property of how these two models were trained. +- **One resource, ten records.** The fixture is engineered for discrimination, + not for realism. Nothing here predicts behaviour on a fifty-field resource. +- **Single-turn, one tool.** A real agent picks among tools, reads results, and + asks again. Only the repair turn touches any of that. +- **`strict: true` is off.** Constrained decoding would drive the schema arms to + 100% structural validity by construction — and it is available to them and to + no other arm, at any price. That is a real advantage this experiment declines + to bank, because MCP servers do not use it today. +- **The mistake counts are upper bounds.** The detector reads shape, not intent: + four gold answers trip it, because excluding the unknowns is exactly what + those questions asked for. +- **The questions are ours.** The answer key is inherited and the leakage rule is + enforced, but sixty-three questions written by the language's author are still + sixty-three questions written by the language's author. diff --git a/experiments/intent-to-filter/arms/bespoke-json.mjs b/experiments/intent-to-filter/arms/bespoke-json.mjs new file mode 100644 index 0000000..967784e --- /dev/null +++ b/experiments/intent-to-filter/arms/bespoke-json.mjs @@ -0,0 +1,295 @@ +/** + * bespoke-json.mjs — the control that isolates the actual variable. + * + * COMPARISON.md:29-30 says an agent "is handed an `inputSchema` and left to + * infer a query syntax from prose". This arm is that: a JSON query language of + * the kind a team invents for one API, with `where: {type: "object"}` as its + * entire schema and a prose paragraph as its entire documentation. + * + * Two design decisions make it a control rather than a strawman. + * + * **It is near-isomorphic to the filter grammar.** Every operator has a + * counterpart, resolution of unknowns included. The claim under test is that + * *describing a language with JSON Schema* beats *describing it with prose* — + * so the language has to be held roughly constant, or the experiment measures + * expressivity instead. Its CEILING is empty, and that is deliberate. + * + * **It is not Mongo-shaped.** Models carry a large MongoDB prior, and the + * filter grammar is deliberately Mongo-adjacent (COMPARISON.md:174). A bespoke + * arm spelled with `$eq` and `$in` would be measuring familiarity. So this one + * uses `{op, value}` pairs and word operators — the other common house style, + * and one no model has memorised. + */ + +import { runFilter, problem } from "./execute.mjs"; + +const OPS = { + is: "$eq", isnot: "$ne", + gt: "$gt", gte: "$gte", lt: "$lt", lte: "$lte", + between: "$between", notbetween: "$nbetween", + anyof: "$in", noneof: "$nin", + exists: "$exists", isblank: "$isNull", + prefix: "$startsWith", suffix: "$endsWith", substring: "$contains", + like: "$like", ilike: "$ilike", matches: "$regex", + type: "$type", hasall: "$hasAll", +}; + +/** Operators whose operand is itself a condition rather than a value. */ +const NESTED = { some: "$some", every: "$every", count: "$size" }; + +const FIELDS = { + id: "id", name: "name", species: "species", status: "status", born: "born", + weightKg: "weightKg", neutered: "neutered", microchip: "microchip", + tags: "tags", notes: "notes", priceCents: "priceCents", costCents: "costCents", + rate: "$$rate", sizeRaw: "size\\.raw", + "shelter.name": "shelter.name", "shelter.city": "shelter.city", + "shelter.capacity": "shelter.capacity", + vaccinations: "vaccinations", + vaccine: "vaccine", administeredAt: "administeredAt", boosterDue: "boosterDue", +}; + +class BespokeError extends Error { + constructor(detail, type = "malformed-query", pointer = "/where") { + super(detail); + this.detail = detail; + this.type = type; + this.pointer = pointer; + } +} + +const isPlainObject = (v) => v !== null && typeof v === "object" && !Array.isArray(v); + +/** A `{op, value}` condition -> a constraint object. `inElement` relaxes field lookup. */ +function condition(node, pointer) { + if (!isPlainObject(node)) { + throw new BespokeError(`expected a condition object at ${pointer}.`, "malformed-query", pointer); + } + // A nested where-object (used as the body of `some`/`every`) rather than a condition. + if (!("op" in node)) return where(node, pointer); + + const { op, value, flags, orUnknown, negate } = node; + const inner = {}; + + if (op in NESTED) { + inner[NESTED[op]] = op === "count" && !isPlainObject(value) ? value : condition(value, `${pointer}/value`); + } else if (op === "first") { + throw new BespokeError("`first` may only appear directly under a field.", "malformed-query", pointer); + } else if (op === "fieldref") { + throw new BespokeError("`fieldref` may only appear as the value of a comparison.", "malformed-query", pointer); + } else if (op in OPS) { + inner[OPS[op]] = isPlainObject(value) && value.op === "fieldref" + ? { $field: resolve(value.value, pointer) } + : value; + if (op === "matches" && flags) inner.$flags = flags; + } else { + throw new BespokeError( + `"${op}" is not an operator. The operators are: ${[...Object.keys(OPS), ...Object.keys(NESTED), "first", "fieldref"].join(", ")}.`, + "unsupported-operator", + pointer, + ); + } + + // `negate: true` negates this field's constraint; `orUnknown` then resolves + // whatever is left UNKNOWN, which is the outer position of the two. + const constraint = negate === true ? { $not: inner } : inner; + + if (orUnknown === true) constraint.$unknownAs = true; + if (orUnknown === false) constraint.$unknownAs = false; + return constraint; +} + +function resolve(name, pointer) { + const path = FIELDS[name]; + if (path === undefined) { + throw new BespokeError( + `"${name}" is not a queryable field. The fields are: ${Object.keys(FIELDS).join(", ")}.`, + "unknown-field", + `${pointer}/${name}`, + ); + } + return path; +} + +const LOGIC = { all: "$and", any: "$or", none: "$nor" }; + +/** A where-object -> a filter. */ +function where(node, pointer = "/where") { + if (!isPlainObject(node)) { + throw new BespokeError("the where argument must be an object.", "malformed-query", pointer); + } + const entries = Object.entries(node); + if (entries.length === 0) { + throw new BespokeError("the where object must carry at least one condition.", "malformed-query", pointer); + } + + const filter = {}; + for (const [key, value] of entries) { + const at = `${pointer}/${key}`; + if (key in LOGIC) { + if (!Array.isArray(value) || value.length === 0) { + throw new BespokeError(`"${key}" takes a non-empty list of conditions.`, "malformed-query", at); + } + filter[LOGIC[key]] = value.map((member, index) => where(member, `${at}/${index}`)); + continue; + } + if (key === "not") { filter.$not = where(value, at); continue; } + + // `first` owns the path, so it is unwrapped here rather than in condition(). + if (isPlainObject(value) && value.op === "first") { + const path = resolve(key, pointer); + const body = value.value; + // `first` on an array of objects addresses a member of element zero; + // on an array of scalars it addresses element zero itself. + if (isPlainObject(body) && !("op" in body)) { + for (const [member, cond] of Object.entries(body)) { + filter[`${path}[0].${resolve(member, pointer)}`] = condition(cond, `${at}/value/${member}`); + } + } else { + filter[`${path}[0]`] = condition(body, `${at}/value`); + } + continue; + } + filter[resolve(key, pointer)] = condition(value, at); + } + return filter; +} + +/** + * The gold query per case. Isomorphic to the filter grammar by construction, + * so every case the grammar can express has an entry — which is the point. + */ +const GOLD = { + a01: { status: { op: "is", value: "available" } }, + a02: { status: { op: "is", value: "available" }, species: { op: "is", value: "cat" } }, + a03: { status: { op: "anyof", value: ["available", "pending"] } }, + a04: { weightKg: { op: "between", value: [1, 5] } }, + a05: { born: { op: "gte", value: "2020-01-01" } }, + a06: { name: { op: "prefix", value: "b" } }, + a07: { name: { op: "ilike", value: "%o%" } }, + a08: { name: { op: "substring", value: "50%" } }, + a09: { tags: { op: "some", value: { op: "anyof", value: ["indoor", "small"] } } }, + a10: { tags: { op: "hasall", value: ["trained", "outdoor"] } }, + a11: { tags: { op: "count", value: { op: "gte", value: 3 } } }, + a12: { tags: { op: "count", value: 0 } }, + a13: { "shelter.city": { op: "is", value: "Athens" } }, + a14: { "shelter.capacity": { op: "lte", value: 12 } }, + a15: { any: [ + { species: { op: "is", value: "cat" }, status: { op: "is", value: "available" } }, + { species: { op: "is", value: "dog" }, weightKg: { op: "lt", value: 20 } }, + ] }, + a16: { all: [ + { status: { op: "is", value: "available" } }, + { any: [ + { "shelter.city": { op: "is", value: "Patras" } }, + { tags: { op: "some", value: { op: "anyof", value: ["trained"] } } }, + ] }, + ] }, + a17: { none: [{ status: { op: "is", value: "sold" } }, { status: { op: "is", value: "pending" } }] }, + a18: { neutered: { op: "is", value: true } }, + a19: { priceCents: { op: "gt", value: { op: "fieldref", value: "costCents" } } }, + a20: { status: { op: "matches", value: "^a" } }, + b01: { vaccinations: { op: "some", value: { + vaccine: { op: "is", value: "rabies" }, + boosterDue: { op: "lt", value: "2025-01-01" }, + } } }, + b02: { all: [ + { vaccinations: { op: "some", value: { vaccine: { op: "is", value: "rabies" } } } }, + { vaccinations: { op: "some", value: { boosterDue: { op: "lt", value: "2025-01-01" } } } }, + ] }, + b03: { vaccinations: { op: "some", value: { administeredAt: { op: "gte", value: "2025-01-01T00:00:00Z" } } } }, + b04: { vaccinations: { op: "first", value: { vaccine: { op: "is", value: "parvo" } } } }, + b05: { vaccinations: { op: "some", value: { vaccine: { op: "anyof", value: ["parvo"] } } } }, + b06: { tags: { op: "some", value: { op: "anyof", value: ["indoor"] } } }, // negated below + b07: { tags: { op: "some", value: { op: "isnot", value: "indoor" } } }, + b08: { tags: { op: "first", value: { op: "is", value: "indoor" } } }, + b09: { vaccinations: { op: "some", value: { boosterDue: { op: "exists", value: true } } } }, + b10: { vaccinations: { op: "some", value: { boosterDue: { op: "exists", value: false } } } }, + b11: { vaccinations: { op: "every", value: { vaccine: { op: "is", value: "rabies" } } } }, + b12: { tags: { op: "every", value: { op: "isnot", value: "indoor" } } }, + b13: { not: { tags: { op: "some", value: { op: "isnot", value: "indoor" } } } }, + c01: { microchip: { op: "isblank", value: true } }, + c02: { microchip: { op: "exists", value: false } }, + c03: { notes: { op: "exists", value: true } }, + c04: { "shelter.capacity": { op: "exists", value: false } }, + c05: { microchip: { op: "isnot", value: "CHIP-001" } }, + c07: { any: [ + { microchip: { op: "isnot", value: "CHIP-001" } }, + { microchip: { op: "isblank", value: true } }, + ] }, + c08: { not: { status: { op: "is", value: "sold" } } }, + c11: { weightKg: { op: "notbetween", value: [1, 5] } }, + c12: { microchip: { op: "isnot", value: "CHIP-001", orUnknown: true } }, + c13: { born: { not: true, op: "gte", value: "2020-01-01", orUnknown: true } }, + d01: { notes: { op: "type", value: "number" } }, + d02: { notes: { op: "type", value: "integer" } }, + d03: { notes: { op: "type", value: "array" } }, + d04: { notes: { op: "type", value: "null" } }, + d05: { notes: { op: "is", value: 3 } }, + d06: { notes: { op: "gt", value: 2 } }, + d07: { notes: { op: "gt", value: "a" } }, + d08: { notes: { op: "noneof", value: [3] } }, + d09: { notes: { not: true, op: "gt", value: 2 } }, + e01: { rate: { op: "gt", value: 5 } }, + e02: { sizeRaw: { op: "is", value: "XS" } }, + e03: { name: { op: "like", value: "C%o" } }, + e04: { name: { op: "matches", value: "^[A-C]" } }, + e05: { name: { op: "matches", value: "^[a-c]", flags: "i" } }, + e06: { name: { op: "suffix", value: "s" } }, + f01: { internalNotes: { op: "is", value: "anything" } }, + f02: { name: { op: "fulltext", value: "ada" } }, + m01: { tags: { op: "some", value: { op: "anyof", value: ["indoor"] } } }, + m02: { microchip: { op: "isnot", value: "CHIP-001", orUnknown: true } }, + m03: { status: { op: "is", value: "available" } }, +}; + +// b06 is "no element is indoor" — the negation of its entry above. +GOLD.b06 = { not: GOLD.b06 }; +// c13/d09 use a field-level negation, which the surface spells as `not: true`. +GOLD.c13 = { born: { op: "gte", value: "2020-01-01", negate: true, orUnknown: true } }; +GOLD.d09 = { notes: { op: "gt", value: 2, negate: true } }; + +export const bespokeJson = { + name: "bespoke-json", + kind: "json", + ceiling: {}, + tool: { + name: "search_pets", + description: null, // filled from arms/docs/bespoke-json.md by index.mjs + input_schema: { + type: "object", + properties: { + where: { type: "object", description: "The query. See the tool description for the shape." }, + limit: { type: "integer", minimum: 1, maximum: 50, default: 20, description: "Maximum number of records to return." }, + }, + required: ["where"], + additionalProperties: false, + }, + }, + extract: (input) => input?.where, + translate(input) { + if (!isPlainObject(input?.where)) { + throw new BespokeError("the where argument is required and must be an object."); + } + return where(input.where); + }, + validate(input) { + try { + this.translate(input); + return null; + } catch (error) { + if (error instanceof BespokeError) return problem(error.type, error.detail, error.pointer); + throw error; + } + }, + execute(input, db) { + let filter; + try { + filter = this.translate(input); + } catch (error) { + if (error instanceof BespokeError) return { problem: problem(error.type, error.detail, error.pointer) }; + throw error; + } + return runFilter(db, filter); + }, + gold: (testCase) => GOLD[testCase.id] ?? null, +}; diff --git a/experiments/intent-to-filter/arms/docs/bespoke-json.md b/experiments/intent-to-filter/arms/docs/bespoke-json.md new file mode 100644 index 0000000..86043d4 --- /dev/null +++ b/experiments/intent-to-filter/arms/docs/bespoke-json.md @@ -0,0 +1,23 @@ +Search the pet collection. `where` is a query object. Returns the matching records. + +**Shape.** Each key of `where` is a field name; each value is a condition `{"op": ..., "value": ...}`. Sibling keys are ANDed. Group with `{"all": [...]}`, `{"any": [...]}`, `{"none": [...]}`, or negate with `{"not": {...}}`. + +**Operators.** `is`, `isnot`, `gt`, `gte`, `lt`, `lte`, `between` (a two-element list, inclusive), `notbetween`, `anyof` (a list — true if the field equals any member), `noneof`, `prefix`, `suffix`, `substring`, `like` (`%` any run, `_` one character), `ilike` (case-insensitive `like`), `matches` (a regular expression; add `"flags": "i"` for case-insensitivity), `type` (`string`, `number`, `integer`, `boolean`, `object`, `array`, `null`), `exists` (`true` or `false`), `isblank`. + +**Negating one field.** Add `"negate": true` to a condition to negate just that field's test. + +**Blanks and missing values.** A field can be present with a value, present but blank, or absent. `exists` distinguishes present from absent; `isblank` distinguishes blank from valued. Comparisons against a blank or absent field are *unknown*, and unknown never matches — so `isnot` and `negate` both leave those records out. To include them, add `"orUnknown": true` to the same condition, which resolves that condition's unknowns to true. + +**Lists.** `tags` and `vaccinations` hold lists. Reach their elements with `{"op": "some", "value": ...}` (at least one element satisfies it) or `{"op": "every", "value": ...}` (all do; true for an empty list). The value is a condition for a list of scalars, or a nested `where` object for a list of records. `{"op": "count", "value": ...}` tests list length; `{"op": "first", "value": ...}` tests the first element only. Two separate `some` conditions may be satisfied by two different elements — put both tests inside one `some` to require the same element. + +**Comparing two fields.** Use `{"op": "gt", "value": {"op": "fieldref", "value": "costCents"}}`. + +**Fields.** `id`, `name`, `species`, `status`, `born`, `weightKg`, `neutered`, `microchip`, `tags`, `notes`, `priceCents`, `costCents`, `rate`, `sizeRaw`, `shelter.name`, `shelter.city`, `shelter.capacity`, `vaccinations`; inside a `vaccinations` condition, `vaccine`, `administeredAt`, `boosterDue`. + +`status` is one of `available`, `pending`, `sold`. `species` is one of `cat`, `dog`, `rabbit`, `bird`. `vaccine` is one of `rabies`, `distemper`, `parvo`. `born` and `boosterDue` are dates (`YYYY-MM-DD`); `administeredAt` is a timestamp. `notes` is unstructured and holds a different type on every record. + +**Examples.** +`{"status": {"op": "is", "value": "available"}, "species": {"op": "anyof", "value": ["cat", "dog"]}}` +`{"weightKg": {"op": "between", "value": [1, 5]}}` +`{"any": [{"shelter.city": {"op": "is", "value": "Athens"}}, {"tags": {"op": "some", "value": {"op": "is", "value": "indoor"}}}]}` +`{"vaccinations": {"op": "some", "value": {"vaccine": {"op": "is", "value": "rabies"}}}}` diff --git a/experiments/intent-to-filter/arms/docs/search-string.md b/experiments/intent-to-filter/arms/docs/search-string.md new file mode 100644 index 0000000..5165084 --- /dev/null +++ b/experiments/intent-to-filter/arms/docs/search-string.md @@ -0,0 +1,23 @@ +Search the pet collection. `q` is a query string. Returns the matching records. + +**Syntax.** `field:value` matches a field. Combine clauses with `AND`, `OR`, `NOT` and parentheses; adjacent clauses are ANDed. Prefix a clause with `-` to negate it. Quote a value with `"` when it contains spaces, or to stop `*` being read as a wildcard. Quote a *field* name with `"` when the name itself contains a dot. + +**Comparison.** `field:>x`, `field:>=x`, `field:=3` matches on list length; `first(field):value` matches the first element only. Two separate clauses against the same list may be satisfied by two different elements. + +**Fields.** `id`, `name`, `species`, `status`, `born`, `weightKg`, `neutered`, `microchip`, `tags`, `notes`, `priceCents`, `costCents`, `$rate`, `"size.raw"`, `shelter.name`, `shelter.city`, `shelter.capacity`, `vaccinations.vaccine`, `vaccinations.administeredAt`, `vaccinations.boosterDue`. + +`status` is one of `available`, `pending`, `sold`. `species` is one of `cat`, `dog`, `rabbit`, `bird`. `vaccinations.vaccine` is one of `rabies`, `distemper`, `parvo`. `born` and `vaccinations.boosterDue` are dates (`YYYY-MM-DD`); `vaccinations.administeredAt` is a timestamp. `notes` is unstructured and holds a different type on every record. + +**Examples.** +`status:available AND species:(cat OR dog)` +`weightKg:[1 TO 5] AND shelter.city:Athens` +`born:>=2020-01-01 AND -tags:indoor` +`count(tags):>=3 OR vaccinations.vaccine:rabies` diff --git a/experiments/intent-to-filter/arms/execute.mjs b/experiments/intent-to-filter/arms/execute.mjs new file mode 100644 index 0000000..8d959c2 --- /dev/null +++ b/experiments/intent-to-filter/arms/execute.mjs @@ -0,0 +1,63 @@ +/** + * execute.mjs — one evaluator, shared by every arm. + * + * The single most important fairness property in this experiment: no arm gets + * its own semantics. Each arm translates whatever the model produced into a + * filter, and every filter is then compiled and run by the same compiler from + * ../../filter-to-sql, against the same ten records. An arm therefore cannot + * win or lose because its executor happened to treat a null differently. + * + * What each arm owns is the translation and the rejection: what its syntax can + * express, and what it refuses. That is the thing under test. + */ + +import { compile, QueryProblem } from "../../filter-to-sql/compile.mjs"; +import { HYBRID_BINDING } from "../../filter-to-sql/dataset.mjs"; + +export { HYBRID_BINDING }; + +/** A SPEC §8-shaped rejection, in the flat form the runner feeds back to the model. */ +export function problem(type, detail, pointer = "/filter") { + return { type, detail, pointer }; +} + +/** + * Compile and run a filter. Returns the matching ids, or the problem the + * compiler raised — the arm-neutral half of every arm's `execute`. + */ +export function runFilter(db, filter, binding = HYBRID_BINDING) { + let compiled; + try { + compiled = compile(filter, binding); + } catch (error) { + if (error instanceof QueryProblem) { + const p = error.toProblem(); + return { problem: problem(p.type.split("/").pop(), p.detail, p.pointer) }; + } + throw error; + } + // A filter can pass validation and still compile to SQL the database will + // not accept. That is a defect in the compiler, not in the model's answer, + // so it is recorded as its own condition rather than crashing the grader or + // being scored against the arm. + try { + const ids = db.prepare(compiled.sql).all(...compiled.params).map((r) => r.id); + return { ids }; + } catch (error) { + return { problem: problem("compiler-defect", `${error.message} — SQL: ${compiled.sql.slice(0, 200)}`, "/filter"), compilerDefect: true }; + } +} + +/** The queryable paths, single source of truth for the arms that enumerate them. */ +export const PATHS = [ + "id", "name", "species", "status", "born", "weightKg", "neutered", + "microchip", "tags", "notes", "priceCents", "costCents", "$$rate", "size\\.raw", + "shelter.name", "shelter.city", "shelter.capacity", "vaccinations", +]; + +/** Closed domains, for the arms whose documentation states them. */ +export const DOMAINS = { + status: ["available", "pending", "sold"], + species: ["cat", "dog", "rabbit", "bird"], + "vaccinations[].vaccine": ["rabies", "distemper", "parvo"], +}; diff --git a/experiments/intent-to-filter/arms/fixed-params.mjs b/experiments/intent-to-filter/arms/fixed-params.mjs new file mode 100644 index 0000000..2f947d2 --- /dev/null +++ b/experiments/intent-to-filter/arms/fixed-params.mjs @@ -0,0 +1,122 @@ +/** + * fixed-params.mjs — the arm that measures the ceiling rather than the accuracy. + * + * README.md:61: "Give it `list_pets(status, species)` and it can ask three + * questions". This arm is that tool, built as well as the shape allows: ten + * typed parameters, enums where the domain is closed, ranges split into a pair + * of bounds. It is the interface most search tools actually ship. + * + * The interesting number here is not how often the model gets it right. It is + * how often the question cannot be asked at all — and, on the questions that + * *can* be asked, whether typed enum parameters catch the closed-domain + * mistake that the filter grammar needs a generated schema to catch. + * + * Its CEILING is by far the largest of the five, and that is the finding. + */ + +import { runFilter, problem, DOMAINS } from "./execute.mjs"; + +const PARAMS = { + status: { type: "string", enum: DOMAINS.status, description: "Listing state." }, + species: { type: "string", enum: DOMAINS.species, description: "Taxon." }, + bornFrom: { type: "string", format: "date", description: "Earliest date of birth, inclusive." }, + bornTo: { type: "string", format: "date", description: "Latest date of birth, inclusive." }, + minWeightKg: { type: "number", minimum: 0, description: "Lowest weight, inclusive." }, + maxWeightKg: { type: "number", minimum: 0, description: "Highest weight, inclusive." }, + tag: { type: "string", description: "A single label the pet must carry." }, + shelterCity: { type: "string", description: "City the shelter is in." }, + neutered: { type: "boolean", description: "Whether the pet has been neutered." }, + nameContains: { type: "string", description: "Case-insensitive substring of the pet's name." }, +}; + +/** Parameter -> the clause it contributes. Order is irrelevant: all AND together. */ +const CLAUSE = { + status: (v) => ({ status: { $eq: v } }), + species: (v) => ({ species: { $eq: v } }), + bornFrom: (v) => ({ born: { $gte: v } }), + bornTo: (v) => ({ born: { $lte: v } }), + minWeightKg: (v) => ({ weightKg: { $gte: v } }), + maxWeightKg: (v) => ({ weightKg: { $lte: v } }), + tag: (v) => ({ tags: { $some: { $eq: v } } }), + shelterCity: (v) => ({ "shelter.city": { $eq: v } }), + neutered: (v) => ({ neutered: { $eq: v } }), + nameContains: (v) => ({ name: { $ilike: `%${v}%` } }), +}; + +/** + * The nine questions this shape can answer, out of sixty-three. Everything + * absent from GOLD is beyond the ceiling — there is no point enumerating the + * fifty-four reasons individually, since they are all the same reason. + */ +const GOLD = { + a01: { status: "available" }, + a02: { status: "available", species: "cat" }, + a04: { minWeightKg: 1, maxWeightKg: 5 }, + a05: { bornFrom: "2020-01-01" }, + a07: { nameContains: "o" }, + a13: { shelterCity: "Athens" }, + a18: { neutered: true }, + m01: { tag: "indoor" }, + m03: { status: "available" }, +}; + +function validateInput(input) { + const given = Object.entries(input ?? {}).filter(([k]) => k !== "limit"); + if (given.length === 0) { + return problem("malformed-query", "at least one search parameter is required.", "/"); + } + for (const [key, value] of given) { + const spec = PARAMS[key]; + if (!spec) { + return problem("unknown-field", `"${key}" is not a parameter of this tool. The parameters are: ${Object.keys(PARAMS).join(", ")}.`, `/${key}`); + } + if (spec.enum && !spec.enum.includes(value)) { + return problem("invalid-operand", `${key} must be one of: ${spec.enum.map((v) => JSON.stringify(v)).join(", ")}.`, `/${key}`); + } + const actual = typeof value; + const wanted = spec.type === "number" ? "number" : spec.type === "boolean" ? "boolean" : "string"; + if (actual !== wanted) { + return problem("invalid-operand", `${key} must be a ${wanted}.`, `/${key}`); + } + } + return null; +} + +export const fixedParams = { + name: "fixed-params", + kind: "params", + /** Everything not in GOLD; computed by index.mjs against the question set. */ + ceiling: {}, + ceilingReason: "the question needs a predicate this parameter list cannot form", + tool: { + name: "search_pets", + description: + "Search the pet collection. Every parameter you supply narrows the result further; omit the ones you do not need. Returns the matching records.", + input_schema: { + type: "object", + properties: { + ...PARAMS, + limit: { type: "integer", minimum: 1, maximum: 50, default: 20, description: "Maximum number of records to return." }, + }, + additionalProperties: false, + }, + }, + extract: (input) => input, + validate: validateInput, + /** The parameters as a filter — the form the differential grader compares. */ + translate(input) { + const rejected = validateInput(input); + if (rejected) throw Object.assign(new Error(rejected.detail), { problem: rejected }); + const clauses = Object.entries(input) + .filter(([k]) => k !== "limit" && CLAUSE[k]) + .map(([k, v]) => CLAUSE[k](v)); + return clauses.length === 1 ? clauses[0] : { $and: clauses }; + }, + execute(input, db) { + const rejected = validateInput(input); + if (rejected) return { problem: rejected }; + return runFilter(db, this.translate(input)); + }, + gold: (testCase) => GOLD[testCase.id] ?? null, + expressible: (id) => Object.hasOwn(GOLD, id), +}; diff --git a/experiments/intent-to-filter/arms/index.mjs b/experiments/intent-to-filter/arms/index.mjs new file mode 100644 index 0000000..6c5ca82 --- /dev/null +++ b/experiments/intent-to-filter/arms/index.mjs @@ -0,0 +1,54 @@ +/** + * index.mjs — the five arms, wired up and budget-checked. + * + * On the documentation budget: parity is enforced on the *prose* channel only — + * the tool description — because that is the channel the competing arms have + * and the schema arms are claiming not to need. The schema itself is the + * treatment, not part of the budget, which is why jql-generated ends up with + * the smallest description and by far the largest tool definition. Reporting + * both numbers side by side is the honest way to present that, and report.mjs + * does. + */ + +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +import { jqlGenerated, jqlPublished, jqlRaw } from "./jql.mjs"; +import { searchString } from "./search-string.mjs"; +import { bespokeJson } from "./bespoke-json.mjs"; +import { fixedParams } from "./fixed-params.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const doc = (name) => readFileSync(join(here, "docs", `${name}.md`), "utf8").trim(); + +searchString.tool.description = doc("search-string"); +bespokeJson.tool.description = doc("bespoke-json"); + +export const ARMS = [jqlRaw, jqlGenerated, jqlPublished, searchString, bespokeJson, fixedParams]; +export const BY_NAME = Object.fromEntries(ARMS.map((a) => [a.name, a])); + +/** + * The prose budget, and who it binds. + * + * It binds the two arms whose language is carried entirely by prose — + * search-string and bespoke-json — because those are the ones an unfair + * experiment would starve. It does not bind fixed-params, whose parameters + * carry their own descriptions and which needs no syntax section, nor the two + * schema arms, whose whole claim is that they need no prose at all. + * + * The figure was set *after* both documents were drafted, at the length the + * longer of the two needed, and the shorter was left as written rather than + * padded. Fixing a number first and then cutting to hit it would have meant + * deciding how good a competitor's documentation is allowed to be, which is + * the one decision this experiment must not make. + */ +export const PROSE_BUDGET = { target: 700, tolerance: 0.1, binds: ["search-string", "bespoke-json"] }; + +/** Rough token count, for the dry run and for tests that must not need a key. */ +export const approxTokens = (text) => Math.ceil(text.length / 4); + +/** The whole tool definition, which is what actually reaches the model. */ +export function toolTokens(arm) { + return approxTokens(JSON.stringify(arm.tool)); +} diff --git a/experiments/intent-to-filter/arms/jql.mjs b/experiments/intent-to-filter/arms/jql.mjs new file mode 100644 index 0000000..6f0a35a --- /dev/null +++ b/experiments/intent-to-filter/arms/jql.mjs @@ -0,0 +1,192 @@ +/** + * jql.mjs — the two arms that hand the model a JSON Schema and nothing else. + * + * jql-generated pet.filter.json, generated from pet.resource.json. Per-field + * operators and per-field operand domains. What README + * §"Exposing search to an agent" actually recommends. + * jql-published query-language-schema.json with $defs/FieldPath narrowed to + * the queryable paths. One Constraint shared by every field, + * so no domains. The cheap setup, and the control that says + * how much of any win belongs to the generator rather than to + * the language. + * + * Neither arm gets a prose tool description beyond one sentence naming the + * resource. That is the claim being tested — that the schema is sufficient — + * and writing a paragraph of guidance here would test something else. + */ + +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +import _Ajv2020 from "ajv/dist/2020.js"; +import _addFormats from "ajv-formats"; + +import { runFilter, problem, PATHS } from "./execute.mjs"; + +const Ajv2020 = _Ajv2020.default ?? _Ajv2020; +const addFormats = _addFormats.default ?? _addFormats; +const here = dirname(fileURLToPath(import.meta.url)); + +const GENERATED = JSON.parse(readFileSync(join(here, "..", "pet.filter.json"), "utf8")); + +/** + * The published grammar with one definition replaced — the override point + * README §"Restricting the queryable field set" documents, applied here rather + * than described. + */ +function narrowedGrammar() { + const grammar = JSON.parse( + readFileSync(join(here, "..", "..", "..", "query-language-schema.json"), "utf8"), + ); + grammar.$id = "https://example.invalid/intent-to-filter/pet.published.json"; + grammar.$defs.FieldPath = { + type: "string", + description: "A queryable path on a pet record.", + // The narrowing applies at every nesting level, quantifiers included, so + // the paths that are relative to a vaccination *element* have to appear in + // the same list as the paths relative to the record. One namespace for two + // scopes is a real wart of the single-override-point design, and it is the + // reason this arm's field list reads oddly. + enum: [ + ...PATHS, + "vaccinations[0].vaccine", "tags[0]", + "vaccine", "administeredAt", "boosterDue", + ], + }; + return grammar; +} + +const PUBLISHED = narrowedGrammar(); + +function compileValidator(schema) { + const ajv = new Ajv2020({ strict: false, allErrors: true }); + addFormats(ajv); + ajv.addVocabulary(["x-profiles"]); + return ajv.compile(schema); +} + +const validators = { + // jql-raw and jql-generated differ only in how the schema is delivered to the + // model; the server validates identically, which is the whole point of the pair. + "jql-raw": compileValidator(GENERATED), + "jql-generated": compileValidator(GENERATED), + "jql-published": compileValidator(PUBLISHED), +}; + +/** JSON Pointer escaping, for the token that names the offending member. */ +const esc = (token) => token.replace(/~/g, "~0").replace(/\//g, "~1"); + +/** + * ajv says where a filter stopped matching; SPEC.md §8 asks which of five + * things went wrong. The mapping is the same one examples/mcp-server uses, + * because that is the server this arm is standing in for. + */ +function problemFromAjv(errors) { + // Two ways a member can be refused, depending on which arm rejected it: a + // generated schema closes `additionalProperties`, a narrowed grammar closes + // `propertyNames`. Same condition, different keyword. + const refused = + errors.find((e) => e.keyword === "additionalProperties") ?? + errors.find((e) => e.keyword === "propertyNames"); + if (refused) { + const name = refused.params.additionalProperty ?? refused.params.propertyName; + const pointer = `/filter${refused.instancePath}/${esc(name)}`; + return name.startsWith("$") + ? problem("unsupported-operator", `${name} is not available here. The operators this field accepts are listed under it in the schema.`, pointer) + : problem("unknown-field", `"${name}" is not a queryable path on this resource.`, pointer); + } + const deepest = errors.reduce((a, b) => (b.instancePath.length > a.instancePath.length ? b : a)); + const at = deepest.instancePath; + const local = errors.filter((e) => e.instancePath === at); + const allowed = [ + ...new Set(local.flatMap((e) => + e.keyword === "const" ? [e.params.allowedValue] + : e.keyword === "enum" ? e.params.allowedValues + : [])), + ]; + const detail = allowed.length + ? `${at || "the filter"} must be one of: ${allowed.map((v) => JSON.stringify(v)).join(", ")}.` + : `${at || "the filter"} ${(local.find((e) => e.keyword !== "anyOf") ?? deepest).message}.`; + return problem("invalid-operand", detail, `/filter${at}`); +} + +/** + * Cases the generated schema provably cannot express. Each one is a property + * of the generator, not of the language — jql-published expresses all four — + * and each is a finding the README does not currently mention: + * + * a19 $field cross-field comparison. The generator narrows $gt's operand to + * the field's own type, leaving no room for a reference operand, so the + * refs profile is unreachable from a generated schema. + * a20 $regex against status. By design: pattern matching is withheld from + * enumerated domains. Correct, and it still costs the case. + * b04 vaccinations[0].vaccine — the generator emits no indexed paths. + * b08 tags[0] — as b04. + */ +const GENERATED_CEILING = { + a19: "the generator narrows $gt to the field's own type, so a $field reference operand cannot validate", + a20: "pattern matching is deliberately withheld from enumerated domains", + b04: "the generator emits no indexed paths", + b08: "the generator emits no indexed paths", +}; + +function makeArm(id, schema, blurb, ceiling) { + return { + name: id, + kind: "schema", + ceiling, + tool: { + name: "search_pets", + description: blurb, + input_schema: { + type: "object", + properties: { + filter: schema, + limit: { type: "integer", minimum: 1, maximum: 50, default: 20, description: "Maximum number of records to return." }, + }, + required: ["filter"], + additionalProperties: false, + }, + }, + /** The query as the model expressed it, for logging and mistake analysis. */ + extract: (input) => input?.filter, + validate(input) { + if (input?.filter === undefined) return problem("malformed-query", "the filter argument is required.", "/filter"); + const validate = validators[id]; + return validate(input.filter) ? null : problemFromAjv(validate.errors); + }, + execute(input, db) { + const rejected = this.validate(input); + if (rejected) return { problem: rejected }; + return runFilter(db, input.filter); + }, + /** The gold query for a case is the gold filter itself — no translation. */ + gold: (testCase) => (ceiling[testCase.id] ? null : testCase.goldFilter ?? testCase.filter ?? null), + }; +} + +const RAW_BLURB = "Search the pet collection. The `filter` argument is a predicate over a pet record; its schema describes the operators and the queryable fields, including which values each field accepts. Returns the matching records."; + +export const jqlGenerated = makeArm( + "jql-generated", + GENERATED, + "Search the pet collection. The `filter` argument is a predicate over a pet record; its schema describes the operators and the queryable fields, including which values each field accepts. Returns the matching records.", + GENERATED_CEILING, +); + +/** + * The generated schema shipped exactly as README §"Exposing search to an agent" + * says to ship it: inlined, `$ref`s and all. It is kept as its own arm because + * the pilot found that this does not deliver the grammar to any model tested — + * Gemini rejects it, OpenAI silently drops the 94% of the file that lives in + * `$defs` — and the size of that gap is worth a row rather than a footnote. + */ +export const jqlRaw = { ...makeArm("jql-raw", GENERATED, RAW_BLURB, GENERATED_CEILING), rawRefs: true }; + +export const jqlPublished = makeArm( + "jql-published", + PUBLISHED, + "Search the pet collection. The `filter` argument is a predicate over a pet record; its schema describes the operators and the queryable fields. Returns the matching records.", + {}, +); diff --git a/experiments/intent-to-filter/arms/search-string.mjs b/experiments/intent-to-filter/arms/search-string.mjs new file mode 100644 index 0000000..209ef78 --- /dev/null +++ b/experiments/intent-to-filter/arms/search-string.mjs @@ -0,0 +1,409 @@ +/** + * search-string.mjs — the arm README.md:63 names as the foil. + * + * "A bespoke query string (?q=status:open AND born>2020) has to be taught: + * the tool description explains the syntax, the prompt carries examples, + * and the model assembles a string nothing can validate until it reaches + * your parser." + * + * So this arm is that string, and the effort here goes into making it *good*. + * The syntax is not a strawman: every construct below is lifted from Lucene, + * Elasticsearch query-string, or GitHub search, because those are what a real + * team reaches for. It has wildcards, ranges, regex with a case-insensitivity + * flag, `_exists_`, quoted field names, boolean grouping, and — the one place + * it is genuinely more ergonomic than the filter grammar — implicit any-element + * semantics on multi-valued fields, so `tags:indoor` means what a user expects + * without naming a quantifier. + * + * THE PARSER IS DELIBERATELY GENEROUS. It accepts every reasonable variant + * spelling it can: `>=` and `≥`, `TO` in either case, unquoted values with + * internal spaces where unambiguous, `NOT x` and `-x`. A stingy parser would + * manufacture rejections and hand the comparison to the schema arms; that + * would be the easiest way to fake this entire experiment, so the bias here + * runs the other way on purpose. + * + * What it cannot express is recorded in CEILING, not hidden as a parse error. + */ + +import { runFilter, problem } from "./execute.mjs"; + +/** Multi-valued paths: a bare match against these means "some element". */ +const ARRAY_PATHS = new Set(["tags", "vaccinations"]); + +/** Every field the syntax accepts, and how it is spelled as a filter path. */ +const FIELDS = { + id: "id", name: "name", species: "species", status: "status", born: "born", + weightKg: "weightKg", neutered: "neutered", microchip: "microchip", + tags: "tags", notes: "notes", priceCents: "priceCents", costCents: "costCents", + "$rate": "$$rate", "size.raw": "size\\.raw", + "shelter.name": "shelter.name", "shelter.city": "shelter.city", + "shelter.capacity": "shelter.capacity", + vaccinations: "vaccinations", + "vaccinations.vaccine": ["vaccinations", "vaccine"], + "vaccinations.administeredAt": ["vaccinations", "administeredAt"], + "vaccinations.boosterDue": ["vaccinations", "boosterDue"], +}; + +class ParseError extends Error { + constructor(detail, type = "malformed-query") { + super(detail); + this.detail = detail; + this.type = type; + } +} + +// --------------------------------------------------------------------------- +// Tokenizer +// --------------------------------------------------------------------------- + +const PUNCT = new Set(["(", ")"]); + +function tokenize(source) { + const tokens = []; + let i = 0; + while (i < source.length) { + const ch = source[i]; + if (/\s/.test(ch)) { i += 1; continue; } + if (PUNCT.has(ch)) { tokens.push({ kind: ch }); i += 1; continue; } + if (ch === "-" && (tokens.length === 0 || tokens.at(-1).kind !== "term")) { + tokens.push({ kind: "NOT" }); i += 1; continue; + } + // A term runs to whitespace or an unnested paren, but quoted runs, + // bracketed ranges and /regex/ bodies swallow whatever is inside them. + let start = i; + let depth = 0; + let text = ""; + while (i < source.length) { + const c = source[i]; + if (c === '"' || c === "'") { + const close = source.indexOf(c, i + 1); + if (close === -1) throw new ParseError(`unterminated quote at position ${i}`); + text += source.slice(i, close + 1); + i = close + 1; + continue; + } + if (c === "/" && text.includes(":")) { + const close = source.indexOf("/", i + 1); + if (close === -1) throw new ParseError(`unterminated regular expression at position ${i}`); + let end = close + 1; + while (end < source.length && /[a-z]/.test(source[end])) end += 1; + text += source.slice(i, end); + i = end; + continue; + } + if (c === "[") { depth += 1; text += c; i += 1; continue; } + if (c === "]") { depth -= 1; text += c; i += 1; continue; } + if (c === "(" && (text.includes(":") || /^(count|first)$/.test(text))) { + depth += 1; text += c; i += 1; continue; + } + if (c === ")" && depth > 0) { depth -= 1; text += c; i += 1; continue; } + if (depth === 0 && (/\s/.test(c) || PUNCT.has(c))) break; + text += c; + i += 1; + } + if (!text) throw new ParseError(`cannot read the query at position ${start}`); + const upper = text.toUpperCase(); + if (upper === "AND" || upper === "OR") tokens.push({ kind: upper }); + else if (upper === "NOT") tokens.push({ kind: "NOT" }); + else tokens.push({ kind: "term", text }); + } + return tokens; +} + +// --------------------------------------------------------------------------- +// Parser — precedence: NOT binds tightest, then AND, then OR +// --------------------------------------------------------------------------- + +function parse(tokens) { + let pos = 0; + const peek = () => tokens[pos]; + const eat = (kind) => (peek()?.kind === kind ? (pos += 1, true) : false); + + function primary() { + if (eat("NOT")) return { $not: primary() }; + if (eat("(")) { + const inner = orExpr(); + if (!eat(")")) throw new ParseError("unbalanced parentheses"); + return inner; + } + const token = peek(); + if (!token || token.kind !== "term") throw new ParseError("expected a search clause"); + pos += 1; + return clause(token.text); + } + + function andExpr() { + const parts = [primary()]; + for (;;) { + if (eat("AND")) { parts.push(primary()); continue; } + // Adjacency is AND, as in every search box that has ever shipped. + const next = peek(); + if (next && (next.kind === "term" || next.kind === "(" || next.kind === "NOT")) { + parts.push(primary()); + continue; + } + break; + } + return parts.length === 1 ? parts[0] : { $and: parts }; + } + + function orExpr() { + const parts = [andExpr()]; + while (eat("OR")) parts.push(andExpr()); + return parts.length === 1 ? parts[0] : { $or: parts }; + } + + const filter = orExpr(); + if (pos !== tokens.length) throw new ParseError("trailing input after the query"); + return filter; +} + +// --------------------------------------------------------------------------- +// One clause -> one filter +// --------------------------------------------------------------------------- + +const unquote = (raw) => + (raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'")) + ? raw.slice(1, -1) + : raw; + +/** Untyped syntax, so scalars are inferred — the coercion a string arm cannot avoid. */ +function scalar(raw) { + const text = unquote(raw); + if (raw !== text) return text; // it was quoted: it is a string, whatever it looks like + if (text === "true") return true; + if (text === "false") return false; + if (/^-?\d+$/.test(text)) return Number.parseInt(text, 10); + if (/^-?\d*\.\d+$/.test(text)) return Number.parseFloat(text); + return text; +} + +const LIKE_ESCAPE = (text) => text.replace(/([%_\\])/g, "\\$1"); + +function resolveField(name) { + const key = unquote(name); + const path = FIELDS[key]; + if (path === undefined) { + throw new ParseError( + `"${key}" is not a searchable field. The searchable fields are: ${Object.keys(FIELDS).join(", ")}.`, + "unknown-field", + ); + } + return path; +} + +/** Wrap a constraint so it applies to elements where the path is multi-valued. */ +function place(path, constraint) { + if (Array.isArray(path)) { + const [array, member] = path; + return { [array]: { $some: { [member]: constraint } } }; + } + if (ARRAY_PATHS.has(path)) return { [path]: { $some: constraint } }; + return { [path]: constraint }; +} + +function clause(text) { + // A bare term with no field is free text. + const split = text.indexOf(":"); + if (split === -1) return { name: { $search: unquote(text) } }; + + let field = text.slice(0, split); + const rest = text.slice(split + 1); + + // count(tags):>=3 and first(tags):indoor + const wrapped = /^(count|first)\((.+)\)$/.exec(field); + if (wrapped) { + const [, fn, inner] = wrapped; + const path = resolveField(inner); + if (fn === "count") { + if (Array.isArray(path) || !ARRAY_PATHS.has(path)) { + throw new ParseError(`count() applies to a multi-valued field; ${unquote(inner)} is not one.`, "invalid-operand"); + } + const cmp = comparison(rest); + return { [path]: { $size: cmp === null ? scalar(rest) : cmp } }; + } + const indexed = Array.isArray(path) ? `${path[0]}[0].${path[1]}` : `${path}[0]`; + return { [indexed]: { $eq: scalar(rest) } }; + } + + if (field === "_exists_" || field === "_missing_") { + const present = field === "_exists_"; + return place(resolveField(rest), { $exists: present }); + } + + const path = resolveField(field); + + // field:null — present, but blank + if (rest === "null") return place(path, { $isNull: true }); + + // field:/pattern/flags + const re = /^\/(.+)\/([a-z]*)$/.exec(rest); + if (re) { + const [, pattern, flags] = re; + const constraint = { $regex: pattern }; + if (flags) constraint.$flags = flags; + return place(path, constraint); + } + + // field:[lo TO hi] + const range = /^\[(.+?)\s+TO\s+(.+?)\]$/i.exec(rest); + if (range) return place(path, { $between: [scalar(range[1]), scalar(range[2])] }); + + // field:(a OR b OR c) + const group = /^\((.+)\)$/.exec(rest); + if (group) { + const members = group[1].split(/\s+OR\s+/i).map((m) => scalar(m.trim())); + return place(path, { $in: members }); + } + + // field:>x field:>=x field:x` `>=x` `=|<=|≥|≤|>|<)\s*(.+)$/.exec(rest); + if (!m) return null; + const op = { ">": "$gt", ">=": "$gte", "≥": "$gte", "<": "$lt", "<=": "$lte", "≤": "$lte" }[m[1]]; + return { [op]: scalar(m[2]) }; +} + +// --------------------------------------------------------------------------- +// The arm +// --------------------------------------------------------------------------- + +/** + * What this syntax cannot say. Every entry is a real limitation of the + * string-query family, not of this parser — which is the point of recording + * them rather than letting them surface as parse failures. + */ +const CEILING = { + a19: "no cross-field comparison: a value position cannot name another field", + b01: "no correlated element match — the two conditions cannot be pinned to the same vaccination", + b07: "`some element is not indoor` is not expressible; `-tags:indoor` means `no element is indoor`", + b11: "no universal quantifier over elements", + b12: "no universal quantifier over elements", + b13: "no universal quantifier over elements", + d01: "no type predicate", + d02: "no type predicate", + d03: "no type predicate", + d04: "no type predicate", +}; + +/** The gold query per case: what a fluent user of this syntax would write. */ +const GOLD = { + a01: "status:available", + a02: "status:available AND species:cat", + a03: "status:(available OR pending)", + a04: "weightKg:[1 TO 5]", + a05: "born:>=2020-01-01", + a06: "name:b*", + a07: "name:/o/i", + a08: "name:*50%*", + a09: "tags:(indoor OR small)", + a10: "tags:trained AND tags:outdoor", + a11: "count(tags):>=3", + a12: "count(tags):0", + a13: "shelter.city:Athens", + a14: "shelter.capacity:<=12", + a15: "(species:cat AND status:available) OR (species:dog AND weightKg:<20)", + a16: "status:available AND (shelter.city:Patras OR tags:trained)", + a17: "-status:sold AND -status:pending", + a18: "neutered:true", + a20: "status:a*", + b02: "vaccinations.vaccine:rabies AND vaccinations.boosterDue:<2025-01-01", + b03: "vaccinations.administeredAt:>=2025-01-01T00:00:00Z", + b04: "first(vaccinations.vaccine):parvo", + b05: "vaccinations.vaccine:parvo", + b06: "-tags:indoor", + b08: "first(tags):indoor", + b09: "_exists_:vaccinations.boosterDue", + b10: "_missing_:vaccinations.boosterDue", + c01: "microchip:null", + c02: "-_exists_:microchip", + c03: "_exists_:notes", + c04: "-_exists_:shelter.capacity", + c05: "-microchip:CHIP-001", + c07: "-microchip:CHIP-001 OR microchip:null", + c08: "-status:sold", + c11: "-weightKg:[1 TO 5]", + c12: "-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip", + c13: "-born:>=2020-01-01 OR born:null", + d05: "notes:3", + d06: "notes:>2", + d07: "notes:>a", + d08: "-notes:3", + d09: "-notes:>2", + e01: "$rate:>5", + e02: '"size.raw":XS', + e03: "name:C*o", + e04: "name:/^[A-C]/", + e05: "name:/^[a-c]/i", + e06: "name:*s", + f01: "internalNotes:anything", + f02: "ada", + m01: "tags:indoor", + m02: "-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip", + m03: "status:available", +}; + +export const searchString = { + name: "search-string", + kind: "string", + ceiling: CEILING, + tool: { + name: "search_pets", + description: null, // filled from arms/docs/search-string.md by index.mjs + input_schema: { + type: "object", + properties: { + q: { type: "string", description: "The search query. See the tool description for the syntax." }, + limit: { type: "integer", minimum: 1, maximum: 50, default: 20, description: "Maximum number of records to return." }, + }, + required: ["q"], + additionalProperties: false, + }, + }, + extract: (input) => input?.q, + /** Translate, or explain why not. Translation failure IS this arm's validation. */ + translate(input) { + if (typeof input?.q !== "string" || !input.q.trim()) { + throw new ParseError("the q argument is required and must be a non-empty query string."); + } + return parse(tokenize(input.q)); + }, + validate(input) { + try { + this.translate(input); + return null; + } catch (error) { + if (error instanceof ParseError) return problem(error.type, error.detail, "/q"); + throw error; + } + }, + execute(input, db) { + let filter; + try { + filter = this.translate(input); + } catch (error) { + if (error instanceof ParseError) return { problem: problem(error.type, error.detail, "/q") }; + throw error; + } + return runFilter(db, filter); + }, + gold: (testCase) => (CEILING[testCase.id] ? null : GOLD[testCase.id] ?? null), +}; + +export const __test = { tokenize, parse, clause }; diff --git a/experiments/intent-to-filter/corpus.mjs b/experiments/intent-to-filter/corpus.mjs new file mode 100644 index 0000000..27e254f --- /dev/null +++ b/experiments/intent-to-filter/corpus.mjs @@ -0,0 +1,44 @@ +/** + * corpus.mjs — the graded set, assembled in one place. + * + * Sixty questions inherited from ../filter-to-sql/cases.mjs, whose answer key + * was hand-derived from SPEC.md months before this experiment existed, plus + * three written here to tempt the three mistakes README catalogues. The two + * groups are tagged `inherited` and `authored` and never averaged together: + * the inherited ones cannot have been chosen to flatter the language, and the + * authored ones plainly were chosen with a hypothesis in mind. + */ + +import { CASES } from "../filter-to-sql/cases.mjs"; +import { QUESTIONS, MISTAKE_PROBES, EXCLUDED } from "./questions.mjs"; + +/** Which corpus group a case's expected answer came from. */ +const inherited = CASES.filter((c) => QUESTIONS[c.id]).map((c) => ({ + id: c.id, + origin: "inherited", + group: c.group, + probe: QUESTIONS[c.id].probe, + ask: QUESTIONS[c.id].ask, + expect: c.expect, + goldFilter: c.filter, + title: c.title, +})); + +const authored = Object.entries(MISTAKE_PROBES).map(([id, p]) => ({ + id, + origin: "authored", + group: "M. Mistake probes", + probe: p.probe, + ask: p.ask, + expect: p.expect, + goldFilter: p.gold, + tempts: p.tempts, + title: `tempts ${p.tempts}`, +})); + +export const CORPUS = [...inherited, ...authored]; +export const BY_ID = Object.fromEntries(CORPUS.map((c) => [c.id, c])); +export { EXCLUDED }; + +/** A. and B. are what a search UI sends; the rest probe the semantic edges. */ +export const isOrdinaryTraffic = (c) => c.group.startsWith("A.") || c.group.startsWith("B."); diff --git a/experiments/intent-to-filter/deref.mjs b/experiments/intent-to-filter/deref.mjs new file mode 100644 index 0000000..06f4193 --- /dev/null +++ b/experiments/intent-to-filter/deref.mjs @@ -0,0 +1,154 @@ +/** + * deref.mjs — inline every `$ref`, because two of three providers will not. + * + * THE PROBLEM THIS EXISTS FOR, which the pilot found and which is the largest + * finding in this directory. + * + * README §"Exposing search to an agent" says to inline the filter schema into + * the tool definition, and `examples/mcp-server` does exactly that. Neither + * mentions that the schema is built almost entirely out of `$ref`. For the + * generated pet schema, the `$defs` block is 57,756 of 61,354 characters — + * 94% of the file. Every operator description and every per-field value domain + * lives in there and is reached by reference. + * + * Handed to a provider as a function's parameter schema: + * + * Gemini rejects the request outright. Its function-declaration schema has + * no `$ref` field at all: HTTP 400. + * OpenAI accepts the request and silently discards `$defs`. Measured: the + * full tool bills 634 prompt tokens, which is the 3,598 characters + * that are *not* `$defs`. The model receives a list of field names + * and a set of dangling references. + * + * So the recommended setup does not deliver the grammar to the model on either. + * The schema has to be flattened first, which is what this file does, and the + * flattening is not free: it multiplies size, and the recursion has to be cut + * at a fixed depth because a recursive schema has no finite tree form. + * + * `$and`, `$or`, `$nor` and `$not` take `{"$ref": "#"}`, so nested filters are + * unbounded by construction. Cutting at MAX_DEPTH means a filter nested deeper + * than that is describable but not *described* — the model is told the nesting + * stops sooner than the server will actually accept. + */ + +/** + * Depth 1. Measured on the generated pet schema: depth 1 is 65,775 characters, + * depth 2 is 245,904, depth 3 is 280,526. Depth 1 buys every field's operand + * domain — the part that prevents the valid-but-wrong failures — and spends + * nothing on re-describing the nesting, which the stub describes in a sentence. + */ +const MAX_DEPTH = 1; + +const resolvePointer = (root, ref) => { + if (ref === "#") return root; + return ref + .replace(/^#\//, "") + .split("/") + .map((token) => token.replace(/~1/g, "/").replace(/~0/g, "~")) + .reduce((node, token) => node?.[token], root); +}; + +/** Keywords no provider's function-parameter parser wants to see. */ +const DROP = ["$schema", "$id", "$comment", "$defs", "$anchor"]; + +/** + * Inline every reference, cutting recursion at `maxDepth`. + * + * At the cut, a recursive position becomes a bare `{"type": "object"}` rather + * than being deleted: deleting it would tell the model that nesting is + * impossible, which is a different and worse lie than telling it the nesting + * is shallower than it is. + */ +export function dereference(schema, { maxDepth = MAX_DEPTH } = {}) { + const root = schema; + + const walk = (node, depth, seen) => { + if (node === null || typeof node !== "object") return node; + if (Array.isArray(node)) return node.map((member) => walk(member, depth, seen)); + + if (typeof node.$ref === "string") { + const { $ref, ...siblings } = node; + if (depth >= maxDepth) { + return { type: "object", description: "A nested filter, same shape as the parent." }; + } + if (seen.has($ref)) { + return { type: "object", description: "A nested filter, same shape as the parent." }; + } + const target = resolvePointer(root, $ref); + if (target === undefined) return { type: "object" }; + const inlined = walk(target, depth + 1, new Set([...seen, $ref])); + return { ...inlined, ...walk(siblings, depth, seen) }; + } + + const out = {}; + for (const [key, value] of Object.entries(node)) { + if (DROP.includes(key)) continue; + out[key] = walk(value, depth, seen); + } + return out; + }; + + return walk(root, 0, new Set()); +} + +/** Gemini additionally rejects these, even with no reference left in the tree. */ +const GEMINI_DROP = ["title", "default", "additionalProperties", "examples", "minProperties", "uniqueItems", "const"]; + +/** + * Gemini's subset is narrower still: it has no `const`, so a `oneOf` of consts + * — which is how the generator expresses a documented enum with per-value + * prose — has to be rewritten as a plain `enum`, losing the per-value + * descriptions. Another cost the README does not mention. + */ +export function forGemini(schema) { + const flat = dereference(schema); + + const walk = (node) => { + if (node === null || typeof node !== "object") return node; + if (Array.isArray(node)) return node.map(walk); + + // oneOf/anyOf of consts -> enum + for (const combinator of ["oneOf", "anyOf"]) { + const members = node[combinator]; + if (Array.isArray(members) && members.length > 0 && members.every((m) => m && "const" in m)) { + const { [combinator]: _drop, ...rest } = node; + return walk({ ...rest, type: rest.type ?? "string", enum: members.map((m) => m.const) }); + } + } + + const out = {}; + for (const [key, value] of Object.entries(node)) { + if (GEMINI_DROP.includes(key)) continue; + out[key] = walk(value); + } + return out; + }; + + return walk(flat); +} + +/** + * Flatten an arm's whole tool definition. + * + * The embedded filter schema keeps its own `$id`, so its `#` and `#/$defs/...` + * fragments resolve against *it*, not against the tool schema that contains it + * — which is exactly the hazard README §"Referencing by URL or by copy" warns + * about, met here from the other side. Dereferencing the wrapper as one + * document silently resolves every fragment to nothing. + */ +export function flattenTool(tool, { dialect = "openai" } = {}) { + const flatten = dialect === "gemini" ? forGemini : dereference; + const schema = structuredClone(tool.input_schema); + if (schema.properties?.filter) { + schema.properties.filter = flatten(schema.properties.filter); + } + const walk = (node) => { + if (!node || typeof node !== "object") return; + if (Array.isArray(node)) return node.forEach(walk); + for (const key of DROP) delete node[key]; + if (dialect === "gemini") for (const key of GEMINI_DROP) delete node[key]; + Object.values(node).forEach(walk); + }; + walk(schema); + return { ...tool, input_schema: schema }; +} diff --git a/experiments/intent-to-filter/differential.mjs b/experiments/intent-to-filter/differential.mjs new file mode 100644 index 0000000..7872ffc --- /dev/null +++ b/experiments/intent-to-filter/differential.mjs @@ -0,0 +1,207 @@ +/** + * differential.mjs — a second, larger collection, used only to tell filters apart. + * + * THE PROBLEM IT SOLVES. Across the 62 corpus cases with a row-set answer there + * are only 44 distinct row sets, over ten records: a01, a17 and a20 all return + * the same six ids. So "returned the right rows" is a weak claim that a model + * wrote the right filter — a wrong filter can land on the right rows by luck, + * and with ten records the luck is not even long odds. + * + * THE FIX. Grade a candidate correct only if it (1) returns the gold row set on + * the ten authoritative records, and (2) returns the same rows as the *gold + * filter* on two hundred generated ones. The ten records keep their authority: + * they are hand-derived from SPEC.md and they decide whether the gold filter is + * right. These two hundred decide only whether the candidate agrees with it, so + * nothing here needs to be hand-checked and nothing here can make a gold answer + * wrong. + * + * It works: the 62 gold filters produce 44 distinct row sets over the ten + * records and 58 over these two hundred, leaving no pair indistinguishable. + * + * The collection is generated from a fixed seed and deliberately dense where + * filters are easy to confuse: present-nulls beside absent keys, empty arrays, + * values sitting exactly on the range boundaries the corpus uses, and a `notes` + * path that holds a different JSON type on every few records. It is not + * schema-conformant everywhere — tags carry the occasional null element — for + * the same reason: its job is to separate filters, not to model the resource. + */ + +import { DatabaseSync } from "node:sqlite"; +import { compile, QueryProblem } from "../filter-to-sql/compile.mjs"; +import { HYBRID_BINDING } from "../filter-to-sql/dataset.mjs"; + +/** mulberry32 — small, seeded, and stable across Node versions. */ +function rng(seed) { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const NAMES = [ + "Ada", "bruno", "Cleo", "Dash", "Echo", "Fig 50% off", "Gus", "Hera", "Iris", "Juno", + "Bo", "boris", "Cass", "delta", "Ochre", "Oso", "Pips", "Quill", "Rufus", "Sol", + "Ames", "Cosmo", "50% Off Milo", "Nox", "Opal", "Bess", "Coco", "Doss", "Enzo", "Flo", +]; +const TAG_POOL = ["indoor", "outdoor", "calm", "loud", "quiet", "small", "trained", "senior"]; +const CITIES = ["Athens", "Patras", "Volos", "Larissa"]; +const VACCINES = ["rabies", "distemper", "parvo"]; + +/** One value per record for the deliberately heterogeneous `notes` path. */ +const NOTES = ["quiet", "loud", 3, 3.0, 3.5, 2, 0, true, false, null, ["a", "b"], [], { k: 1 }, "a", "zed"]; + +export const SEED = 20260909; + +function generate(count = 200) { + const random = rng(SEED); + const pick = (list) => list[Math.floor(random() * list.length)]; + const chance = (p) => random() < p; + const records = []; + + for (let i = 0; i < count; i += 1) { + const record = { id: `d${String(i).padStart(3, "0")}`, name: pick(NAMES) }; + + // A rogue value outside the documented domain, on purpose: without one, + // `status is available`, `status is neither sold nor pending` and `status + // starts with a` are indistinguishable on any conformant data, and three + // corpus cases collapse into one. Real collections have values like this; + // this set is not shown to a model, only used to tell filters apart. + if (chance(0.94)) record.status = pick(["available", "pending", "sold", "available", "pending", "sold", "archived", "withdrawn"]); + if (chance(0.88)) record.species = pick(["cat", "dog", "rabbit", "bird"]); + + // Dates clustered on the 2020-01-01 boundary the corpus keeps testing. + if (chance(0.9)) { + const year = pick([2016, 2017, 2018, 2019, 2019, 2020, 2020, 2021, 2022, 2023, 2024]); + const month = 1 + Math.floor(random() * 12); + const day = 1 + Math.floor(random() * 28); + record.born = `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`; + } + + // Weights that land exactly on 1 and 5, the inclusive bounds of a04/c11. + if (chance(0.95)) { + record.weightKg = pick([0.4, 0.9, 1, 1, 1.2, 2, 3.1, 4.2, 5, 5, 5.5, 12, 18, 22.5, 31, 40]); + } + if (chance(0.9)) record.neutered = chance(0.5); + + // The three states §4.2 keeps apart, in roughly equal thirds. + const chipState = random(); + if (chipState < 0.5) record.microchip = pick(["CHIP-001", "CHIP-002", "CHIP-004", "CHIP-010"]); + else if (chipState < 0.8) record.microchip = null; + // else: absent + + if (chance(0.92)) { + const size = Math.floor(random() * 4); // 0..3, so empty lists are common + record.tags = Array.from({ length: size }, () => (chance(0.05) ? null : pick(TAG_POOL))); + } + + if (chance(0.85)) record.notes = NOTES[Math.floor(random() * NOTES.length)]; + + record.priceCents = pick([2500, 3000, 5000, 6000, 7000, 9000, 12000, 15000, 20000]); + record.costCents = pick([1000, 2000, 3500, 6500, 7000, 8000, 9500, 11000, 15000]); + + if (chance(0.25)) record.$rate = pick([3, 5, 5, 7, 9]); + if (chance(0.3)) record["size.raw"] = pick(["XS", "S", "M", "L"]); + + if (chance(0.96)) { + record.shelter = { name: `${pick(["North", "South", "East", "West"])} Shelter` }; + if (chance(0.9)) record.shelter.city = pick(CITIES); + if (chance(0.85)) record.shelter.capacity = pick([0, 5, 12, 12, 22, 40, 60]); + } + + if (chance(0.9)) { + const shots = Math.floor(random() * 3); // 0..2, so empty histories are common + record.vaccinations = Array.from({ length: shots }, () => { + const shot = { + vaccine: pick(VACCINES), + administeredAt: `${pick([2021, 2022, 2023, 2024, 2025])}-0${1 + Math.floor(random() * 9)}-15T10:00:00Z`, + }; + if (chance(0.6)) { + shot.boosterDue = `${pick([2022, 2023, 2024, 2024, 2025, 2026])}-0${1 + Math.floor(random() * 9)}-15`; + } + return shot; + }); + } + + records.push(record); + } + return records; +} + +export const RECORDS = generate(); + +const DDL = ` +CREATE TABLE pets ( + id TEXT PRIMARY KEY, + status TEXT, + species TEXT, + born TEXT, + weight_kg REAL, + price_cents INTEGER, + cost_cents INTEGER, + doc TEXT NOT NULL +);`; + +/** + * The same binding the graded run uses, pointed at this table. It has to be the + * same: a promoted column and a JSON path answer $exists differently (§4.2), so + * a differential test under a different binding would disagree with the real + * one for reasons that have nothing to do with the candidate filter. + */ +export const BINDING = { ...HYBRID_BINDING, table: "pets" }; + +export function openDifferential() { + const db = new DatabaseSync(":memory:"); + db.exec("PRAGMA case_sensitive_like = ON"); + db.function("jql_regex", { deterministic: true }, (value, pattern, flags) => { + if (value === null || value === undefined) return null; + try { + return new RegExp(pattern, flags || undefined).test(String(value)) ? 1 : 0; + } catch { + return null; + } + }); + db.exec(DDL); + const insert = db.prepare( + "INSERT INTO pets (id, status, species, born, weight_kg, price_cents, cost_cents, doc) VALUES (?,?,?,?,?,?,?,?)", + ); + for (const r of RECORDS) { + insert.run( + r.id, r.status ?? null, r.species ?? null, r.born ?? null, + r.weightKg ?? null, r.priceCents ?? null, r.costCents ?? null, JSON.stringify(r), + ); + } + return db; +} + +function idsFor(db, filter) { + try { + const compiled = compile(filter, BINDING); + return { ids: db.prepare(compiled.sql).all(...compiled.params).map((r) => r.id) }; + } catch (error) { + if (error instanceof QueryProblem) return { problem: error.type.split("/").pop() }; + throw error; + } +} + +/** + * Do two filters mean the same thing? Answered by running both, not by + * comparing their shapes — a great many spellings of the same predicate are + * correct, and grading on structure would fail them all. + */ +export function equivalent(db, goldFilter, candidateFilter) { + const gold = idsFor(db, goldFilter); + const candidate = idsFor(db, candidateFilter); + if (gold.problem || candidate.problem) return gold.problem === candidate.problem; + if (gold.ids.length !== candidate.ids.length) return false; + const set = new Set(gold.ids); + return candidate.ids.every((id) => set.has(id)); +} + +/** How many of the 200 a filter matches — reported so a vacuous filter is visible. */ +export function selectivity(db, filter) { + const result = idsFor(db, filter); + return result.problem ? null : result.ids.length; +} diff --git a/experiments/intent-to-filter/gold.test.mjs b/experiments/intent-to-filter/gold.test.mjs new file mode 100644 index 0000000..2da7d99 --- /dev/null +++ b/experiments/intent-to-filter/gold.test.mjs @@ -0,0 +1,203 @@ +/** + * gold.test.mjs — everything that must hold before a single request is paid for. + * + * The failure this guards against is the one that would quietly invalidate the + * whole experiment: an arm losing a case because its own executor could not run + * the right answer. If the hand-written gold query for a case does not return + * the gold row set through that arm, then a model producing exactly that query + * would be marked wrong, and the arm's score would measure the harness rather + * than the interface. + * + * So: every arm, every case, the right answer, executed. + * + * node --test experiments/intent-to-filter/gold.test.mjs + */ + +import test from "node:test"; +import assert from "node:assert/strict"; + +import { openDatabase } from "../filter-to-sql/harness.mjs"; +import { CASES } from "../filter-to-sql/cases.mjs"; +import { CORPUS, EXCLUDED } from "./corpus.mjs"; +import { QUESTIONS, BANNED_TOKENS, LEAK_EXEMPTIONS } from "./questions.mjs"; +import { ARMS, PROSE_BUDGET, approxTokens } from "./arms/index.mjs"; +import { openDifferential, equivalent, BINDING } from "./differential.mjs"; +import { compile } from "../filter-to-sql/compile.mjs"; +import { mistakes } from "./grade.mjs"; + +const db = openDatabase(); +const differentialDb = openDifferential(); + +const normalise = (expect) => + Array.isArray(expect) || expect === undefined ? { ids: expect ?? [] } : expect; + +const asOutcome = (result) => (result.problem ? { problem: result.problem.type } : { ids: result.ids }); + +// --------------------------------------------------------------------------- +// The load-bearing test +// --------------------------------------------------------------------------- + +for (const arm of ARMS) { + test(`${arm.name}: every gold query returns the gold answer, or is declared unreachable`, () => { + let answered = 0; + let unreachable = 0; + for (const testCase of CORPUS) { + const gold = arm.gold(testCase); + if (gold === null || gold === undefined) { + unreachable += 1; + continue; + } + const input = + arm.kind === "schema" ? { filter: gold } + : arm.kind === "string" ? { q: gold } + : arm.kind === "json" ? { where: gold } + : gold; + const expected = normalise(testCase.expect); + const want = expected.problem ? { problem: expected.problem } : { ids: expected.ids }; + assert.deepEqual( + asOutcome(arm.execute(input, db)), + want, + `${arm.name}/${testCase.id}: gold query does not produce the gold answer`, + ); + answered += 1; + } + assert.ok(answered > 0, `${arm.name} answered nothing`); + assert.equal(answered + unreachable, CORPUS.length); + }); +} + +test("every arm's declared ceiling is real — the case is unreachable, not merely unwritten", () => { + for (const arm of ARMS) { + for (const id of Object.keys(arm.ceiling ?? {})) { + assert.ok(BY_ID_HAS(id), `${arm.name} declares a ceiling for ${id}, which is not in the corpus`); + assert.equal(arm.gold({ id }), null, `${arm.name} declares ${id} unreachable but supplies a gold query`); + } + } +}); + +const BY_ID_HAS = (id) => CORPUS.some((c) => c.id === id); + +// --------------------------------------------------------------------------- +// The corpus itself +// --------------------------------------------------------------------------- + +test("every corpus case is either asked or excluded, with a reason", () => { + for (const testCase of CASES) { + const asked = Boolean(QUESTIONS[testCase.id]); + const excluded = Boolean(EXCLUDED[testCase.id]); + assert.ok(asked !== excluded, `${testCase.id} must be exactly one of asked or excluded`); + if (excluded) assert.match(EXCLUDED[testCase.id], /\S/, `${testCase.id} needs a reason`); + } +}); + +test("no question leaks an arm's syntax", () => { + for (const testCase of CORPUS) { + const exempt = LEAK_EXEMPTIONS[testCase.id] ?? []; + let text = testCase.ask; + for (const allowed of exempt) text = text.split(allowed).join(""); + for (const token of BANNED_TOKENS) { + assert.ok( + !text.toLowerCase().includes(token.toLowerCase()), + `${testCase.id} contains the banned token "${token}": ${testCase.ask}`, + ); + } + } +}); + +test("every gold filter is distinguishable from every other on the differential set", () => { + const seen = new Map(); + // Only the inherited set: an authored probe deliberately restates an + // inherited case with a phrasing that tempts a mistake (m02 is c12 asked in + // the way a user would ask it), so the two are meant to coincide. + for (const testCase of CORPUS.filter((c) => c.origin === "inherited")) { + if (normalise(testCase.expect).problem) continue; + let key; + try { + const compiled = compile(testCase.goldFilter, BINDING); + key = differentialDb.prepare(compiled.sql).all(...compiled.params).map((r) => r.id).join(","); + } catch { + continue; // does not compile on this binding; graded on the fixture alone + } + const clash = seen.get(key); + assert.equal( + clash, + undefined, + `${testCase.id} and ${clash} return identical rows on all 200 records — the grader cannot tell them apart`, + ); + seen.set(key, testCase.id); + } +}); + +test("the differential check accepts a re-spelling and rejects a near-miss", () => { + const between = { weightKg: { $between: [1, 5] } }; + const spelledOut = { weightKg: { $gte: 1, $lte: 5 } }; + const offByOne = { weightKg: { $gte: 1, $lt: 5 } }; + assert.ok(equivalent(differentialDb, between, spelledOut), "an equivalent re-spelling must pass"); + assert.ok(!equivalent(differentialDb, between, offByOne), "an exclusive upper bound must fail"); +}); + +// --------------------------------------------------------------------------- +// Fairness +// --------------------------------------------------------------------------- + +test("the prose budget binds the arms whose language is carried by prose", () => { + const { target, tolerance, binds } = PROSE_BUDGET; + for (const name of binds) { + const arm = ARMS.find((a) => a.name === name); + const size = approxTokens(arm.tool.description); + assert.ok( + size >= target * (1 - tolerance) && size <= target * (1 + tolerance), + `${name}'s description is ~${size} tokens, outside ${Math.round(target * (1 - tolerance))}–${Math.round(target * (1 + tolerance))}`, + ); + } +}); + +test("the two schema arms are given no prose to make up for the schema", () => { + for (const name of ["jql-raw", "jql-generated", "jql-published"]) { + const arm = ARMS.find((a) => a.name === name); + assert.ok( + approxTokens(arm.tool.description) < 100, + `${name} has a ${approxTokens(arm.tool.description)}-token description; the claim is that it needs none`, + ); + for (const token of ["$unknownAs", "$some", "three-valued", "null"]) { + assert.ok( + !arm.tool.description.includes(token), + `${name}'s description mentions ${token}, which would be coaching`, + ); + } + } +}); + +// --------------------------------------------------------------------------- +// The mistake detector +// --------------------------------------------------------------------------- + +test("the mistake detector finds each catalogued mistake, and nothing in the gold answers", () => { + assert.deepEqual(mistakes({ status: "Available" }), ["value-outside-domain"]); + assert.deepEqual(mistakes({ tags: { $in: ["indoor"] } }), ["in-as-membership"]); + assert.deepEqual(mistakes({ microchip: { $ne: "CHIP-001" } }), ["ne-drops-nulls"]); + assert.deepEqual(mistakes({ microchip: { $ne: "CHIP-001", $unknownAs: true } }), []); + assert.deepEqual(mistakes({ tags: { $some: { $in: ["indoor"] } } }), []); + assert.deepEqual(mistakes({ status: "available" }), []); + + // The gold answers are by definition correct, so none may trip a detector, + // with four declared exceptions. + // + // c05, d08 and d09 all ask, in words, to *exclude* the records we cannot + // confirm — so dropping the unknowns is the right answer, and the detector + // firing on them is the detector being right about the shape and wrong about + // the intent. Intent is not recoverable from the filter alone. + // + // c07 is the §4.1 longhand: the detector sees its $ne but not the $isNull arm + // of the $or that rescues it. + // + // Both limits push the same way — they inflate the count — so the per-mistake + // numbers in the report are an upper bound on ne-drops-nulls, and are labelled + // as such. + const expectedTrips = new Set(["c05", "c07", "d08", "d09"]); + for (const testCase of CORPUS) { + const found = mistakes(testCase.goldFilter); + if (expectedTrips.has(testCase.id)) continue; + assert.deepEqual(found, [], `gold answer for ${testCase.id} trips the detector: ${found.join(", ")}`); + } +}); diff --git a/experiments/intent-to-filter/grade.mjs b/experiments/intent-to-filter/grade.mjs new file mode 100644 index 0000000..2ca468a --- /dev/null +++ b/experiments/intent-to-filter/grade.mjs @@ -0,0 +1,208 @@ +/** + * grade.mjs — turning one model response into one outcome. + * + * The distinction the whole experiment turns on is between a failure the agent + * is told about and one it is not. README.md:210 claims that is the real + * difference between a schema-described language and a string: + * + * "What the language cannot catch is a filter that is *valid and wrong*. + * Those fail as an empty result set, which an agent cannot distinguish from + * 'no such records', so it reports a confident false negative." + * + * So `loud` and `silent` are separate outcomes, and the headline number is the + * ratio between them rather than raw accuracy. An arm that is wrong often but + * always says so is, for an agent, a better arm than one that is wrong half as + * often and never says so. + */ + +import { expectationFor } from "../filter-to-sql/cases.mjs"; +import { equivalent } from "./differential.mjs"; + +export const OUTCOMES = [ + "correct", + "loud-failure", + "silent-failure", + "fabrication", + "correct-abstention", + "no-call", + "refusal", + "compiler-defect", +]; + +/** Paths that can be null or absent, so a bare negation over them drops rows. */ +const NULLABLE = new Set([ + "microchip", "notes", "born", "species", "weightKg", "neutered", + "shelter.capacity", "shelter.city", "$$rate", "size\\.raw", "boosterDue", +]); + +/** Paths holding a list, where $in compares the list rather than its elements. */ +const ARRAY_PATHS = new Set(["tags", "vaccinations"]); + +const DOMAINS = { + status: ["available", "pending", "sold"], + species: ["cat", "dog", "rabbit", "bird"], + vaccine: ["rabies", "distemper", "parvo"], +}; + +/** + * The three mistakes README §"Exposing search to an agent" catalogues, detected + * on the translated filter so that one detector serves all five arms. + * + * They are not symmetrical, and the asymmetry is the point of H5: a generated + * schema turns 1 and 3 into validation errors, but 2 stays valid and wrong + * against every arm, mitigated only by prose. Counting them separately is how + * we find out whether the prose works. + */ +export function mistakes(filter) { + const found = new Set(); + if (filter === null || typeof filter !== "object") return []; + + const walk = (node, path = null, negated = false) => { + if (node === null || typeof node !== "object") return; + if (Array.isArray(node)) return node.forEach((m) => walk(m, path, negated)); + + for (const [key, value] of Object.entries(node)) { + if (key === "$and" || key === "$or") { walk(value, path, negated); continue; } + // A negation only drops the unknowns if nothing alongside it resolves + // them: {"born": {"$not": {...}, "$unknownAs": true}} is the correct + // spelling of c13, not a mistake. + const resolvedHere = node.$unknownAs !== undefined; + if (key === "$nor") { walk(value, path, !resolvedHere); continue; } + if (key === "$not") { walk(value, path, !resolvedHere); continue; } + + if (key === "$in" || key === "$nin") { + if (path && ARRAY_PATHS.has(path)) found.add("in-as-membership"); + continue; + } + if (key === "$ne") { + if (path && NULLABLE.has(path) && node.$unknownAs === undefined) found.add("ne-drops-nulls"); + continue; + } + if (key === "$eq" || key === undefined) { + // fall through to the domain check below + } + if (key.startsWith("$")) { + if ((key === "$some" || key === "$every") && value && typeof value === "object") { + walk(value, null, negated); + } + continue; + } + + // A member name: either a field path, or an element member inside a quantifier. + const leaf = key.includes(".") ? key : key; + const domain = DOMAINS[leaf.split(".").pop()]; + const operand = value !== null && typeof value === "object" ? value.$eq : value; + if (domain && typeof operand === "string" && !domain.includes(operand)) { + found.add("value-outside-domain"); + } + if (domain && value !== null && typeof value === "object" && Array.isArray(value.$in)) { + if (value.$in.some((v) => typeof v === "string" && !domain.includes(v))) { + found.add("value-outside-domain"); + } + } + walk(value, leaf, negated); + } + + // A bare field-level $not over a nullable path drops the unknowns too. + if (negated && path && NULLABLE.has(path) && node.$unknownAs === undefined) { + found.add("ne-drops-nulls"); + } + }; + + walk(filter); + return [...found]; +} + +/** + * Did the model say, in words, that the question cannot be answered here? Only + * consulted for `unanswerable` probes, and only when no tool call was made. + */ +function abstained(text) { + if (!text) return false; + return /\b(cannot|can't|unable|not (?:a )?(?:possible|supported|available|searchable|exposed)|no (?:such|way to)|not exposed|isn't (?:searchable|available)|does not support)\b/i + .test(text); +} + +/** + * Grade one attempt. + * + * @param {object} args + * @param {object} args.testCase the corpus case, carrying `expect` and `filter` + * @param {object} args.arm the arm module + * @param {object|undefined} args.input the tool input the model produced + * @param {string} args.text any prose the model returned alongside + * @param {string} args.stopReason + * @param {object} args.db the ten-record fixture + * @param {object} args.differentialDb the two-hundred-record set + */ +export function grade({ testCase, arm, input, text, stopReason, db, differentialDb }) { + const probe = testCase.probe; + + if (stopReason === "refusal") return { outcome: "refusal" }; + + if (input === undefined) { + if (probe === "unanswerable" && abstained(text)) return { outcome: "correct-abstention" }; + return { outcome: "no-call", note: abstained(text) ? "declined" : "no tool call" }; + } + + const result = arm.execute(input, db); + const candidateFilter = arm.translate ? tryTranslate(arm, input) : arm.extract(input); + const made = candidateFilter ? mistakes(candidateFilter) : []; + + // A compiler defect is not the model's fault and not the arm's; it is ours. + if (result.compilerDefect) { + return { outcome: "compiler-defect", problem: result.problem.detail, mistakes: made }; + } + + if (result.problem) { + // For a question that genuinely cannot be asked, a rejection IS the answer. + if (probe === "unanswerable") return { outcome: "correct-abstention", problem: result.problem.type, mistakes: made }; + return { outcome: "loud-failure", problem: result.problem.type, mistakes: made }; + } + + if (probe === "unanswerable") { + return { outcome: "fabrication", ids: result.ids, mistakes: made }; + } + + const expected = testCase.expect !== undefined && !Array.isArray(testCase.expect) + ? testCase.expect + : { ids: testCase.expect }; + const wanted = expected.problem ? null : expected.ids; + if (wanted === null) { + // The corpus says this should have been rejected and it was not. + return { outcome: "silent-failure", ids: result.ids, mistakes: made, note: `expected ${expected.problem}` }; + } + + const sameRows = result.ids.length === wanted.length && result.ids.every((id) => wanted.includes(id)); + if (!sameRows) return { outcome: "silent-failure", ids: result.ids, mistakes: made }; + + // Right rows on ten records is necessary but not sufficient: check the + // candidate means the same thing as the gold filter on two hundred more. + const goldFilter = testCase.goldFilter; + if (goldFilter && candidateFilter && !equivalent(differentialDb, goldFilter, candidateFilter)) { + return { outcome: "silent-failure", ids: result.ids, mistakes: made, note: "right rows, different predicate" }; + } + + return { outcome: "correct", ids: result.ids, mistakes: made }; +} + +function tryTranslate(arm, input) { + try { + return arm.translate(input); + } catch { + return null; + } +} + +/** Wilson score interval — honest error bars on a proportion from few trials. */ +export function wilson(successes, total, z = 1.96) { + if (total === 0) return { low: 0, high: 0, point: 0 }; + const p = successes / total; + const denominator = 1 + (z * z) / total; + const centre = p + (z * z) / (2 * total); + const spread = z * Math.sqrt((p * (1 - p)) / total + (z * z) / (4 * total * total)); + return { point: p, low: Math.max(0, (centre - spread) / denominator), high: Math.min(1, (centre + spread) / denominator) }; +} + +/** The expectation for a case, normalised — re-exported so the runner has one import. */ +export { expectationFor }; diff --git a/experiments/intent-to-filter/pet.filter.json b/experiments/intent-to-filter/pet.filter.json new file mode 100644 index 0000000..b041e7a --- /dev/null +++ b/experiments/intent-to-filter/pet.filter.json @@ -0,0 +1,2326 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.invalid/intent-to-filter/pet.filter.json", + "title": "Pet — filter", + "description": "A filter over Pet, in the JSON Query Language. Sibling members are combined with implicit AND, at every level. A bare scalar is equality: {\"status\": \"open\"} is {\"status\": {\"$eq\": \"open\"}}. Comparisons use three-valued logic: $ne and $not do NOT match records where the field is null or absent. To include those records, add \"$unknownAs\": true to the same constraint. $in compares the whole field value; it is not array membership. To say something about the elements of an array, quantify: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "$comment": "Generated by tools/generate-filter-schema.mjs from https://example.invalid/intent-to-filter/pet.json against https://christosgkoros.com/json/query-language/v0.4.0/query-language-schema.json. Profiles: core, strings, regex, ranges, types, collections, refs. Do not edit by hand.", + "type": "object", + "minProperties": 1, + "properties": { + "$and": { + "title": "$and", + "description": "All of the listed filters must evaluate TRUE.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#" + } + }, + "$or": { + "title": "$or", + "description": "At least one of the listed filters must evaluate TRUE.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#" + } + }, + "$nor": { + "title": "$nor", + "description": "NOT (a OR b OR …), evaluated under three-valued logic. It is TRUE only when every listed filter is FALSE: if one of them is UNKNOWN the result is UNKNOWN, not TRUE, so a $nor over a nullable field excludes the records whose field is null or absent. See SPEC.md §4.1.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#" + } + }, + "$not": { + "title": "$not", + "description": "Negates a filter. Under three-valued logic NOT UNKNOWN is UNKNOWN, not TRUE — so this does not match records whose field is null or absent. Add $unknownAs to the inner constraint to include them. See SPEC.md §4.1.", + "$ref": "#" + }, + "id": { + "anyOf": [ + { + "$ref": "#/$defs/V_id" + }, + { + "$ref": "#/$defs/C_id" + } + ] + }, + "name": { + "anyOf": [ + { + "$ref": "#/$defs/V_name" + }, + { + "$ref": "#/$defs/C_name" + } + ] + }, + "species": { + "anyOf": [ + { + "$ref": "#/$defs/V_species" + }, + { + "$ref": "#/$defs/C_species" + } + ] + }, + "status": { + "anyOf": [ + { + "$ref": "#/$defs/V_status" + }, + { + "$ref": "#/$defs/C_status" + } + ] + }, + "born": { + "anyOf": [ + { + "$ref": "#/$defs/V_born" + }, + { + "$ref": "#/$defs/C_born" + } + ] + }, + "weightKg": { + "anyOf": [ + { + "$ref": "#/$defs/V_weightKg" + }, + { + "$ref": "#/$defs/C_weightKg" + } + ] + }, + "neutered": { + "anyOf": [ + { + "$ref": "#/$defs/V_neutered" + }, + { + "$ref": "#/$defs/C_neutered" + } + ] + }, + "microchip": { + "anyOf": [ + { + "$ref": "#/$defs/V_microchip" + }, + { + "$ref": "#/$defs/C_microchip" + } + ] + }, + "tags": { + "$ref": "#/$defs/C_tags" + }, + "notes": { + "$ref": "#/$defs/C_notes" + }, + "priceCents": { + "anyOf": [ + { + "$ref": "#/$defs/V_priceCents" + }, + { + "$ref": "#/$defs/C_priceCents" + } + ] + }, + "costCents": { + "anyOf": [ + { + "$ref": "#/$defs/V_costCents" + }, + { + "$ref": "#/$defs/C_costCents" + } + ] + }, + "$$rate": { + "anyOf": [ + { + "$ref": "#/$defs/V_rate" + }, + { + "$ref": "#/$defs/C_rate" + } + ] + }, + "size\\.raw": { + "anyOf": [ + { + "$ref": "#/$defs/V_size_raw" + }, + { + "$ref": "#/$defs/C_size_raw" + } + ] + }, + "shelter": { + "$ref": "#/$defs/C_shelter" + }, + "shelter.name": { + "anyOf": [ + { + "$ref": "#/$defs/V_shelter_name" + }, + { + "$ref": "#/$defs/C_shelter_name" + } + ] + }, + "shelter.city": { + "anyOf": [ + { + "$ref": "#/$defs/V_shelter_city" + }, + { + "$ref": "#/$defs/C_shelter_city" + } + ] + }, + "shelter.capacity": { + "anyOf": [ + { + "$ref": "#/$defs/V_shelter_capacity" + }, + { + "$ref": "#/$defs/C_shelter_capacity" + } + ] + }, + "vaccinations": { + "$ref": "#/$defs/C_vaccinations" + } + }, + "additionalProperties": false, + "$defs": { + "V_id": { + "type": "string" + }, + "C_id": { + "title": "id", + "description": "Stable identifier.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_id" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_id" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_id" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_id" + } + }, + "$like": { + "title": "$like", + "description": "SQL LIKE pattern: '%' matches any run of characters, '_' matches exactly one, and '\\' escapes either. Case-sensitive.", + "type": "string", + "examples": [ + "%urgent%", + "INV-____", + "100\\%" + ] + }, + "$nlike": { + "title": "$nlike", + "type": "string" + }, + "$ilike": { + "title": "$ilike", + "description": "Case-insensitive $like. Collation is server-defined; see SPEC.md §5.5.", + "type": "string" + }, + "$nilike": { + "title": "$nilike", + "type": "string" + }, + "$startsWith": { + "title": "$startsWith", + "type": "string" + }, + "$endsWith": { + "title": "$endsWith", + "type": "string" + }, + "$contains": { + "title": "$contains", + "description": "Field contains this literal substring. String-only — for arrays, quantify over the elements with $some or $every.", + "type": "string" + }, + "$regex": { + "title": "$regex", + "description": "Field matches this regular expression (ECMA-262 syntax). Servers SHOULD bound execution time; see SPEC.md §7.", + "type": "string", + "format": "regex" + }, + "$flags": { + "title": "$flags", + "description": "Modifiers for $regex: 'i' case-insensitive, 'm' multiline, 's' dot-matches-newline. Only valid alongside $regex.", + "type": "string", + "pattern": "^[ims]{0,3}$" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_id" + } + }, + "additionalProperties": false, + "dependentRequired": { + "$flags": [ + "$regex" + ] + } + }, + "V_name": { + "type": "string", + "minLength": 1 + }, + "C_name": { + "title": "name", + "description": "Display name.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_name" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_name" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_name" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_name" + } + }, + "$like": { + "title": "$like", + "description": "SQL LIKE pattern: '%' matches any run of characters, '_' matches exactly one, and '\\' escapes either. Case-sensitive.", + "type": "string", + "examples": [ + "%urgent%", + "INV-____", + "100\\%" + ] + }, + "$nlike": { + "title": "$nlike", + "type": "string" + }, + "$ilike": { + "title": "$ilike", + "description": "Case-insensitive $like. Collation is server-defined; see SPEC.md §5.5.", + "type": "string" + }, + "$nilike": { + "title": "$nilike", + "type": "string" + }, + "$startsWith": { + "title": "$startsWith", + "type": "string" + }, + "$endsWith": { + "title": "$endsWith", + "type": "string" + }, + "$contains": { + "title": "$contains", + "description": "Field contains this literal substring. String-only — for arrays, quantify over the elements with $some or $every.", + "type": "string" + }, + "$regex": { + "title": "$regex", + "description": "Field matches this regular expression (ECMA-262 syntax). Servers SHOULD bound execution time; see SPEC.md §7.", + "type": "string", + "format": "regex" + }, + "$flags": { + "title": "$flags", + "description": "Modifiers for $regex: 'i' case-insensitive, 'm' multiline, 's' dot-matches-newline. Only valid alongside $regex.", + "type": "string", + "pattern": "^[ims]{0,3}$" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_name" + } + }, + "additionalProperties": false, + "dependentRequired": { + "$flags": [ + "$regex" + ] + } + }, + "V_species": { + "type": "string", + "enum": [ + "cat", + "dog", + "rabbit", + "bird" + ] + }, + "C_species": { + "title": "species", + "description": "Taxon, coarse-grained.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_species" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_species" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_species" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_species" + } + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_species" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false + }, + "V_status": { + "type": "string", + "anyOf": [ + { + "const": "available", + "description": "Adoptable now." + }, + { + "const": "pending", + "description": "An application is in review." + }, + { + "const": "sold", + "description": "Adopted; kept for history." + } + ] + }, + "C_status": { + "title": "status", + "description": "Listing state.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_status" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_status" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_status" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_status" + } + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_status" + } + }, + "additionalProperties": false + }, + "V_born": { + "type": "string", + "format": "date" + }, + "O_born": { + "type": "string", + "format": "date" + }, + "C_born": { + "title": "born", + "description": "Date of birth, RFC 3339 full-date.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_born" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_born" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_born" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_born" + } + }, + "$gt": { + "title": "$gt", + "$ref": "#/$defs/O_born" + }, + "$gte": { + "title": "$gte", + "$ref": "#/$defs/O_born" + }, + "$lt": { + "title": "$lt", + "$ref": "#/$defs/O_born" + }, + "$lte": { + "title": "$lte", + "$ref": "#/$defs/O_born" + }, + "$between": { + "title": "$between", + "description": "Field falls within [lower, upper], inclusive of both bounds.", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_born" + } + }, + "$nbetween": { + "title": "$nbetween", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_born" + } + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_born" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false + }, + "V_weightKg": { + "type": "number", + "minimum": 0 + }, + "O_weightKg": { + "type": "number" + }, + "C_weightKg": { + "title": "weightKg", + "description": "Last recorded weight.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_weightKg" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_weightKg" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_weightKg" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_weightKg" + } + }, + "$gt": { + "title": "$gt", + "$ref": "#/$defs/O_weightKg" + }, + "$gte": { + "title": "$gte", + "$ref": "#/$defs/O_weightKg" + }, + "$lt": { + "title": "$lt", + "$ref": "#/$defs/O_weightKg" + }, + "$lte": { + "title": "$lte", + "$ref": "#/$defs/O_weightKg" + }, + "$between": { + "title": "$between", + "description": "Field falls within [lower, upper], inclusive of both bounds.", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_weightKg" + } + }, + "$nbetween": { + "title": "$nbetween", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_weightKg" + } + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_weightKg" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false + }, + "V_neutered": { + "type": "boolean" + }, + "C_neutered": { + "title": "neutered", + "description": "Whether the pet has been neutered.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_neutered" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_neutered" + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_neutered" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false + }, + "V_microchip": { + "type": [ + "string", + "null" + ] + }, + "C_microchip": { + "title": "microchip", + "description": "Chip number. Null where the pet is on record as unchipped; the property is absent where nobody has looked.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_microchip" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_microchip" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_microchip" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_microchip" + } + }, + "$like": { + "title": "$like", + "description": "SQL LIKE pattern: '%' matches any run of characters, '_' matches exactly one, and '\\' escapes either. Case-sensitive.", + "type": "string", + "examples": [ + "%urgent%", + "INV-____", + "100\\%" + ] + }, + "$nlike": { + "title": "$nlike", + "type": "string" + }, + "$ilike": { + "title": "$ilike", + "description": "Case-insensitive $like. Collation is server-defined; see SPEC.md §5.5.", + "type": "string" + }, + "$nilike": { + "title": "$nilike", + "type": "string" + }, + "$startsWith": { + "title": "$startsWith", + "type": "string" + }, + "$endsWith": { + "title": "$endsWith", + "type": "string" + }, + "$contains": { + "title": "$contains", + "description": "Field contains this literal substring. String-only — for arrays, quantify over the elements with $some or $every.", + "type": "string" + }, + "$regex": { + "title": "$regex", + "description": "Field matches this regular expression (ECMA-262 syntax). Servers SHOULD bound execution time; see SPEC.md §7.", + "type": "string", + "format": "regex" + }, + "$flags": { + "title": "$flags", + "description": "Modifiers for $regex: 'i' case-insensitive, 'm' multiline, 's' dot-matches-newline. Only valid alongside $regex.", + "type": "string", + "pattern": "^[ims]{0,3}$" + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$isNull": { + "title": "$isNull", + "description": "true if the field's value is null (equivalent to {\"$eq\": null}); false if it is non-null (equivalent to {\"$ne\": null}). UNKNOWN only when the path resolves to nothing.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_microchip" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false, + "dependentRequired": { + "$flags": [ + "$regex" + ] + } + }, + "I_tags": { + "type": "string" + }, + "V_tags_elem": { + "type": "string" + }, + "C_tags_elem": { + "title": "tags element", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_tags_elem" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_tags_elem" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_tags_elem" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_tags_elem" + } + }, + "$like": { + "title": "$like", + "description": "SQL LIKE pattern: '%' matches any run of characters, '_' matches exactly one, and '\\' escapes either. Case-sensitive.", + "type": "string", + "examples": [ + "%urgent%", + "INV-____", + "100\\%" + ] + }, + "$nlike": { + "title": "$nlike", + "type": "string" + }, + "$ilike": { + "title": "$ilike", + "description": "Case-insensitive $like. Collation is server-defined; see SPEC.md §5.5.", + "type": "string" + }, + "$nilike": { + "title": "$nilike", + "type": "string" + }, + "$startsWith": { + "title": "$startsWith", + "type": "string" + }, + "$endsWith": { + "title": "$endsWith", + "type": "string" + }, + "$contains": { + "title": "$contains", + "description": "Field contains this literal substring. String-only — for arrays, quantify over the elements with $some or $every.", + "type": "string" + }, + "$regex": { + "title": "$regex", + "description": "Field matches this regular expression (ECMA-262 syntax). Servers SHOULD bound execution time; see SPEC.md §7.", + "type": "string", + "format": "regex" + }, + "$flags": { + "title": "$flags", + "description": "Modifiers for $regex: 'i' case-insensitive, 'm' multiline, 's' dot-matches-newline. Only valid alongside $regex.", + "type": "string", + "pattern": "^[ims]{0,3}$" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_tags_elem" + } + }, + "additionalProperties": false, + "dependentRequired": { + "$flags": [ + "$regex" + ] + } + }, + "Size": { + "title": "Size constraint", + "description": "An exact array length, or a comparison on that length.", + "anyOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "type": "integer", + "minimum": 0 + }, + "$ne": { + "type": "integer", + "minimum": 0 + }, + "$gt": { + "type": "integer", + "minimum": 0 + }, + "$gte": { + "type": "integer", + "minimum": 0 + }, + "$lt": { + "type": "integer", + "minimum": 0 + }, + "$lte": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": false + } + ], + "examples": [ + 3, + { + "$gte": 1 + } + ] + }, + "C_tags": { + "title": "tags", + "description": "Free-form labels used for browsing.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "type": "array", + "items": { + "$ref": "#/$defs/I_tags" + } + }, + "$ne": { + "title": "$ne", + "type": "array", + "items": { + "$ref": "#/$defs/I_tags" + } + }, + "$some": { + "title": "$some", + "description": "At least one element of the array-valued field satisfies this condition. Supply a Filter for arrays of objects (paths are relative to the element) or a constraint object for arrays of scalars. An element for which the condition is UNKNOWN does not satisfy it, so the quantifier yields TRUE or FALSE over any array — FALSE for an empty one. It is UNKNOWN only when the field is not an array, including when it is null or absent. See SPEC.md §5.8.", + "$ref": "#/$defs/C_tags_elem" + }, + "$every": { + "title": "$every", + "description": "Every element of the array-valued field satisfies this condition; an element for which it is UNKNOWN does not. TRUE for an empty array, vacuously. UNKNOWN only when the field is not an array, including when it is null or absent — a missing array is not vacuously TRUE. Same operand shape as $some. See SPEC.md §5.8.", + "$ref": "#/$defs/C_tags_elem" + }, + "$hasAll": { + "title": "$hasAll", + "description": "The array-valued field contains every member of this list. Quantifies over the operand rather than over the elements, which is why it is not expressible as a $some or $every. Set semantics: multiplicity is ignored. UNKNOWN when the field is not an array.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/I_tags" + } + }, + "$size": { + "title": "$size", + "description": "Length of the array-valued field: either an exact count or a nested numeric constraint.", + "$ref": "#/$defs/Size" + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_tags" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false + }, + "V_notes": { + "type": [ + "string", + "number", + "boolean", + "object", + "array", + "null" + ] + }, + "O_notes": { + "type": [ + "string", + "number", + "boolean", + "object", + "array" + ] + }, + "C_notes": { + "title": "notes", + "description": "Unstructured staff annotation. Deliberately heterogeneous: every record holds a different JSON type here.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_notes" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_notes" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_notes" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_notes" + } + }, + "$gt": { + "title": "$gt", + "$ref": "#/$defs/O_notes" + }, + "$gte": { + "title": "$gte", + "$ref": "#/$defs/O_notes" + }, + "$lt": { + "title": "$lt", + "$ref": "#/$defs/O_notes" + }, + "$lte": { + "title": "$lte", + "$ref": "#/$defs/O_notes" + }, + "$between": { + "title": "$between", + "description": "Field falls within [lower, upper], inclusive of both bounds.", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_notes" + } + }, + "$nbetween": { + "title": "$nbetween", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_notes" + } + }, + "$like": { + "title": "$like", + "description": "SQL LIKE pattern: '%' matches any run of characters, '_' matches exactly one, and '\\' escapes either. Case-sensitive.", + "type": "string", + "examples": [ + "%urgent%", + "INV-____", + "100\\%" + ] + }, + "$nlike": { + "title": "$nlike", + "type": "string" + }, + "$ilike": { + "title": "$ilike", + "description": "Case-insensitive $like. Collation is server-defined; see SPEC.md §5.5.", + "type": "string" + }, + "$nilike": { + "title": "$nilike", + "type": "string" + }, + "$startsWith": { + "title": "$startsWith", + "type": "string" + }, + "$endsWith": { + "title": "$endsWith", + "type": "string" + }, + "$contains": { + "title": "$contains", + "description": "Field contains this literal substring. String-only — for arrays, quantify over the elements with $some or $every.", + "type": "string" + }, + "$regex": { + "title": "$regex", + "description": "Field matches this regular expression (ECMA-262 syntax). Servers SHOULD bound execution time; see SPEC.md §7.", + "type": "string", + "format": "regex" + }, + "$flags": { + "title": "$flags", + "description": "Modifiers for $regex: 'i' case-insensitive, 'm' multiline, 's' dot-matches-newline. Only valid alongside $regex.", + "type": "string", + "pattern": "^[ims]{0,3}$" + }, + "$type": { + "title": "$type", + "description": "Field's JSON type. 'integer' matches a number with no fractional part.", + "type": "string", + "enum": [ + "string", + "number", + "integer", + "boolean", + "object", + "array", + "null" + ] + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$isNull": { + "title": "$isNull", + "description": "true if the field's value is null (equivalent to {\"$eq\": null}); false if it is non-null (equivalent to {\"$ne\": null}). UNKNOWN only when the path resolves to nothing.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_notes" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false, + "dependentRequired": { + "$flags": [ + "$regex" + ] + } + }, + "V_priceCents": { + "type": "integer", + "minimum": 0 + }, + "O_priceCents": { + "type": "integer" + }, + "C_priceCents": { + "title": "priceCents", + "description": "Asking price, in cents.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_priceCents" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_priceCents" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_priceCents" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_priceCents" + } + }, + "$gt": { + "title": "$gt", + "$ref": "#/$defs/O_priceCents" + }, + "$gte": { + "title": "$gte", + "$ref": "#/$defs/O_priceCents" + }, + "$lt": { + "title": "$lt", + "$ref": "#/$defs/O_priceCents" + }, + "$lte": { + "title": "$lte", + "$ref": "#/$defs/O_priceCents" + }, + "$between": { + "title": "$between", + "description": "Field falls within [lower, upper], inclusive of both bounds.", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_priceCents" + } + }, + "$nbetween": { + "title": "$nbetween", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_priceCents" + } + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_priceCents" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false + }, + "V_costCents": { + "type": "integer", + "minimum": 0 + }, + "O_costCents": { + "type": "integer" + }, + "C_costCents": { + "title": "costCents", + "description": "What the shelter spent, in cents.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_costCents" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_costCents" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_costCents" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_costCents" + } + }, + "$gt": { + "title": "$gt", + "$ref": "#/$defs/O_costCents" + }, + "$gte": { + "title": "$gte", + "$ref": "#/$defs/O_costCents" + }, + "$lt": { + "title": "$lt", + "$ref": "#/$defs/O_costCents" + }, + "$lte": { + "title": "$lte", + "$ref": "#/$defs/O_costCents" + }, + "$between": { + "title": "$between", + "description": "Field falls within [lower, upper], inclusive of both bounds.", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_costCents" + } + }, + "$nbetween": { + "title": "$nbetween", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_costCents" + } + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_costCents" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false + }, + "V_rate": { + "type": "integer" + }, + "O_rate": { + "type": "integer" + }, + "C_rate": { + "title": "$$rate", + "description": "Daily boarding rate. The property name really does begin with a dollar sign.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_rate" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_rate" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_rate" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_rate" + } + }, + "$gt": { + "title": "$gt", + "$ref": "#/$defs/O_rate" + }, + "$gte": { + "title": "$gte", + "$ref": "#/$defs/O_rate" + }, + "$lt": { + "title": "$lt", + "$ref": "#/$defs/O_rate" + }, + "$lte": { + "title": "$lte", + "$ref": "#/$defs/O_rate" + }, + "$between": { + "title": "$between", + "description": "Field falls within [lower, upper], inclusive of both bounds.", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_rate" + } + }, + "$nbetween": { + "title": "$nbetween", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_rate" + } + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_rate" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false + }, + "V_size_raw": { + "type": "string" + }, + "C_size_raw": { + "title": "size\\.raw", + "description": "Size code as supplied by the intake form. The property name really does contain a dot.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_size_raw" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_size_raw" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_size_raw" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_size_raw" + } + }, + "$like": { + "title": "$like", + "description": "SQL LIKE pattern: '%' matches any run of characters, '_' matches exactly one, and '\\' escapes either. Case-sensitive.", + "type": "string", + "examples": [ + "%urgent%", + "INV-____", + "100\\%" + ] + }, + "$nlike": { + "title": "$nlike", + "type": "string" + }, + "$ilike": { + "title": "$ilike", + "description": "Case-insensitive $like. Collation is server-defined; see SPEC.md §5.5.", + "type": "string" + }, + "$nilike": { + "title": "$nilike", + "type": "string" + }, + "$startsWith": { + "title": "$startsWith", + "type": "string" + }, + "$endsWith": { + "title": "$endsWith", + "type": "string" + }, + "$contains": { + "title": "$contains", + "description": "Field contains this literal substring. String-only — for arrays, quantify over the elements with $some or $every.", + "type": "string" + }, + "$regex": { + "title": "$regex", + "description": "Field matches this regular expression (ECMA-262 syntax). Servers SHOULD bound execution time; see SPEC.md §7.", + "type": "string", + "format": "regex" + }, + "$flags": { + "title": "$flags", + "description": "Modifiers for $regex: 'i' case-insensitive, 'm' multiline, 's' dot-matches-newline. Only valid alongside $regex.", + "type": "string", + "pattern": "^[ims]{0,3}$" + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_size_raw" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false, + "dependentRequired": { + "$flags": [ + "$regex" + ] + } + }, + "C_shelter": { + "title": "shelter", + "description": "Where the pet is housed.", + "type": "object", + "minProperties": 1, + "properties": { + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false + }, + "V_shelter_name": { + "type": "string" + }, + "C_shelter_name": { + "title": "shelter.name", + "description": "Shelter name.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_shelter_name" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_shelter_name" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_shelter_name" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_shelter_name" + } + }, + "$like": { + "title": "$like", + "description": "SQL LIKE pattern: '%' matches any run of characters, '_' matches exactly one, and '\\' escapes either. Case-sensitive.", + "type": "string", + "examples": [ + "%urgent%", + "INV-____", + "100\\%" + ] + }, + "$nlike": { + "title": "$nlike", + "type": "string" + }, + "$ilike": { + "title": "$ilike", + "description": "Case-insensitive $like. Collation is server-defined; see SPEC.md §5.5.", + "type": "string" + }, + "$nilike": { + "title": "$nilike", + "type": "string" + }, + "$startsWith": { + "title": "$startsWith", + "type": "string" + }, + "$endsWith": { + "title": "$endsWith", + "type": "string" + }, + "$contains": { + "title": "$contains", + "description": "Field contains this literal substring. String-only — for arrays, quantify over the elements with $some or $every.", + "type": "string" + }, + "$regex": { + "title": "$regex", + "description": "Field matches this regular expression (ECMA-262 syntax). Servers SHOULD bound execution time; see SPEC.md §7.", + "type": "string", + "format": "regex" + }, + "$flags": { + "title": "$flags", + "description": "Modifiers for $regex: 'i' case-insensitive, 'm' multiline, 's' dot-matches-newline. Only valid alongside $regex.", + "type": "string", + "pattern": "^[ims]{0,3}$" + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_shelter_name" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false, + "dependentRequired": { + "$flags": [ + "$regex" + ] + } + }, + "V_shelter_city": { + "type": "string" + }, + "C_shelter_city": { + "title": "shelter.city", + "description": "Shelter city.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_shelter_city" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_shelter_city" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_shelter_city" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_shelter_city" + } + }, + "$like": { + "title": "$like", + "description": "SQL LIKE pattern: '%' matches any run of characters, '_' matches exactly one, and '\\' escapes either. Case-sensitive.", + "type": "string", + "examples": [ + "%urgent%", + "INV-____", + "100\\%" + ] + }, + "$nlike": { + "title": "$nlike", + "type": "string" + }, + "$ilike": { + "title": "$ilike", + "description": "Case-insensitive $like. Collation is server-defined; see SPEC.md §5.5.", + "type": "string" + }, + "$nilike": { + "title": "$nilike", + "type": "string" + }, + "$startsWith": { + "title": "$startsWith", + "type": "string" + }, + "$endsWith": { + "title": "$endsWith", + "type": "string" + }, + "$contains": { + "title": "$contains", + "description": "Field contains this literal substring. String-only — for arrays, quantify over the elements with $some or $every.", + "type": "string" + }, + "$regex": { + "title": "$regex", + "description": "Field matches this regular expression (ECMA-262 syntax). Servers SHOULD bound execution time; see SPEC.md §7.", + "type": "string", + "format": "regex" + }, + "$flags": { + "title": "$flags", + "description": "Modifiers for $regex: 'i' case-insensitive, 'm' multiline, 's' dot-matches-newline. Only valid alongside $regex.", + "type": "string", + "pattern": "^[ims]{0,3}$" + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_shelter_city" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false, + "dependentRequired": { + "$flags": [ + "$regex" + ] + } + }, + "V_shelter_capacity": { + "type": "integer", + "minimum": 0 + }, + "O_shelter_capacity": { + "type": "integer" + }, + "C_shelter_capacity": { + "title": "shelter.capacity", + "description": "Total kennels.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/V_shelter_capacity" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/V_shelter_capacity" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_shelter_capacity" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/V_shelter_capacity" + } + }, + "$gt": { + "title": "$gt", + "$ref": "#/$defs/O_shelter_capacity" + }, + "$gte": { + "title": "$gte", + "$ref": "#/$defs/O_shelter_capacity" + }, + "$lt": { + "title": "$lt", + "$ref": "#/$defs/O_shelter_capacity" + }, + "$lte": { + "title": "$lte", + "$ref": "#/$defs/O_shelter_capacity" + }, + "$between": { + "title": "$between", + "description": "Field falls within [lower, upper], inclusive of both bounds.", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_shelter_capacity" + } + }, + "$nbetween": { + "title": "$nbetween", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/O_shelter_capacity" + } + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_shelter_capacity" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false + }, + "I_vaccinations": { + "type": "object" + }, + "F_vaccinations_elem_V_vaccine": { + "type": "string", + "enum": [ + "rabies", + "distemper", + "parvo" + ] + }, + "F_vaccinations_elem_C_vaccine": { + "title": "vaccine", + "description": "Vaccine given.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/F_vaccinations_elem_V_vaccine" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/F_vaccinations_elem_V_vaccine" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/F_vaccinations_elem_V_vaccine" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/F_vaccinations_elem_V_vaccine" + } + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/F_vaccinations_elem_C_vaccine" + } + }, + "additionalProperties": false + }, + "F_vaccinations_elem_V_administeredAt": { + "type": "string", + "format": "date-time" + }, + "F_vaccinations_elem_O_administeredAt": { + "type": "string", + "format": "date-time" + }, + "F_vaccinations_elem_C_administeredAt": { + "title": "administeredAt", + "description": "When it was given.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/F_vaccinations_elem_V_administeredAt" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/F_vaccinations_elem_V_administeredAt" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/F_vaccinations_elem_V_administeredAt" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/F_vaccinations_elem_V_administeredAt" + } + }, + "$gt": { + "title": "$gt", + "$ref": "#/$defs/F_vaccinations_elem_O_administeredAt" + }, + "$gte": { + "title": "$gte", + "$ref": "#/$defs/F_vaccinations_elem_O_administeredAt" + }, + "$lt": { + "title": "$lt", + "$ref": "#/$defs/F_vaccinations_elem_O_administeredAt" + }, + "$lte": { + "title": "$lte", + "$ref": "#/$defs/F_vaccinations_elem_O_administeredAt" + }, + "$between": { + "title": "$between", + "description": "Field falls within [lower, upper], inclusive of both bounds.", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/F_vaccinations_elem_O_administeredAt" + } + }, + "$nbetween": { + "title": "$nbetween", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/F_vaccinations_elem_O_administeredAt" + } + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/F_vaccinations_elem_C_administeredAt" + } + }, + "additionalProperties": false + }, + "F_vaccinations_elem_V_boosterDue": { + "type": "string", + "format": "date" + }, + "F_vaccinations_elem_O_boosterDue": { + "type": "string", + "format": "date" + }, + "F_vaccinations_elem_C_boosterDue": { + "title": "boosterDue", + "description": "When the booster is due.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "$ref": "#/$defs/F_vaccinations_elem_V_boosterDue" + }, + "$ne": { + "title": "$ne", + "$ref": "#/$defs/F_vaccinations_elem_V_boosterDue" + }, + "$in": { + "title": "$in", + "description": "Field equals at least one member of the list. Compares the whole value — it does NOT test membership inside an array-valued field: {\"tags\": {\"$in\": [\"a\"]}} asks whether tags equals \"a\". For array elements quantify explicitly: {\"tags\": {\"$some\": {\"$in\": [\"a\"]}}}.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/F_vaccinations_elem_V_boosterDue" + } + }, + "$nin": { + "title": "$nin", + "description": "Field equals no member of the list. Like $in, compares the whole value rather than array elements. The negation of $in under three-valued logic, so it is UNKNOWN when the field is null or absent.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/F_vaccinations_elem_V_boosterDue" + } + }, + "$gt": { + "title": "$gt", + "$ref": "#/$defs/F_vaccinations_elem_O_boosterDue" + }, + "$gte": { + "title": "$gte", + "$ref": "#/$defs/F_vaccinations_elem_O_boosterDue" + }, + "$lt": { + "title": "$lt", + "$ref": "#/$defs/F_vaccinations_elem_O_boosterDue" + }, + "$lte": { + "title": "$lte", + "$ref": "#/$defs/F_vaccinations_elem_O_boosterDue" + }, + "$between": { + "title": "$between", + "description": "Field falls within [lower, upper], inclusive of both bounds.", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/F_vaccinations_elem_O_boosterDue" + } + }, + "$nbetween": { + "title": "$nbetween", + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "$ref": "#/$defs/F_vaccinations_elem_O_boosterDue" + } + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/F_vaccinations_elem_C_boosterDue" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false + }, + "F_vaccinations_elem": { + "type": "object", + "minProperties": 1, + "properties": { + "$and": { + "title": "$and", + "description": "All of the listed filters must evaluate TRUE.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/F_vaccinations_elem" + } + }, + "$or": { + "title": "$or", + "description": "At least one of the listed filters must evaluate TRUE.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/F_vaccinations_elem" + } + }, + "$nor": { + "title": "$nor", + "description": "NOT (a OR b OR …), evaluated under three-valued logic. It is TRUE only when every listed filter is FALSE: if one of them is UNKNOWN the result is UNKNOWN, not TRUE, so a $nor over a nullable field excludes the records whose field is null or absent. See SPEC.md §4.1.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/F_vaccinations_elem" + } + }, + "$not": { + "title": "$not", + "description": "Negates a filter. Under three-valued logic NOT UNKNOWN is UNKNOWN, not TRUE — so this does not match records whose field is null or absent. Add $unknownAs to the inner constraint to include them. See SPEC.md §4.1.", + "$ref": "#/$defs/F_vaccinations_elem" + }, + "vaccine": { + "anyOf": [ + { + "$ref": "#/$defs/F_vaccinations_elem_V_vaccine" + }, + { + "$ref": "#/$defs/F_vaccinations_elem_C_vaccine" + } + ] + }, + "administeredAt": { + "anyOf": [ + { + "$ref": "#/$defs/F_vaccinations_elem_V_administeredAt" + }, + { + "$ref": "#/$defs/F_vaccinations_elem_C_administeredAt" + } + ] + }, + "boosterDue": { + "anyOf": [ + { + "$ref": "#/$defs/F_vaccinations_elem_V_boosterDue" + }, + { + "$ref": "#/$defs/F_vaccinations_elem_C_boosterDue" + } + ] + } + }, + "additionalProperties": false + }, + "C_vaccinations": { + "title": "vaccinations", + "description": "Vaccination history.", + "type": "object", + "minProperties": 1, + "properties": { + "$eq": { + "title": "$eq", + "type": "array", + "items": { + "$ref": "#/$defs/I_vaccinations" + } + }, + "$ne": { + "title": "$ne", + "type": "array", + "items": { + "$ref": "#/$defs/I_vaccinations" + } + }, + "$some": { + "title": "$some", + "description": "At least one element of the array-valued field satisfies this condition. Supply a Filter for arrays of objects (paths are relative to the element) or a constraint object for arrays of scalars. An element for which the condition is UNKNOWN does not satisfy it, so the quantifier yields TRUE or FALSE over any array — FALSE for an empty one. It is UNKNOWN only when the field is not an array, including when it is null or absent. See SPEC.md §5.8.", + "$ref": "#/$defs/F_vaccinations_elem" + }, + "$every": { + "title": "$every", + "description": "Every element of the array-valued field satisfies this condition; an element for which it is UNKNOWN does not. TRUE for an empty array, vacuously. UNKNOWN only when the field is not an array, including when it is null or absent — a missing array is not vacuously TRUE. Same operand shape as $some. See SPEC.md §5.8.", + "$ref": "#/$defs/F_vaccinations_elem" + }, + "$size": { + "title": "$size", + "description": "Length of the array-valued field: either an exact count or a nested numeric constraint.", + "$ref": "#/$defs/Size" + }, + "$exists": { + "title": "$exists", + "description": "true if the field is present on the record, even when its value is null; false if absent. Total — never UNKNOWN. Distinct from $isNull — see SPEC.md §4.2.", + "type": "boolean" + }, + "$not": { + "title": "$not (field-level)", + "description": "Negates the constraint on this field. {\"age\": {\"$not\": {\"$gt\": 5}}} is TRUE when age is not greater than 5 and UNKNOWN when age is null or absent.", + "$ref": "#/$defs/C_vaccinations" + }, + "$unknownAs": { + "title": "$unknownAs", + "description": "Resolves UNKNOWN to TRUE or FALSE for this constraint, applied last — after every sibling operator, including a field-level $not. {\"status\": {\"$ne\": \"archived\", \"$unknownAs\": true}} means 'not archived, and count the records where status is null or absent'. A modifier, not a predicate: it needs at least one operator beside it. No-op on $exists, which is already total. See SPEC.md §4.6.", + "type": "boolean" + } + }, + "additionalProperties": false + } + } +} diff --git a/experiments/intent-to-filter/pet.resource.json b/experiments/intent-to-filter/pet.resource.json new file mode 100644 index 0000000..537f022 --- /dev/null +++ b/experiments/intent-to-filter/pet.resource.json @@ -0,0 +1,111 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.invalid/intent-to-filter/pet.json", + "title": "Pet", + "description": "A pet listed in the store.", + "$comment": "The resource schema for the experiment's fixture collection. It is NOT examples/pet.schema.json: the fixture in ../filter-to-sql/dataset.mjs carries four paths that file does not model — priceCents, costCents, the deliberately heterogeneous notes, and the two adversarially-named keys $rate and size.raw — and thirteen corpus cases query them. Modelling the fixture as it actually is keeps every arm's ceiling a property of the arm rather than of a mismatched schema.", + "type": "object", + "required": ["id", "name", "status"], + "properties": { + "id": { + "type": "string", + "description": "Stable identifier." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name." + }, + "species": { + "type": "string", + "enum": ["cat", "dog", "rabbit", "bird"], + "description": "Taxon, coarse-grained." + }, + "status": { + "type": "string", + "description": "Listing state.", + "oneOf": [ + { "const": "available", "description": "Adoptable now." }, + { "const": "pending", "description": "An application is in review." }, + { "const": "sold", "description": "Adopted; kept for history." } + ] + }, + "born": { + "type": "string", + "format": "date", + "description": "Date of birth, RFC 3339 full-date." + }, + "weightKg": { + "type": "number", + "minimum": 0, + "description": "Last recorded weight." + }, + "neutered": { + "type": "boolean", + "description": "Whether the pet has been neutered." + }, + "microchip": { + "type": ["string", "null"], + "description": "Chip number. Null where the pet is on record as unchipped; the property is absent where nobody has looked." + }, + "tags": { + "type": "array", + "description": "Free-form labels used for browsing.", + "items": { "type": "string" } + }, + "notes": { + "type": ["string", "number", "boolean", "object", "array", "null"], + "description": "Unstructured staff annotation. Deliberately heterogeneous: every record holds a different JSON type here." + }, + "priceCents": { + "type": "integer", + "minimum": 0, + "description": "Asking price, in cents." + }, + "costCents": { + "type": "integer", + "minimum": 0, + "description": "What the shelter spent, in cents." + }, + "$rate": { + "type": "integer", + "description": "Daily boarding rate. The property name really does begin with a dollar sign." + }, + "size.raw": { + "type": "string", + "description": "Size code as supplied by the intake form. The property name really does contain a dot." + }, + "shelter": { + "type": "object", + "description": "Where the pet is housed.", + "required": ["name"], + "properties": { + "name": { "type": "string", "description": "Shelter name." }, + "city": { "type": "string", "description": "Shelter city." }, + "capacity": { "type": "integer", "minimum": 0, "description": "Total kennels." } + } + }, + "vaccinations": { + "type": "array", + "description": "Vaccination history.", + "items": { + "type": "object", + "required": ["vaccine", "administeredAt"], + "properties": { + "vaccine": { + "type": "string", + "enum": ["rabies", "distemper", "parvo"], + "description": "Vaccine given." + }, + "administeredAt": { "type": "string", "format": "date-time", "description": "When it was given." }, + "boosterDue": { "type": "string", "format": "date", "description": "When the booster is due." } + } + } + }, + "internalNotes": { + "type": "string", + "description": "Staff-only. Not exposed to search.", + "x-jql": false + } + } +} diff --git a/experiments/intent-to-filter/providers.mjs b/experiments/intent-to-filter/providers.mjs new file mode 100644 index 0000000..133182b --- /dev/null +++ b/experiments/intent-to-filter/providers.mjs @@ -0,0 +1,247 @@ +/** + * providers.mjs — the same tool definition, put to three model families. + * + * The experiment was designed for one provider and had to move to three when + * no Anthropic key was available. That turned out to be the better design: + * `README.md:3` claims a property of "a model", not of one vendor's models, so + * evidence from three independent families answers an objection that two + * models from one family never could. + * + * IT ALSO SURFACED THE LARGEST FINDING IN THIS DIRECTORY, before any question + * was asked. Gemini cannot host the recommended setup at all. Its function + * declaration schema is a restricted subset with no `$ref`, and the filter + * grammar is recursive — `{"$and": {"items": {"$ref": "#"}}}` — so it cannot be + * flattened into a finite tree either. Not degraded: impossible. Both schema + * arms are therefore unavailable on that provider, and on Gemini the string + * syntax wins by default. README §"Exposing search to an agent" tells the + * reader to inline the schema and does not mention that a major provider will + * reject it. + * + * That is recorded as `provider-incompatible`, not as a failure of the arm, and + * it is reported as its own row. + */ + +import { flattenTool } from "./deref.mjs"; + +const OPENAI = "https://api.openai.com/v1/chat/completions"; +const GROQ = "https://api.groq.com/openai/v1/chat/completions"; +const GEMINI = (model) => + `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`; + +export const PROVIDERS = { + "gpt-5.5": { family: "openai", model: "gpt-5.5", url: OPENAI, keyVar: "OPENAI_API_KEY", label: "GPT-5.5 (OpenAI)" }, + "gpt-5.4-mini": { family: "openai", model: "gpt-5.4-mini", url: OPENAI, keyVar: "OPENAI_API_KEY", label: "GPT-5.4-mini (OpenAI)" }, + "gpt-oss-120b": { family: "openai", model: "openai/gpt-oss-120b", url: GROQ, keyVar: "GROQ_API_KEY", label: "gpt-oss-120b (Groq, open weight)" }, + "gemini-3.8-flash": { family: "gemini", model: "gemini-3.8-flash", url: GEMINI("gemini-3.8-flash"), keyVar: "GEMINI_API_KEY", label: "Gemini 3.8 Flash" }, +}; + +/** + * The same system prompt everywhere. The last sentence exists so that declining + * is a permitted move — two of the questions cannot be answered by the tool, + * and without it those would be a trick rather than a probe. + */ +export const SYSTEM = [ + "You are an assistant with access to a pet-store search tool.", + "Answer the user's question by calling the tool with the arguments that express it.", + "If the question cannot be expressed with the tool as described, say so plainly instead of approximating it.", +].join(" "); + +// --------------------------------------------------------------------------- +// Gemini's schema subset +// --------------------------------------------------------------------------- + +/** Keywords Gemini's function-declaration parser rejects outright. */ +const GEMINI_STRIPS = ["$schema", "$id", "$comment", "title", "default", "additionalProperties", "examples"]; + +function hasRef(node) { + if (!node || typeof node !== "object") return false; + if (Array.isArray(node)) return node.some(hasRef); + if ("$ref" in node || "$defs" in node) return true; + return Object.values(node).some(hasRef); +} + +function sanitiseForGemini(schema) { + const clone = structuredClone(schema); + const walk = (node) => { + if (!node || typeof node !== "object") return; + if (Array.isArray(node)) return node.forEach(walk); + for (const key of GEMINI_STRIPS) delete node[key]; + Object.values(node).forEach(walk); + }; + walk(clone); + return clone; +} + +/** + * Can this provider carry this arm's tool definition at all? + * Returns null when it can, or the reason it cannot. + */ +/** + * Keywords Gemini's function-declaration schema has no field for. Each one is + * load-bearing in the filter grammar, so dropping them to force the request + * through would ship a different language than the one under test. + */ +const GEMINI_UNSUPPORTED = ["$ref", "$defs", "dependentRequired", "dependentSchemas", "propertyNames", "patternProperties", "not", "if", "then"]; + +function usesAny(node, keywords) { + if (!node || typeof node !== "object") return null; + if (Array.isArray(node)) { + for (const member of node) { const hit = usesAny(member, keywords); if (hit) return hit; } + return null; + } + for (const key of Object.keys(node)) if (keywords.includes(key)) return key; + for (const value of Object.values(node)) { const hit = usesAny(value, keywords); if (hit) return hit; } + return null; +} + +/** + * Can this provider carry this arm's tool definition at all? + * + * For Gemini the answer for both schema arms is no, and it took two rounds to + * establish. First `$ref`, which cannot be flattened away because the grammar + * is recursive. Then, once flattened anyway, `dependentRequired` — the keyword + * that makes `$flags` require `$regex` — and `dependentSchemas`, the one that + * stops `$unknownAs` standing alone. Both are semantics, not decoration; a + * request that omits them describes a different language. + */ +export function incompatibility(providerKey, arm) { + const provider = PROVIDERS[providerKey]; + if (provider.family !== "gemini") return null; + if (arm.rawRefs && hasRef(arm.tool.input_schema)) { + return "Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"; + } + const offender = usesAny(flattenTool(arm.tool, { dialect: "gemini" }).input_schema, GEMINI_UNSUPPORTED); + if (offender) { + return `Gemini function declarations have no "${offender}" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language`; + } + return null; +} + +/** + * The tool as this provider will actually receive it. + * + * Every arm but `jql-raw` is flattened first, because a `$ref` reaches no model + * intact: Gemini 400s on it and OpenAI silently drops `$defs`. `jql-raw` is + * deliberately left alone so the gap between "what the README says to ship" and + * "what a model can actually read" is a measured row rather than a claim. + */ +export function toolFor(providerKey, arm) { + if (arm.rawRefs) return arm.tool; + const dialect = PROVIDERS[providerKey].family === "gemini" ? "gemini" : "openai"; + return flattenTool(arm.tool, { dialect }); +} + +// --------------------------------------------------------------------------- +// Calling +// --------------------------------------------------------------------------- + +async function postJson(url, headers, body, { attempts = 5 } = {}) { + let wait = 2000; + for (let attempt = 1; ; attempt += 1) { + const response = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json", ...headers }, + body: JSON.stringify(body), + }); + if (response.ok) return { ok: true, json: await response.json() }; + const text = await response.text(); + // 429 and 5xx are worth waiting out; a 400 is our bug and never will be. + const retryable = response.status === 429 || response.status >= 500; + if (!retryable || attempt >= attempts) { + return { ok: false, status: response.status, error: text.slice(0, 400) }; + } + await new Promise((resolve) => setTimeout(resolve, wait)); + wait = Math.min(wait * 2, 30_000); + } +} + +function openAiTool(providerKey, arm) { + const tool = toolFor(providerKey, arm); + return [{ + type: "function", + function: { name: tool.name, description: tool.description, parameters: tool.input_schema }, + }]; +} + +async function callOpenAi(providerKey, provider, arm, question, priorTurns) { + const result = await postJson(provider.url, { authorization: `Bearer ${process.env[provider.keyVar]}` }, { + model: provider.model, + messages: [{ role: "system", content: SYSTEM }, { role: "user", content: question }, ...priorTurns], + tools: openAiTool(providerKey, arm), + }); + if (!result.ok) return { error: `${result.status}: ${result.error}` }; + + const choice = result.json.choices?.[0]; + const call = choice?.message?.tool_calls?.[0]; + let input; + if (call) { + // Never string-match a tool call's arguments; parse them. + try { input = JSON.parse(call.function.arguments); } catch { input = undefined; } + } + return { + input, + toolCallId: call?.id, + text: choice?.message?.content ?? "", + stopReason: choice?.finish_reason, + usage: { input_tokens: result.json.usage?.prompt_tokens, output_tokens: result.json.usage?.completion_tokens }, + assistantMessage: choice?.message, + }; +} + +async function callGemini(providerKey, provider, arm, question, priorTurns) { + const result = await postJson(`${provider.url}?key=${process.env[provider.keyVar]}`, {}, { + systemInstruction: { parts: [{ text: SYSTEM }] }, + contents: [{ role: "user", parts: [{ text: question }] }, ...priorTurns], + tools: [{ + functionDeclarations: [{ + name: arm.tool.name, + description: arm.tool.description, + parameters: toolFor(providerKey, arm).input_schema, + }], + }], + }); + if (!result.ok) return { error: `${result.status}: ${result.error}` }; + + const candidate = result.json.candidates?.[0]; + const parts = candidate?.content?.parts ?? []; + const call = parts.find((p) => p.functionCall)?.functionCall; + return { + input: call?.args, + text: parts.filter((p) => p.text).map((p) => p.text).join("\n"), + stopReason: candidate?.finishReason, + usage: { + input_tokens: result.json.usageMetadata?.promptTokenCount, + output_tokens: result.json.usageMetadata?.candidatesTokenCount, + }, + assistantMessage: candidate?.content, + }; +} + +/** One question, one arm, one provider. */ +export async function ask(providerKey, arm, question, priorTurns = []) { + const provider = PROVIDERS[providerKey]; + const blocked = incompatibility(providerKey, arm); + if (blocked) return { incompatible: blocked }; + return provider.family === "gemini" + ? callGemini(providerKey, provider, arm, question, priorTurns) + : callOpenAi(providerKey, provider, arm, question, priorTurns); +} + +/** Run tasks with a bounded number in flight — providers rate-limit on tokens/minute. */ +export async function pooled(tasks, limit, onDone) { + const results = new Array(tasks.length); + let next = 0; + let finished = 0; + await Promise.all( + Array.from({ length: Math.min(limit, tasks.length) }, async () => { + for (;;) { + const index = next++; + if (index >= tasks.length) return; + results[index] = await tasks[index](); + finished += 1; + onDone?.(finished, tasks.length); + } + }), + ); + return results; +} diff --git a/experiments/intent-to-filter/questions.mjs b/experiments/intent-to-filter/questions.mjs new file mode 100644 index 0000000..87ef3e5 --- /dev/null +++ b/experiments/intent-to-filter/questions.mjs @@ -0,0 +1,215 @@ +/** + * questions.mjs — the natural-language side of the corpus. + * + * `../filter-to-sql/cases.mjs` supplies the answer key: a gold filter and the + * row set SPEC.md says it returns, hand-derived from the fixture records. What + * it does not supply is an information need. Its `title` names the mechanism + * under test — "The §4.1 surprise: $not over a nullable field" — which, handed + * to a model as a prompt, is the answer. + * + * So each case gets a question here, and `cases.mjs` is not touched. + * + * THE AUTHORING RULE, enforced by gold.test.mjs: + * + * A question may name fields and values. It may not contain an operator, a + * syntax token, or a structural concept belonging to any arm. + * + * yes — "Which pets are available cats?" + * no — "pets where tags has some element in [indoor]" + * + * The rule is what keeps the corpus neutral between five arms that are given + * the same questions. Where a case cannot be phrased without breaking it, the + * case is excluded and the reason is recorded in EXCLUDED — published rather + * than quietly dropped, because the exclusion set is itself a finding about + * which parts of the language have no user-facing intent behind them. + * + * Two of the questions below carry a clarifying second sentence about missing + * or blank data ("include the ones we can't confirm"). That is not leakage: + * it is the distinction a user actually draws, and whether an interface can + * express it is the thing under test. Without it the question would be + * ambiguous and any grading of it dishonest. + * + * `probe` classes: + * retrieval a row set is expected + * edge a row set is expected, but the case turns on null, absence, + * or type semantics — reported separately from ordinary traffic + * unanswerable the field is not exposed, or the operator is outside the + * advertised profiles. Correct behaviour is to say so. + * Fabricating an answer is the failure. + */ + +/** Questions for cases that already exist in ../filter-to-sql/cases.mjs. */ +export const QUESTIONS = { + // -- A. What a search UI sends --------------------------------------------- + a01: { probe: "retrieval", ask: "Which pets are available for adoption right now?" }, + a02: { probe: "retrieval", ask: "Show me the cats that are available." }, + a03: { probe: "retrieval", ask: "List every pet that is either available or has an application pending." }, + a04: { probe: "retrieval", ask: "Which pets weigh between 1 and 5 kg? Count a pet at exactly 1 kg or exactly 5 kg as inside the range." }, + a05: { probe: "retrieval", ask: "Which pets were born on or after 1 January 2020?" }, + a06: { probe: "retrieval", ask: "Which pets have a name beginning with a lower-case b? A capital B is a different letter and should not match." }, + a07: { probe: "retrieval", ask: "Which pets have the letter o somewhere in their name? Ignore capitalisation." }, + a08: { probe: "retrieval", ask: "Which pets have the text 50% somewhere in their name?" }, + a09: { probe: "retrieval", ask: "Which pets carry at least one of the labels indoor or small?" }, + a10: { probe: "retrieval", ask: "Which pets carry both the trained label and the outdoor label?" }, + a11: { probe: "retrieval", ask: "Which pets have three or more labels?" }, + a12: { probe: "retrieval", ask: "Which pets have no labels at all?" }, + a13: { probe: "retrieval", ask: "Which pets are housed at a shelter in Athens?" }, + a14: { probe: "retrieval", ask: "Which pets are at a shelter with 12 kennels or fewer?" }, + a15: { probe: "retrieval", ask: "Find pets that are either available cats, or dogs weighing under 20 kg." }, + a16: { probe: "retrieval", ask: "Among the available pets, which are either at a shelter in Patras or carry the trained label?" }, + a17: { probe: "retrieval", ask: "Which pets are neither sold nor pending?" }, + a18: { probe: "retrieval", ask: "Which pets have been neutered?" }, + a19: { probe: "retrieval", ask: "Which pets are listed for more than they cost us?" }, + a20: { probe: "retrieval", ask: "Which pets have a status beginning with the letter a?" }, + + // -- B. Collections and nesting -------------------------------------------- + b01: { + probe: "retrieval", + ask: "Which pets have a rabies shot whose booster fell due before 1 January 2025? The overdue booster has to belong to the rabies shot itself, not to some other shot the pet had.", + }, + b02: { + probe: "retrieval", + ask: "Which pets have had a rabies shot at some point, and also have a shot — any shot at all — whose booster fell due before 1 January 2025? The two do not have to be the same shot.", + }, + b03: { probe: "retrieval", ask: "Which pets were vaccinated on or after 1 January 2025?" }, + b04: { probe: "retrieval", ask: "Which pets have parvo as the very first entry in their vaccination history?" }, + b05: { probe: "retrieval", ask: "Which pets have had a parvo vaccination?" }, + b06: { probe: "retrieval", ask: "Which pets do not carry the indoor label?" }, + b07: { probe: "retrieval", ask: "Which pets carry at least one label that is something other than indoor?" }, + b08: { probe: "retrieval", ask: "Which pets have indoor as the first label in their list?" }, + b09: { probe: "retrieval", ask: "Which pets have at least one vaccination with a booster date recorded?" }, + b10: { probe: "retrieval", ask: "Which pets have at least one vaccination with no booster date recorded?" }, + b11: { + probe: "edge", + ask: "Which pets have never been given anything other than rabies? A pet with no vaccinations on record counts as satisfying this.", + }, + b12: { + probe: "edge", + ask: "Which pets have not a single label that is indoor? A pet with an empty label list counts as satisfying this.", + }, + b13: { probe: "edge", ask: "Which pets have nothing but indoor labels, or no labels at all?" }, + + // -- C. Null, missing and negation ----------------------------------------- + c01: { probe: "edge", ask: "Which pets have a microchip field that is present but blank?" }, + c02: { probe: "edge", ask: "Which pets have no microchip field recorded at all?" }, + c03: { probe: "edge", ask: "Which pets have a notes field present, whatever it happens to contain?" }, + c04: { probe: "edge", ask: "Which pets are at a shelter whose kennel capacity has not been recorded?" }, + c05: { + probe: "edge", + ask: "Which pets can we confirm have a microchip other than CHIP-001? Leave out any pet whose chip we cannot confirm either way.", + }, + c07: { + probe: "edge", + ask: "Which pets either have a chip other than CHIP-001, or are on record as having no chip? Pets with no chip field at all should be left out.", + }, + c08: { probe: "retrieval", ask: "Which pets are not sold?" }, + c11: { probe: "retrieval", ask: "Which pets weigh outside the 1-to-5 kg range?" }, + c12: { + probe: "edge", + ask: "Which pets might have a microchip other than CHIP-001? Include every pet we cannot confirm is CHIP-001, whether the chip is blank, missing, or simply different.", + }, + c13: { + probe: "edge", + ask: "Which pets were not born on or after 1 January 2020? Include pets with no birth date on record, since we cannot confirm they were.", + }, + + // -- D. Types and coercion ------------------------------------------------- + d01: { probe: "edge", ask: "Which pets have a notes value that is a number?" }, + d02: { probe: "edge", ask: "Which pets have a notes value that is a whole number? Treat 3.0 as whole." }, + d03: { probe: "edge", ask: "Which pets have a notes value that is a list?" }, + d04: { probe: "edge", ask: "Which pets have a notes value that is explicitly blank, as opposed to missing?" }, + d05: { probe: "edge", ask: "Which pets have a notes value equal to the number 3?" }, + d06: { probe: "edge", ask: "Which pets have a notes value that is a number above 2?" }, + d07: { probe: "edge", ask: "Which pets have a notes value that is text sorting after the letter a?" }, + d08: { probe: "edge", ask: "Which pets can we confirm have a notes value that is not 3?" }, + d09: { probe: "edge", ask: "Which pets can we confirm have a notes value that is not above 2?" }, + + // -- E. Paths and patterns ------------------------------------------------- + e01: { probe: "retrieval", ask: "Which pets have a rate above 5? The field is literally named with a leading dollar sign." }, + e02: { probe: "retrieval", ask: "Which pets have a raw size of XS? The field is literally named size.raw, dot included — it is one field name, not a nested path." }, + e03: { probe: "retrieval", ask: "Which pets have a name that starts with C and ends with o?" }, + e04: { probe: "retrieval", ask: "Which pets have a name starting with A, B or C? Capitals only." }, + e05: { probe: "retrieval", ask: "Which pets have a name starting with a, b or c, in either case?" }, + e06: { probe: "retrieval", ask: "Which pets have a name ending in s?" }, + + // -- F. What has to be rejected -------------------------------------------- + f01: { probe: "unanswerable", ask: "Which pets have internal staff notes recorded as exactly the word anything?" }, + f02: { probe: "unanswerable", ask: "Run a free-text search across the pet records for ada and tell me which pets come back." }, +}; + +/** + * Questions written for this experiment rather than inherited, each aimed at + * one of the three valid-but-wrong mistakes README §"Exposing search to an + * agent" catalogues. They are the only questions in the corpus authored with a + * hypothesis in mind, so they are reported as their own block and never folded + * into the inherited-corpus numbers. + * + * `gold` is the filter a correct answer needs; `tempts` names the mistake the + * phrasing invites, so grade.mjs can report how often each one is actually made. + */ +export const MISTAKE_PROBES = { + m01: { + probe: "retrieval", + tempts: "in-as-membership", + ask: "Which pets carry the indoor label?", + gold: { tags: { $some: { $in: ["indoor"] } } }, + expect: ["p01", "p05", "p06", "p10"], + }, + m02: { + probe: "edge", + tempts: "ne-drops-nulls", + ask: "Which pets do not have microchip CHIP-001? Include the ones whose chip is blank or missing — I want everything that is not confirmed to be CHIP-001.", + gold: { microchip: { $ne: "CHIP-001", $unknownAs: true } }, + expect: ["p02", "p03", "p04", "p05", "p06", "p07", "p08", "p09", "p10"], + }, + m03: { + probe: "retrieval", + tempts: "value-outside-domain", + ask: "Which pets have the status Available?", + gold: { status: "available" }, + expect: ["p01", "p02", "p05", "p06", "p09", "p10"], + }, +}; + +/** + * Cases deliberately left out, and why. Two reasons recur, and they say + * different things about the language: + * + * "no user intent" — the case probes a rule that no one would ever ask for + * in words. Not a criticism: a specification has to + * pin down behaviour nobody requests. + * "not separable" — the case has a real intent, but one indistinguishable + * by row set from another case already in the corpus. + * Including both would double-count one question. + */ +export const EXCLUDED = { + b14: "no user intent — probes $every applied to a field that is not an array.", + c06: "not separable — same intent and same row set as c05; the pair differs only in spelling, which a row-set grader cannot see.", + c09: "not separable — the expected answer depends on the storage binding, not on the query.", + c10: "not separable — as c09.", + c14: "not separable — differs from c13 by one row, and only by where the resolution modifier nests. No English phrasing separates the two without naming the construct.", + f03: "no user intent — a malformed pattern escape is a defect in a generated query, not an information need.", + f04: "no user intent — nobody asks for a weight greater than the word 'heavy'.", + f05: "no user intent — a misspelled operator is a probe of the schema, not of intent.", + f06: "no user intent — nesting past the depth limit is a denial-of-service probe.", + f07: "no user intent — structural equality against an object operand.", + f08: "moved — the intent behind it ('which pets carry the indoor label') is m01, where the mistake it encodes is what gets measured rather than assumed.", + f09: "moved — as f08.", +}; + +/** Tokens that would hand a model one arm's syntax. Enforced by gold.test.mjs. */ +export const BANNED_TOKENS = [ + "$", "{", "}", "[", "]", + ">=", "<=", "!=", "==", + "operator", "regex", "wildcard", "schema", "json", "query language", + "quantifier", "quantify", "predicate", "clause", "boolean", +]; + +/** Questions may quote a data value that happens to look like syntax. */ +export const LEAK_EXEMPTIONS = { + a08: ["50%"], // the value under test is literally the string "50%" + e01: ["dollar sign"], // describes the data, not a syntax +}; + +/** Every question, inherited and authored, keyed by case id. */ +export const ALL = { ...QUESTIONS, ...MISTAKE_PROBES }; diff --git a/experiments/intent-to-filter/report.mjs b/experiments/intent-to-filter/report.mjs new file mode 100644 index 0000000..8c9db3b --- /dev/null +++ b/experiments/intent-to-filter/report.mjs @@ -0,0 +1,280 @@ +/** + * report.mjs — grade the recorded responses and print the tables. + * + * Every number in README.md comes from here, the way every number in + * ../filter-to-sql/README.md comes from its `--metrics`. Nothing is typed into + * the prose by hand, so the write-up cannot drift from the run. + * + * node experiments/intent-to-filter/report.mjs + * node experiments/intent-to-filter/report.mjs --file results/turn1.jsonl + */ + +import { readFileSync, existsSync, readdirSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { parseArgs } from "node:util"; + +import { openDatabase } from "../filter-to-sql/harness.mjs"; +import { openDifferential } from "./differential.mjs"; +import { CORPUS, BY_ID, EXCLUDED, isOrdinaryTraffic } from "./corpus.mjs"; +import { ARMS, BY_NAME, approxTokens } from "./arms/index.mjs"; +import { grade, wilson } from "./grade.mjs"; +import { PROVIDERS } from "./providers.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const RESULTS = join(here, "results"); + +const pct = (n) => `${(n * 100).toFixed(0)}%`; +const bar = (width = 78) => console.log("─".repeat(width)); + +// --------------------------------------------------------------------------- +// The part that runs with no results — the pre-registration report +// --------------------------------------------------------------------------- + +function corpusReport() { + console.log("\nCorpus\n"); + const groups = new Map(); + for (const c of CORPUS) groups.set(c.group, (groups.get(c.group) ?? 0) + 1); + for (const [group, n] of groups) console.log(` ${String(n).padStart(3)} ${group}`); + console.log(` ${String(CORPUS.length).padStart(3)} total asked`); + console.log(` ${String(Object.keys(EXCLUDED).length).padStart(3)} excluded, with reasons`); + + const probes = new Map(); + for (const c of CORPUS) probes.set(c.probe, (probes.get(c.probe) ?? 0) + 1); + console.log(""); + for (const [probe, n] of probes) console.log(` ${String(n).padStart(3)} ${probe}`); + console.log(`\n ${CORPUS.filter(isOrdinaryTraffic).length} of ${CORPUS.length} are ordinary traffic (groups A and B); the rest probe semantic edges.`); + console.log(` ${CORPUS.filter((c) => c.origin === "inherited").length} inherited from a corpus written before this experiment; ${CORPUS.filter((c) => c.origin === "authored").length} authored for it.`); + + console.log("\nWhat each arm can and cannot express\n"); + console.log(` ${"arm".padEnd(15)} ${"expressible".padStart(11)} ${"ceiling".padStart(7)} tool tokens`); + for (const arm of ARMS) { + let reachable = 0; + for (const c of CORPUS) if (arm.gold(c) !== null && arm.gold(c) !== undefined) reachable += 1; + const ceiling = CORPUS.length - reachable; + console.log( + ` ${arm.name.padEnd(15)} ${String(reachable).padStart(6)}/${CORPUS.length} ${String(ceiling).padStart(7)} ${String(approxTokens(JSON.stringify(arm.tool))).padStart(11)}`, + ); + } + console.log("\n `ceiling` counts questions the interface cannot put at all. They are"); + console.log(" excluded from that arm's accuracy denominator and reported here instead,"); + console.log(" so an arm is never marked wrong for a question it was never able to ask.\n"); +} + +// --------------------------------------------------------------------------- +// Grading a run +// --------------------------------------------------------------------------- + +function load(file) { + return readFileSync(file, "utf8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); +} + +function gradeAll(rows) { + const db = openDatabase(); + const differentialDb = openDifferential(); + const graded = []; + + for (const row of rows) { + const testCase = BY_ID[row.caseId]; + const arm = BY_NAME[row.arm]; + if (!testCase || !arm) continue; + + // The provider cannot carry this arm's tool definition at all. Recorded as + // its own outcome and kept out of every accuracy denominator: it says + // something about the provider, not about the model's competence. + if (row.incompatible) { + graded.push({ ...row, outcome: "provider-incompatible" }); + continue; + } + + // A question this arm cannot put is not scored against it either. + const gold = arm.gold(testCase); + if (gold === null || gold === undefined) { + graded.push({ ...row, outcome: "beyond-ceiling" }); + continue; + } + + if (row.error) { + // Some errors are not transport trouble, they are the provider saying it + // cannot carry this tool definition. Discovered at run time rather than + // predicted, so classified here rather than pre-declared. + const code = row.error.slice(0, 3); + const cannotCarry = + code === "413" ? "the flattened schema exceeds the provider's per-request input limit" + : /Tool call validation failed/.test(row.error) ? "the provider validates tool calls against the declared schema and cannot resolve its $refs" + : null; + graded.push(cannotCarry + ? { ...row, outcome: "provider-incompatible", incompatible: cannotCarry } + : { ...row, outcome: "api-error", note: row.error.slice(0, 80) }); + continue; + } + + const verdict = grade({ + testCase, arm, db, differentialDb, + input: row.input, text: row.text, stopReason: row.stopReason, + }); + + // If a repair turn was taken, grade it too — H3 is the difference. + let repaired; + if (row.repair && !row.repair.error) { + repaired = grade({ + testCase, arm, db, differentialDb, + input: row.repair.input, text: row.repair.text, stopReason: row.repair.stopReason, + }); + } + graded.push({ ...row, ...verdict, repaired: repaired?.outcome }); + } + return graded; +} + +function tally(graded, filter = () => true) { + const rows = graded + .filter(filter) + // Three things are not the model's answer and are kept out of every + // denominator: a question the arm cannot put, a tool the provider cannot + // carry, and a call that never completed. + .filter((r) => !["beyond-ceiling", "provider-incompatible", "api-error"].includes(r.outcome)); + const counts = {}; + for (const r of rows) counts[r.outcome] = (counts[r.outcome] ?? 0) + 1; + return { n: rows.length, counts }; +} + +function outcomeTable(graded, providers) { + for (const provider of providers) { + const failed = graded.filter((r) => r.provider === provider && r.outcome === "api-error").length; + console.log(`\n${PROVIDERS[provider]?.label ?? provider}${failed ? ` — ${failed} calls never completed and are excluded` : ""}\n`); + console.log(` ${"arm".padEnd(15)} ${"n".padStart(5)} ${"correct".padStart(9)} ${"loud".padStart(7)} ${"silent".padStart(8)} ${"loud:silent".padStart(12)}`); + for (const arm of ARMS) { + const { n, counts } = tally(graded, (r) => r.provider === provider && r.arm === arm.name); + if (n === 0) continue; + const correct = counts.correct ?? 0; + const loud = counts["loud-failure"] ?? 0; + const silent = counts["silent-failure"] ?? 0; + const ci = wilson(correct, n); + const ratio = silent === 0 ? (loud === 0 ? "—" : "all loud") : (loud / silent).toFixed(1); + console.log( + ` ${arm.name.padEnd(15)} ${String(n).padStart(5)} ${`${pct(correct / n)}`.padStart(5)} ±${String(Math.round(((ci.high - ci.low) / 2) * 100)).padStart(2)} ${pct(loud / n).padStart(7)} ${pct(silent / n).padStart(8)} ${ratio.padStart(12)}`, + ); + } + } +} + +function mistakeTable(graded) { + console.log("\nCatalogued mistakes, as a share of the attempts that made them\n"); + console.log(" Upper bounds: the detector reads shape, not intent, so a question that"); + console.log(" genuinely asked to exclude the unknowns counts as ne-drops-nulls too.\n"); + const kinds = ["value-outside-domain", "ne-drops-nulls", "in-as-membership"]; + console.log(` ${"arm".padEnd(15)} ${kinds.map((k) => k.padStart(21)).join("")}`); + for (const arm of ARMS) { + const rows = graded.filter((r) => r.arm === arm.name && Array.isArray(r.mistakes)); + if (rows.length === 0) continue; + const cells = kinds.map((kind) => { + const hits = rows.filter((r) => r.mistakes.includes(kind)).length; + return `${pct(hits / rows.length)} (${hits})`.padStart(21); + }); + console.log(` ${arm.name.padEnd(15)} ${cells.join("")}`); + } +} + +function probeTable(graded) { + console.log("\nThe three authored mistake probes\n"); + for (const id of ["m01", "m02", "m03"]) { + const testCase = BY_ID[id]; + console.log(` ${id} — ${testCase.tempts}: "${testCase.ask.slice(0, 62)}${testCase.ask.length > 62 ? "…" : ""}"`); + for (const arm of ARMS) { + const { n, counts } = tally(graded, (r) => r.caseId === id && r.arm === arm.name); + if (n === 0) continue; + console.log(` ${arm.name.padEnd(15)} correct ${String(counts.correct ?? 0).padStart(2)}/${n} loud ${String(counts["loud-failure"] ?? 0).padStart(2)} silent ${String(counts["silent-failure"] ?? 0).padStart(2)}`); + } + console.log(""); + } +} + +/** + * H3: "enough for an agent to repair its own request in one round trip". The + * claim appears five times in the repository and has never been measured. + */ +function repairTable(graded, providers) { + const attempted = graded.filter((r) => r.repaired !== undefined); + if (attempted.length === 0) return; + console.log("\nRepair after one round trip, over the attempts that were rejected\n"); + console.log(` ${"provider".padEnd(18)} ${"arm".padEnd(15)} ${"rejected".padStart(9)} ${"fixed".padStart(7)} ${"yield".padStart(7)}`); + for (const provider of providers) { + for (const arm of ARMS) { + const rows = attempted.filter((r) => r.provider === provider && r.arm === arm.name); + if (rows.length === 0) continue; + const fixed = rows.filter((r) => r.repaired === "correct").length; + console.log(` ${provider.padEnd(18)} ${arm.name.padEnd(15)} ${String(rows.length).padStart(9)} ${String(fixed).padStart(7)} ${pct(fixed / rows.length).padStart(7)}`); + } + } +} + +function costTable(graded) { + console.log("\nInput tokens per call — the price of the schema\n"); + console.log(` ${"arm".padEnd(15)} ${"tok/call".padStart(10)} ${"correct".padStart(10)} ${"tok/correct".padStart(12)}`); + for (const arm of ARMS) { + const rows = graded.filter((r) => r.arm === arm.name && r.usage?.input_tokens); + if (rows.length === 0) continue; + const perCall = rows.reduce((sum, r) => sum + r.usage.input_tokens, 0) / rows.length; + const correct = rows.filter((r) => r.outcome === "correct").length; + console.log( + ` ${arm.name.padEnd(15)} ${String(Math.round(perCall)).padStart(10)} ${`${correct}/${rows.length}`.padStart(10)} ${(correct ? Math.round((perCall * rows.length) / correct) : "—").toString().padStart(12)}`, + ); + } +} + +function incompatibilityNote(graded) { + const blocked = graded.filter((r) => r.outcome === "provider-incompatible"); + if (blocked.length === 0) return; + console.log("\nProvider incompatibility\n"); + const seen = new Map(); + for (const r of blocked) seen.set(`${r.provider}|${r.arm}`, r.incompatible); + for (const [pair, reason] of seen) { + const [provider, arm] = pair.split("|"); + console.log(` ${provider} x ${arm}: ${reason}`); + } + console.log(`\n ${blocked.length} cells were never called for this reason.`); +} + +// --------------------------------------------------------------------------- + +const { values } = parseArgs({ options: { file: { type: "string" } } }); + +const file = values.file + ? join(process.cwd(), values.file) + : existsSync(RESULTS) + ? readdirSync(RESULTS).filter((f) => f.endsWith(".jsonl")).map((f) => join(RESULTS, f)).sort().at(-1) + : undefined; + +bar(); +console.log("intent-to-filter"); +bar(); + +corpusReport(); + +if (!file) { + bar(); + console.log("\nNo results yet. The tables above are the pre-registration report:"); + console.log("what is being asked, of which arms, and what each arm is able to answer."); + console.log("\n node experiments/intent-to-filter/runner.mjs --dry-run cost estimate"); + console.log(" node experiments/intent-to-filter/runner.mjs --pilot 15 cases, one model"); + console.log(" node experiments/intent-to-filter/runner.mjs --run the full run\n"); + process.exit(0); +} + +const rows = load(file); +const graded = gradeAll(rows); +const providers = [...new Set(graded.map((r) => r.provider))]; + +bar(); +console.log(`\nResults — ${rows.length} responses from ${file.replace(process.cwd() + "/", "")}`); +outcomeTable(graded, providers); +repairTable(graded, providers); +mistakeTable(graded); +probeTable(graded); +costTable(graded); +incompatibilityNote(graded); +console.log(""); diff --git a/experiments/intent-to-filter/results/full.jsonl b/experiments/intent-to-filter/results/full.jsonl new file mode 100644 index 0000000..9592e3f --- /dev/null +++ b/experiments/intent-to-filter/results/full.jsonl @@ -0,0 +1,3419 @@ +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a02","trial":3,"input":{"filter":{"species":"cat","status":"available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a02","trial":1,"input":{"filter":{"species":"cat","status":"available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a02","trial":2,"input":{"filter":{"species":"cat","status":"available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a01","trial":5,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a01","trial":2,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":63}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a01","trial":4,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":112}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a02","trial":4,"input":{"filter":{"species":"cat","status":"available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a01","trial":3,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":115}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a01","trial":1,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":126}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a02","trial":5,"input":{"filter":{"species":"cat","status":"available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a03","trial":1,"input":{"filter":{"status":{"$in":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":85}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a03","trial":3,"input":{"filter":{"status":{"$in":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a04","trial":1,"input":{"filter":{"weightKg":{"$gte":1,"$lte":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":714,"output_tokens":34}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a04","trial":3,"input":{"filter":{"weightKg":{"$gte":1,"$lte":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":714,"output_tokens":34}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a04","trial":2,"input":{"filter":{"weightKg":{"$gte":1,"$lte":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":714,"output_tokens":34}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a03","trial":2,"input":{"filter":{"status":{"$in":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":130}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a03","trial":4,"input":{"filter":{"status":{"$in":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":103}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a04","trial":4,"input":{"filter":{"weightKg":{"$gte":1,"$lte":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":714,"output_tokens":34}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a04","trial":5,"input":{"filter":{"weightKg":{"$gte":1,"$lte":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":714,"output_tokens":34}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a05","trial":1,"input":{"filter":{"born":{"$gte":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":33}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a05","trial":3,"input":{"filter":{"born":{"$gte":"2020-01-01"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a05","trial":4,"input":{"filter":{"born":{"$gte":"2020-01-01"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a05","trial":2,"input":{"filter":{"born":{"$gte":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":62}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a03","trial":5,"input":{"filter":{"status":{"$in":["available","application pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":87},"repair":{"input":{"filter":{"status":{"$in":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":790,"output_tokens":80},"problemShown":{"type":"invalid-operand","detail":"/status/$in/1 must be one of: \"available\", \"pending\", \"sold\".","pointer":"/filter/status/$in/1"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a05","trial":5,"input":{"filter":{"born":{"$gte":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":70}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a06","trial":3,"input":{"filter":{"name":{"$regex":"^b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":116}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a06","trial":1,"input":{"filter":{"name":{"$regex":"^b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":164}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a06","trial":2,"input":{"filter":{"name":{"$regex":"^b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":130}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a06","trial":4,"input":{"filter":{"name":{"$regex":"^b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":174}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a08","trial":1,"input":{"filter":{"name":{"$contains":"50%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":122}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a07","trial":1,"input":{"filter":{"name":{"$regex":"o","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":118},"repair":{"input":{"filter":{"name":{"$regex":"(?i)o"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":790,"output_tokens":116},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a07","trial":2,"input":{"filter":{"name":{"$regex":"o","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":189},"repair":{"input":{"filter":{"$or":[{"name":{"$regex":"o"}},{"name":{"$regex":"O"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":790,"output_tokens":113},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a08","trial":2,"input":{"filter":{"name":{"$regex":"50%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":161}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a07","trial":4,"input":{"filter":{"name":{"$regex":"o","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":163},"repair":{"input":{"filter":{"name":{"$regex":"(?i)o"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":790,"output_tokens":114},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a09","trial":1,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a07","trial":3,"input":{"filter":{"name":{"$regex":"[oO]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":392}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a07","trial":5,"input":{"filter":{"name":{"$regex":"o","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":166},"repair":{"input":{"filter":{"name":{"$regex":"(?i)o"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":790,"output_tokens":142},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a08","trial":5,"input":{"filter":{"name":{"$contains":"50%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":149}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a06","trial":5,"input":{"filter":{"name":{"$gte":"b","$lt":"c"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":299},"repair":{"input":{"filter":{"name":{"$regex":"^b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":799,"output_tokens":221},"problemShown":{"type":"unsupported-operator","detail":"$gte is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$gte"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a09","trial":4,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a09","trial":2,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a09","trial":5,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a09","trial":3,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":64}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a08","trial":4,"input":{"filter":{"name":{"$regex":"50%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":259}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a08","trial":3,"input":{"filter":{"name":{"$contains":"50%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":330}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a10","trial":1,"input":{"filter":{"$and":[{"tags":{"$some":{"$eq":"trained"}}},{"tags":{"$some":{"$eq":"outdoor"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":697,"output_tokens":114}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a10","trial":3,"input":{"filter":{"$and":[{"tags":{"$some":{"$eq":"trained"}}},{"tags":{"$some":{"$eq":"outdoor"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":697,"output_tokens":122}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a10","trial":4,"input":{"filter":{"$and":[{"tags":{"$some":{"$eq":"trained"}}},{"tags":{"$some":{"$eq":"outdoor"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":697,"output_tokens":109}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a10","trial":2,"input":{"filter":{"$and":[{"tags":{"$some":{"$eq":"trained"}}},{"tags":{"$some":{"$eq":"outdoor"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":697,"output_tokens":124}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a10","trial":5,"input":{"filter":{"$and":[{"tags":{"$some":{"$eq":"trained"}}},{"tags":{"$some":{"$eq":"outdoor"}}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":697,"output_tokens":121}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a11","trial":4,"input":{"filter":{"tags":{"$size":{"$gte":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":207}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a12","trial":1,"text":"I can’t express “labels” with the available pet-search fields. The searchable field that seems closest is `tags`, but I shouldn’t assume that’s what you mean.","stopReason":"stop","usage":{"input_tokens":693,"output_tokens":165}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a11","trial":1,"text":"I can’t express “three or more labels” with the search tool: it has no queryable `labels` field or array length/count operator for tags.","stopReason":"stop","usage":{"input_tokens":693,"output_tokens":327}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a13","trial":1,"input":{"filter":{"shelter.city":"Athens"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":695,"output_tokens":52}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a11","trial":2,"input":{"filter":{"tags":{"$size":{"$gte":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":482}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a12","trial":3,"input":{"filter":{"tags":{"$eq":[]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":350}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a13","trial":2,"input":{"filter":{"shelter.city":"Athens"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":695,"output_tokens":52}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a13","trial":3,"input":{"filter":{"shelter.city":"Athens"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":695,"output_tokens":60}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a11","trial":5,"input":{"filter":{"tags":{"$length":{"$gte":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":315},"repair":{"text":"I can’t express “three or more labels” with the available pet search tool because it doesn’t support counting the number of labels/tags on a pet.","stopReason":"stop","usage":{"input_tokens":782,"output_tokens":118},"problemShown":{"type":"unsupported-operator","detail":"$length is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/tags/$length"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a11","trial":3,"input":{"filter":{"tags":{"$size":{"$gte":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":534}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a13","trial":4,"input":{"filter":{"shelter.city":"Athens"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":695,"output_tokens":60}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a13","trial":5,"input":{"filter":{"shelter.city":"Athens"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":695,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a14","trial":1,"input":{"filter":{"shelter.capacity":{"$lte":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a14","trial":2,"input":{"filter":{"shelter.capacity":{"$lte":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":58}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a14","trial":3,"input":{"filter":{"shelter.capacity":{"$lte":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":58}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a14","trial":5,"input":{"filter":{"shelter.capacity":{"$lte":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a14","trial":4,"input":{"filter":{"shelter.capacity":{"$lte":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a15","trial":2,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":46}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a15","trial":1,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":109}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a15","trial":3,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a15","trial":5,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":46}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a12","trial":5,"input":{"filter":{"tags":[]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":315},"repair":{"input":{"filter":{"tags":{"$eq":[]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":759,"output_tokens":234},"problemShown":{"type":"invalid-operand","detail":"/tags must be object.","pointer":"/filter/tags"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a15","trial":4,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a12","trial":4,"input":{"filter":{"tags":[]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":419},"repair":{"input":{"filter":{"tags":{"$eq":[]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":759,"output_tokens":149},"problemShown":{"type":"invalid-operand","detail":"/tags must be object.","pointer":"/filter/tags"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a16","trial":3,"input":{"filter":{"status":"available","$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$eq":"trained"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":705,"output_tokens":114}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a16","trial":2,"input":{"filter":{"status":"available","$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$eq":"trained"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":705,"output_tokens":124}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a16","trial":4,"input":{"filter":{"status":"available","$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$eq":"trained"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":705,"output_tokens":133}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a16","trial":1,"input":{"filter":{"$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$eq":"trained"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":705,"output_tokens":168}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a16","trial":5,"input":{"filter":{"$and":[{"status":"available"},{"$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$in":["trained"]}}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":705,"output_tokens":182}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a12","trial":2,"input":{"filter":{"tags":[]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":542},"repair":{"input":{"filter":{"tags":{"$not":{"$some":{}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":759,"output_tokens":433},"problemShown":{"type":"invalid-operand","detail":"/tags must be object.","pointer":"/filter/tags"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a17","trial":2,"input":{"filter":{"status":{"$nin":["sold","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":125}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a17","trial":1,"input":{"filter":{"$nor":[{"status":"sold"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":196}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a18","trial":2,"input":{"filter":{"neutered":true},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a18","trial":3,"input":{"filter":{"neutered":true},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a18","trial":4,"input":{"filter":{"neutered":true},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a18","trial":5,"input":{"filter":{"neutered":true},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a18","trial":1,"input":{"filter":{"neutered":true},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a17","trial":3,"input":{"filter":{"$nor":[{"status":"sold"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":265}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a17","trial":4,"input":{"filter":{"$nor":[{"status":"sold"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":293}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a17","trial":5,"input":{"filter":{"$nor":[{"status":"sold"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":211}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a19","trial":4,"text":"I can’t express “priceCents is greater than costCents” with the available pet search tool, because it doesn’t support comparing one field to another.","stopReason":"stop","usage":{"input_tokens":696,"output_tokens":315}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a19","trial":2,"text":"I can’t express “pets listed for more than they cost us” with the available search tool, because it requires comparing two fields (`priceCents > costCents`) within each pet record.","stopReason":"stop","usage":{"input_tokens":696,"output_tokens":279}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a19","trial":1,"text":"I can’t express “priceCents is greater than costCents” with the pet search tool as described, because it doesn’t support comparing one field to another.","stopReason":"stop","usage":{"input_tokens":696,"output_tokens":326}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a19","trial":5,"text":"I can’t express this with the available pet-search tool, because it doesn’t support comparing one field to another field, such as `priceCents > costCents`.","stopReason":"stop","usage":{"input_tokens":696,"output_tokens":336}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a19","trial":3,"text":"I can’t express “priceCents is greater than costCents” with the available pet-search tool, because it doesn’t support comparing one field to another.","stopReason":"stop","usage":{"input_tokens":696,"output_tokens":341}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a20","trial":1,"input":{"filter":{"status":{"$regex":"^a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":115},"repair":{"text":"I can’t express “status beginning with the letter a” with the available pet-search tool, because it doesn’t support prefix/regex matching on `status`.","stopReason":"stop","usage":{"input_tokens":782,"output_tokens":167},"problemShown":{"type":"unsupported-operator","detail":"$regex is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/status/$regex"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b01","trial":2,"input":{"filter":{"vaccinations":{"$some":{"vaccine":"rabies","boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":726,"output_tokens":108}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a20","trial":2,"input":{"filter":{"status":{"$regex":"^a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":348},"repair":{"text":"I can’t express “status beginning with the letter a” with the available pet-search tool, because it doesn’t support prefix/regex matching for `status`.","stopReason":"stop","usage":{"input_tokens":782,"output_tokens":145},"problemShown":{"type":"unsupported-operator","detail":"$regex is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/status/$regex"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b01","trial":4,"input":{"filter":{"vaccinations":{"$some":{"vaccine":"rabies","boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":726,"output_tokens":227}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a20","trial":3,"input":{"filter":{"status":{"$regex":"^a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":181},"repair":{"text":"I can’t express “status beginning with the letter a” with this search tool, because it doesn’t support prefix or regex matching on `status`.","stopReason":"stop","usage":{"input_tokens":782,"output_tokens":312},"problemShown":{"type":"unsupported-operator","detail":"$regex is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/status/$regex"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b01","trial":5,"input":{"filter":{"vaccinations":{"$some":{"name":"rabies","boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":726,"output_tokens":121},"repair":{"input":{"filter":{"vaccinations":{"$some":{"vaccine":"rabies","boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":825,"output_tokens":295},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b02","trial":2,"input":{"filter":{"$and":[{"vaccinations":{"$some":{"vaccine":"rabies"}}},{"vaccinations":{"$some":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":731,"output_tokens":331}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b01","trial":1,"input":{"filter":{"vaccinations":{"$some":{"$and":[{"name":"rabies"},{"boosterDue":{"$lt":"2025-01-01"}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":726,"output_tokens":215},"repair":{"input":{"filter":{"vaccinations":{"$some":{"$and":[{"vaccine":"rabies"},{"boosterDue":{"$lt":"2025-01-01"}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":834,"output_tokens":529},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/$and/0/name"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a20","trial":5,"input":{"filter":{"status":{"$regex":"^a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":547},"repair":{"text":"I can’t express “status beginning with the letter a” with the available pet search operators, because the tool does not support prefix/regex matching for `status`.","stopReason":"stop","usage":{"input_tokens":782,"output_tokens":151},"problemShown":{"type":"unsupported-operator","detail":"$regex is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/status/$regex"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"a20","trial":4,"input":{"filter":{"status":{"$regex":"^a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":415},"repair":{"text":"I can’t express “status beginning with the letter a” with the available pet search tool, because prefix/regex matching on `status` isn’t supported.","stopReason":"stop","usage":{"input_tokens":782,"output_tokens":350},"problemShown":{"type":"unsupported-operator","detail":"$regex is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/status/$regex"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b02","trial":1,"input":{"filter":{"$and":[{"vaccinations":{"$some":{"name":"rabies"}}},{"vaccinations":{"$some":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":731,"output_tokens":260},"repair":{"input":{"filter":{"$and":[{"vaccinations":{"$some":{"vaccine":"rabies"}}},{"vaccinations":{"$some":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":846,"output_tokens":232},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/$and/0/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b01","trial":3,"input":{"filter":{"vaccinations":{"$some":{"name":"rabies","boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":726,"output_tokens":403},"repair":{"input":{"filter":{"vaccinations":{"$exists":true}},"limit":5},"text":"","stopReason":"tool_calls","usage":{"input_tokens":825,"output_tokens":264},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b02","trial":3,"input":{"filter":{"$and":[{"vaccinations":{"$some":{"vaccine":"rabies"}}},{"vaccinations":{"$some":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":731,"output_tokens":428}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b03","trial":1,"input":{"filter":{"vaccinations":{"$some":{"date":{"$gte":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":134},"repair":{"text":"I can’t express that search with the available pet-store tool because vaccination dates are not a queryable field.","stopReason":"stop","usage":{"input_tokens":791,"output_tokens":130},"problemShown":{"type":"unknown-field","detail":"\"date\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/date"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b03","trial":2,"input":{"filter":{"vaccinations":{"$some":{"date":{"$gte":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":155},"repair":{"input":{"filter":{"vaccinations":{"$some":{"$gte":"2025-01-01"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":791,"output_tokens":96},"problemShown":{"type":"unknown-field","detail":"\"date\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/date"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b03","trial":3,"input":{"filter":{"vaccinations":{"$some":{"date":{"$gte":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":135},"repair":{"text":"I can’t express that search with the available pet-store tool: vaccination dates are not a queryable field.","stopReason":"stop","usage":{"input_tokens":791,"output_tokens":133},"problemShown":{"type":"unknown-field","detail":"\"date\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/date"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b03","trial":4,"input":{"filter":{"vaccinations":{"$some":{"date":{"$gte":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":146},"repair":{"text":"I can’t express “vaccinated on or after 1 January 2025” with the available pet search tool, because vaccination dates are not a queryable field.","stopReason":"stop","usage":{"input_tokens":791,"output_tokens":183},"problemShown":{"type":"unknown-field","detail":"\"date\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/date"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b02","trial":5,"input":{"filter":{"$and":[{"vaccinations":{"$some":{"name":"rabies"}}},{"vaccinations":{"$some":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":731,"output_tokens":329},"repair":{"input":{"filter":{"$and":[{"vaccinations":{"$some":{"vaccine":"rabies"}}},{"vaccinations":{"$some":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":846,"output_tokens":245},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/$and/0/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b03","trial":5,"input":{"filter":{"vaccinations":{"$some":{"date":{"$gte":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":188},"repair":{"text":"I can’t answer that with the available pet-search tool: vaccination dates are not a queryable field, so the request “vaccinated on or after 1 January 2025” can’t be expressed.","stopReason":"stop","usage":{"input_tokens":791,"output_tokens":243},"problemShown":{"type":"unknown-field","detail":"\"date\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/date"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b04","trial":2,"text":"I can’t express “parvo as the very first entry in vaccination history” with the available pet search tool, because it doesn’t describe any operator for checking the first/positioned element of the `vaccinations` array.","stopReason":"stop","usage":{"input_tokens":700,"output_tokens":298}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b04","trial":4,"text":"I can’t express “parvo as the very first entry in vaccination history” with the available pet-search tool. It supports filtering array contents, but not specifically the first/ordered entry, so I shouldn’t approximate it with “has parvo anywhere.”","stopReason":"stop","usage":{"input_tokens":700,"output_tokens":353}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b04","trial":3,"text":"I can’t express “parvo as the very first entry” with the available pet-search tool, because it doesn’t support filtering by array position/order in vaccination history.","stopReason":"stop","usage":{"input_tokens":700,"output_tokens":347}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b05","trial":1,"input":{"filter":{"vaccinations":{"$some":{"name":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":156},"repair":{"input":{"filter":{"vaccinations":{"$some":"parvo"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":778,"output_tokens":97},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b04","trial":5,"text":"I can’t express “parvo as the very first entry” with the available pet-search tool, because it doesn’t support filtering by array position/order in vaccination history.","stopReason":"stop","usage":{"input_tokens":700,"output_tokens":370}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b02","trial":4,"input":{"filter":{"$and":[{"vaccinations":{"$some":{"name":"rabies"}}},{"vaccinations":{"$some":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":731,"output_tokens":359},"repair":{"input":{"filter":{"vaccinations":{"$some":{"vaccine":"rabies"}}},"limit":1},"text":"","stopReason":"tool_calls","usage":{"input_tokens":846,"output_tokens":551},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/$and/0/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b04","trial":1,"input":{"filter":{"vaccinations":{"$first":{"name":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":550},"repair":{"text":"I’m sorry, but this search can’t be expressed with the available pet-search tool: it does not support querying “the very first entry” of the `vaccinations` array.","stopReason":"stop","usage":{"input_tokens":791,"output_tokens":129},"problemShown":{"type":"unsupported-operator","detail":"$first is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$first"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b05","trial":3,"input":{"filter":{"vaccinations":{"$some":{"name":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":91},"repair":{"input":{"filter":{"vaccinations":{"$some":"parvo"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":778,"output_tokens":139},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b06","trial":1,"input":{"filter":{"$not":{"tags":{"$some":{"$in":["indoor"]}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":203}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b05","trial":2,"input":{"filter":{"vaccinations":{"$some":{"name":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":357},"repair":{"input":{"filter":{"vaccinations":{"$some":"parvo"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":778,"output_tokens":136},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b06","trial":2,"input":{"filter":{"$not":{"tags":{"$some":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":344}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b07","trial":1,"input":{"filter":{"tags":{"$some":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":123}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b07","trial":2,"input":{"filter":{"tags":{"$some":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":122}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b05","trial":4,"input":{"filter":{"vaccinations":{"$some":{"name":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":326},"repair":{"input":{"filter":{"vaccinations":{"$some":{"$eq":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":778,"output_tokens":113},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b06","trial":4,"input":{"filter":{"$not":{"tags":{"$some":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":303}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b06","trial":3,"input":{"filter":{"$not":{"tags":{"$some":{"$in":["indoor"]}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":422}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b07","trial":3,"input":{"filter":{"tags":{"$some":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":110}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b07","trial":4,"input":{"filter":{"tags":{"$some":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":99}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b06","trial":5,"input":{"filter":{"$not":{"tags":{"$some":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":378}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b07","trial":5,"input":{"filter":{"tags":{"$some":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":106}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b05","trial":5,"input":{"filter":{"vaccinations":{"$some":{"name":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":241},"repair":{"input":{"filter":{"vaccinations":{"$some":{"$eq":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":778,"output_tokens":324},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b08","trial":4,"text":"I can’t express “indoor as the first label in their list” with the available pet-search tool, because it doesn’t support filtering by array position.","stopReason":"stop","usage":{"input_tokens":697,"output_tokens":229}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b08","trial":2,"text":"I can’t express “indoor as the first label” with the available pet search tool. It supports filtering pets by fields like `tags`, but not by the position of an item within that list.","stopReason":"stop","usage":{"input_tokens":697,"output_tokens":320}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b08","trial":3,"text":"I can’t express “indoor as the first label” with the available pet search tool, because it supports checking array elements but not their position/order.","stopReason":"stop","usage":{"input_tokens":697,"output_tokens":292}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b08","trial":1,"text":"I can’t express “indoor as the first label” with the available pet search tool. It can filter whether a label appears in the list, but not by its position in the list.","stopReason":"stop","usage":{"input_tokens":697,"output_tokens":365}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b08","trial":5,"text":"I can’t express “indoor as the first label in their list” with the available pet search filter. It can search array contents, but not a label’s position/order in the list.","stopReason":"stop","usage":{"input_tokens":697,"output_tokens":413}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b09","trial":3,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":{"$ne":null}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":253},"repair":{"text":"I can’t express that search with the available pet-store tool because vaccination booster dates are not a queryable field.","stopReason":"stop","usage":{"input_tokens":792,"output_tokens":105},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b09","trial":2,"input":{"filter":{"vaccinations":{"$some":{"boosterDue":{"$ne":null}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":243},"repair":{"input":{"filter":{"vaccinations":{"$some":{"boosterDue":{"$gt":""}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":795,"output_tokens":221},"problemShown":{"type":"invalid-operand","detail":"/vaccinations/$some/boosterDue/$ne must be string.","pointer":"/filter/vaccinations/$some/boosterDue/$ne"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b09","trial":1,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":{"$ne":null}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":395},"repair":{"text":"I can’t express that search with the available pet-store tool because the vaccination booster date field isn’t queryable.","stopReason":"stop","usage":{"input_tokens":792,"output_tokens":72},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b09","trial":5,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":{"$ne":null}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":369},"repair":{"text":"I’m sorry, but that question can’t be expressed with the available pet search tool because vaccination `boosterDate` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":792,"output_tokens":110},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b10","trial":1,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":{"$eq":null,"$unknownAs":true}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":365},"repair":{"text":"I can’t express that search with the available pet-store tool, because vaccination booster-date fields are not queryable.","stopReason":"stop","usage":{"input_tokens":798,"output_tokens":142},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b10","trial":3,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":null}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":281},"repair":{"text":"I can’t express that search with the available pet-store tool because `boosterDate` inside `vaccinations` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":788,"output_tokens":114},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b10","trial":2,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":{"$eq":null,"$unknownAs":true}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":411},"repair":{"text":"I can’t express that search with the available pet-store tool because `vaccinations.boosterDate` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":798,"output_tokens":142},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b09","trial":4,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":{"$ne":null}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":555},"repair":{"text":"I can’t express that search with the available pet-store tool: vaccination booster date is not a queryable field.","stopReason":"stop","usage":{"input_tokens":792,"output_tokens":129},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b10","trial":4,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":null}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":460},"repair":{"text":"I can’t express that query with the available pet-search tool because `boosterDate` inside `vaccinations` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":788,"output_tokens":131},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b12","trial":1,"input":{"filter":{"$not":{"tags":{"$some":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":708,"output_tokens":310}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b10","trial":5,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":{"$eq":null,"$unknownAs":true}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":560},"repair":{"text":"I can’t express that search with the available pet-store tool: the vaccination booster date is not a queryable field.","stopReason":"stop","usage":{"input_tokens":798,"output_tokens":150},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b12","trial":2,"input":{"filter":{"$not":{"tags":{"$some":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":708,"output_tokens":288}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b11","trial":1,"input":{"filter":{"vaccinations":{"$not":{"$some":{"$ne":"rabies"}},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":561},"repair":{"input":{"filter":{"$or":[{"vaccinations":{"$not":{"$some":{"$nin":["rabies"]}}}},{"vaccinations":null}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":815,"output_tokens":182},"problemShown":{"type":"unsupported-operator","detail":"$ne is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$not/$some/$ne"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b12","trial":4,"input":{"filter":{"tags":{"$not":{"$some":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":708,"output_tokens":230}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b12","trial":5,"input":{"filter":{"$not":{"tags":{"$some":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":708,"output_tokens":209}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b12","trial":3,"input":{"filter":{"$not":{"tags":{"$some":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":708,"output_tokens":283}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b11","trial":4,"input":{"filter":{"$not":{"vaccinations":{"$some":{"$ne":"rabies"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":555},"repair":{"input":{"filter":{"$not":{"vaccinations":{"$some":{"$in":["rabies"]}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":809,"output_tokens":199},"problemShown":{"type":"unsupported-operator","detail":"$ne is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$not/vaccinations/$some/$ne"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b11","trial":5,"input":{"filter":{"vaccinations":{"$not":{"$some":{"name":{"$ne":"rabies"}}},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":563},"repair":{"input":{"filter":{"vaccinations":{"$not":{"$some":{"$ne":"rabies"}},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":808,"output_tokens":194},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$not/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b11","trial":3,"input":{"filter":{"vaccinations":{"$not":{"$some":{"$ne":"rabies"}},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":561},"repair":{"input":{"filter":{"$nor":[{"vaccinations":{"$some":{"$not":{"$eq":"rabies"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":815,"output_tokens":290},"problemShown":{"type":"unsupported-operator","detail":"$ne is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$not/$some/$ne"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b11","trial":2,"input":{"filter":{"$or":[{"vaccinations":{"$eq":null}},{"$not":{"vaccinations":{"$some":{"$ne":"rabies"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":569},"repair":{"input":{"filter":{"$not":{"vaccinations":{"$some":{"$not":{"$eq":"rabies"}}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":827,"output_tokens":278},"problemShown":{"type":"unsupported-operator","detail":"$ne is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$or/1/$not/vaccinations/$some/$ne"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c01","trial":1,"input":{"filter":{"microchip":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":81}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c01","trial":2,"input":{"filter":{"microchip":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":114}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c01","trial":3,"input":{"filter":{"microchip":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c01","trial":5,"input":{"filter":{"microchip":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":59}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c01","trial":4,"input":{"filter":{"microchip":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":94}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c02","trial":1,"input":{"filter":{"microchip":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":152}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c02","trial":2,"input":{"filter":{"microchip":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":124}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c02","trial":3,"input":{"filter":{"microchip":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":140}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c02","trial":4,"input":{"filter":{"microchip":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":149}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b13","trial":5,"input":{"filter":{"tags":{"$not":{"$some":{"$ne":"indoor"}},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":560}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c02","trial":5,"input":{"filter":{"microchip":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":145}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b13","trial":4,"input":{"filter":{"$or":[{"tags":{"$every":{"$eq":"indoor"}}},{"tags":{"$exists":false}},{"tags":null}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":568},"repair":{"input":{"filter":{"$or":[{"tags":{"$every":{"$eq":"indoor"}}},{"tags":{"$exists":false}},{"tags":{"$eq":null}},{"tags":{"$size":{"$eq":0}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":800,"output_tokens":280},"problemShown":{"type":"invalid-operand","detail":"/$or/2/tags must be object.","pointer":"/filter/$or/2/tags"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c03","trial":1,"input":{"filter":{"notes":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":148}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c03","trial":3,"input":{"filter":{"notes":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":87}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c03","trial":2,"input":{"filter":{"notes":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":121}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c03","trial":4,"input":{"filter":{"notes":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c03","trial":5,"input":{"filter":{"notes":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":117}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b13","trial":1,"input":{"filter":{"$or":[{"tags":{"$every":{"$eq":"indoor"}}},{"tags":{"$eq":[]}},{"tags":{"$eq":null}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":570},"repair":{"input":{"filter":{"$or":[{"tags":{"$every":{"$eq":"indoor"}}},{"tags":{"$eq":[]}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":806,"output_tokens":458},"problemShown":{"type":"invalid-operand","detail":"/$or/2/tags/$eq must be array.","pointer":"/filter/$or/2/tags/$eq"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b13","trial":3,"input":{"filter":{"$or":[{"tags":{"$every":{"$eq":"indoor"}}},{"tags":{"$exists":false}},{"tags":null}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":568},"repair":{"input":{"filter":{"$or":[{"tags":{"$every":{"$eq":"indoor"}}},{"tags":{"$exists":false}},{"tags":{"$eq":null}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":800,"output_tokens":390},"problemShown":{"type":"invalid-operand","detail":"/$or/2/tags must be object.","pointer":"/filter/$or/2/tags"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"b13","trial":2,"input":{"filter":{"$or":[{"tags":{"$every":{"$eq":"indoor"}}},{"tags":{"$size":0}},{"tags":{"$exists":false}},{"tags":null}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":576},"repair":{"input":{"filter":{"tags":{"$every":{"$eq":"indoor"},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":808,"output_tokens":443},"problemShown":{"type":"invalid-operand","detail":"/$or/3/tags must be object.","pointer":"/filter/$or/3/tags"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c05","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":712,"output_tokens":132}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c05","trial":3,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":712,"output_tokens":130}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c05","trial":4,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":712,"output_tokens":129}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c05","trial":2,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":712,"output_tokens":201}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c05","trial":5,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":712,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c04","trial":1,"input":{"filter":{"shelter.capacity":{"$eq":null,"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":354},"repair":{"input":{"filter":{"shelter.capacity":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":787,"output_tokens":170},"problemShown":{"type":"invalid-operand","detail":"/shelter.capacity/$eq must be integer.","pointer":"/filter/shelter.capacity/$eq"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c04","trial":2,"input":{"filter":{"shelter.capacity":{"$eq":null,"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":331},"repair":{"input":{"filter":{"shelter.capacity":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":787,"output_tokens":286},"problemShown":{"type":"invalid-operand","detail":"/shelter.capacity/$eq must be integer.","pointer":"/filter/shelter.capacity/$eq"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c07","trial":1,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":null}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":718,"output_tokens":194}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c07","trial":2,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":null}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":718,"output_tokens":254}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c04","trial":3,"input":{"filter":{"shelter.capacity":{"$eq":null,"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":284},"repair":{"input":{"filter":{"shelter.capacity":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":787,"output_tokens":319},"problemShown":{"type":"invalid-operand","detail":"/shelter.capacity/$eq must be integer.","pointer":"/filter/shelter.capacity/$eq"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c04","trial":4,"input":{"filter":{"shelter.capacity":{"$eq":null,"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":483},"repair":{"input":{"filter":{"shelter.capacity":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":787,"output_tokens":130},"problemShown":{"type":"invalid-operand","detail":"/shelter.capacity/$eq must be integer.","pointer":"/filter/shelter.capacity/$eq"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c04","trial":5,"input":{"filter":{"shelter.capacity":{"$eq":null,"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":479},"repair":{"input":{"filter":{"shelter.capacity":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":787,"output_tokens":198},"problemShown":{"type":"invalid-operand","detail":"/shelter.capacity/$eq must be integer.","pointer":"/filter/shelter.capacity/$eq"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c08","trial":2,"input":{"filter":{"status":{"$ne":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":691,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c08","trial":1,"input":{"filter":{"status":{"$ne":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":691,"output_tokens":110}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c08","trial":3,"input":{"filter":{"status":{"$ne":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":691,"output_tokens":76}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c07","trial":3,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null,"$exists":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":718,"output_tokens":356}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c08","trial":4,"input":{"filter":{"status":{"$ne":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":691,"output_tokens":99}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c11","trial":1,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":114}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c11","trial":4,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":105}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c11","trial":3,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":124}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c08","trial":5,"input":{"filter":{"status":{"$ne":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":691,"output_tokens":82}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c11","trial":2,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":117}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c11","trial":5,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":99}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c12","trial":2,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":721,"output_tokens":116}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c12","trial":4,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":721,"output_tokens":131}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c07","trial":4,"input":{"filter":{"$and":[{"microchip":{"$exists":true}},{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":718,"output_tokens":578}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c12","trial":3,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":721,"output_tokens":171}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c12","trial":5,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":721,"output_tokens":128}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c12","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":721,"output_tokens":166}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c07","trial":5,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":718,"output_tokens":564}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d01","trial":1,"input":{"filter":{"notes":{"$type":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":106}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d01","trial":3,"input":{"filter":{"notes":{"$type":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":122}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d01","trial":2,"input":{"filter":{"notes":{"$type":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":153}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c13","trial":5,"input":{"filter":{"born":{"$lt":"2020-01-01","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":716,"output_tokens":194}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c13","trial":3,"input":{"filter":{"born":{"$lt":"2020-01-01","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":716,"output_tokens":235}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c13","trial":1,"input":{"filter":{"born":{"$lt":"2020-01-01","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":716,"output_tokens":384}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c13","trial":4,"input":{"filter":{"born":{"$lt":"2020-01-01","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":716,"output_tokens":351}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d01","trial":4,"input":{"filter":{"notes":{"$type":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":214}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"c13","trial":2,"input":{"filter":{"born":{"$lt":"2020-01-01","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":716,"output_tokens":531}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d02","trial":1,"text":"I can’t express “notes value is a whole number, treating 3.0 as whole” with the available pet-search filter operators.","stopReason":"stop","usage":{"input_tokens":705,"output_tokens":305}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d01","trial":5,"input":{"filter":{"notes":{"$type":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":340}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d03","trial":1,"input":{"filter":{"notes":{"$type":"array"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":183}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d02","trial":2,"input":{"filter":{"notes":{"$type":"integer"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":705,"output_tokens":475}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d03","trial":2,"input":{"filter":{"notes":{"$type":"array"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":257}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d03","trial":3,"input":{"filter":{"notes":{"$type":"array"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":157}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d02","trial":3,"text":"I can’t express “notes value is a whole number, treating 3.0 as whole” with the available pet-search filter operators.","stopReason":"stop","usage":{"input_tokens":705,"output_tokens":549}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d03","trial":5,"input":{"filter":{"notes":{"$type":"array"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":696,"output_tokens":167}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d02","trial":5,"text":"I can’t express “notes is a whole number, treating 3.0 as whole” with the available pet-search filter operators as described.","stopReason":"stop","usage":{"input_tokens":705,"output_tokens":550}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d03","trial":4,"text":"I can’t express “notes value is a list/array” with the available pet search filter operators.","stopReason":"stop","usage":{"input_tokens":696,"output_tokens":313}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d04","trial":1,"input":{"filter":{"notes":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":67},"repair":{"input":{"filter":{"notes":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":767,"output_tokens":62},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d04","trial":2,"input":{"filter":{"notes":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":84},"repair":{"input":{"filter":{"notes":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":767,"output_tokens":71},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d04","trial":3,"input":{"filter":{"notes":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":85},"repair":{"input":{"filter":{"notes":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":767,"output_tokens":68},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d02","trial":4,"input":{"filter":{"notes":{"$type":"number","$mod":[1,0]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":705,"output_tokens":386},"repair":{"input":{"filter":{"notes":{"$type":"integer"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":799,"output_tokens":355},"problemShown":{"type":"unsupported-operator","detail":"$mod is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/notes/$mod"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d05","trial":2,"input":{"filter":{"notes":3},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":56},"repair":{"input":{"filter":{"notes":{"$eq":3}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":764,"output_tokens":78},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d05","trial":1,"input":{"filter":{"notes":3},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":55},"repair":{"input":{"filter":{"notes":{"$eq":3}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":764,"output_tokens":101},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d04","trial":5,"input":{"filter":{"notes":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":99},"repair":{"input":{"filter":{"notes":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":767,"output_tokens":75},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d06","trial":1,"input":{"filter":{"notes":{"$gt":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":102}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d05","trial":3,"input":{"filter":{"notes":3},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":55},"repair":{"input":{"filter":{"notes":{"$eq":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":764,"output_tokens":68},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d05","trial":5,"input":{"filter":{"notes":3},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":50},"repair":{"input":{"filter":{"notes":{"$eq":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":764,"output_tokens":91},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d05","trial":4,"input":{"filter":{"notes":3},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":55},"repair":{"input":{"filter":{"notes":{"$eq":3}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":764,"output_tokens":95},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d04","trial":4,"input":{"filter":{"notes":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":180},"repair":{"input":{"filter":{"notes":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":767,"output_tokens":79},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d06","trial":3,"input":{"filter":{"notes":{"$gt":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":94}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d06","trial":2,"input":{"filter":{"notes":{"$gt":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d06","trial":5,"input":{"filter":{"notes":{"$gt":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":99}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d07","trial":1,"input":{"filter":{"notes":{"$gt":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":93}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d06","trial":4,"input":{"filter":{"notes":{"$gt":2}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":99}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d07","trial":3,"input":{"filter":{"notes":{"$gt":"a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d07","trial":2,"input":{"filter":{"notes":{"$gt":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":92}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d07","trial":5,"input":{"filter":{"notes":{"$gt":"a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":88}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d07","trial":4,"input":{"filter":{"notes":{"$gt":"a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":110}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d08","trial":2,"input":{"filter":{"notes":{"$ne":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":86}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d08","trial":1,"input":{"filter":{"notes":{"$ne":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":68}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d08","trial":3,"input":{"filter":{"notes":{"$ne":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d08","trial":4,"input":{"filter":{"notes":{"$ne":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":99}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d08","trial":5,"input":{"filter":{"notes":{"$ne":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":700,"output_tokens":103}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d09","trial":1,"input":{"filter":{"notes":{"$lte":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":116}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d09","trial":4,"input":{"filter":{"notes":{"$lte":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":108}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e01","trial":1,"input":{"filter":{"$$rate":{"$gt":5}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":705,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d09","trial":3,"input":{"filter":{"notes":{"$lte":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":133}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d09","trial":5,"input":{"filter":{"notes":{"$lte":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":139}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e01","trial":2,"input":{"filter":{"$$rate":{"$gt":5}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":705,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e01","trial":3,"input":{"filter":{"$$rate":{"$gt":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":705,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e01","trial":4,"input":{"filter":{"$$rate":{"$gt":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":705,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"d09","trial":2,"input":{"filter":{"notes":{"$lte":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":186}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e02","trial":3,"input":{"filter":{"size\\.raw":"XS"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":716,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e02","trial":1,"input":{"filter":{"size\\.raw":"XS"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":716,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e02","trial":2,"input":{"filter":{"size\\.raw":"XS"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":716,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e02","trial":4,"input":{"filter":{"size\\.raw":"XS"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":716,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e02","trial":5,"input":{"filter":{"size\\.raw":"XS"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":716,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e01","trial":5,"input":{"filter":{"$$rate":{"$gt":5}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":705,"output_tokens":85}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e03","trial":5,"input":{"filter":{"name":{"$regex":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":99}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e03","trial":4,"input":{"filter":{"name":{"$regex":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":114}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e03","trial":2,"input":{"filter":{"name":{"$regex":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":120}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e04","trial":1,"input":{"filter":{"name":{"$regex":"^[ABC]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":116}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e03","trial":1,"input":{"filter":{"name":{"$regex":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":165}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e03","trial":3,"input":{"filter":{"name":{"$regex":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":112}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e04","trial":2,"input":{"filter":{"name":{"$regex":"^[ABC]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":107}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e04","trial":3,"input":{"filter":{"name":{"$regex":"^[ABC]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e04","trial":4,"input":{"filter":{"name":{"$regex":"^[ABC]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":160}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e06","trial":1,"input":{"filter":{"name":{"$regex":"s$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":106}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e04","trial":5,"input":{"filter":{"name":{"$regex":"^[ABC]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":230}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e05","trial":2,"input":{"filter":{"name":{"$regex":"^[abc]","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":702,"output_tokens":167},"repair":{"input":{"filter":{"name":{"$regex":"(?i)^[abc]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":794,"output_tokens":80},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e05","trial":3,"input":{"filter":{"name":{"$regex":"^[abc]","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":702,"output_tokens":137},"repair":{"input":{"filter":{"name":{"$regex":"^[AaBbCc]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":794,"output_tokens":78},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e06","trial":2,"input":{"filter":{"name":{"$regex":"s$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":104}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e05","trial":4,"input":{"filter":{"name":{"$regex":"^[abc]","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":702,"output_tokens":140},"repair":{"input":{"filter":{"name":{"$regex":"^[AaBbCc]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":794,"output_tokens":84},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e06","trial":3,"input":{"filter":{"name":{"$regex":"s$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":120}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e05","trial":5,"input":{"filter":{"name":{"$regex":"^[abc]","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":702,"output_tokens":137},"repair":{"input":{"filter":{"name":{"$regex":"^[AaBbCc]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":794,"output_tokens":91},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e05","trial":1,"input":{"filter":{"name":{"$regex":"^[abc]","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":702,"output_tokens":170},"repair":{"input":{"filter":{"name":{"$regex":"^[AaBbCc]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":794,"output_tokens":106},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e06","trial":4,"input":{"filter":{"name":{"$regex":"s$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":119}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"e06","trial":5,"input":{"filter":{"name":{"$regex":"s$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"f01","trial":4,"input":{"filter":{"notes":"anything"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":60},"repair":{"input":{"filter":{"notes":{"$eq":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":765,"output_tokens":86},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"f01","trial":1,"input":{"filter":{"notes":"anything"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":53},"repair":{"input":{"filter":{"notes":{"$eq":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":765,"output_tokens":109},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"f01","trial":3,"input":{"filter":{"notes":"anything"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":75},"repair":{"input":{"filter":{"notes":{"$eq":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":765,"output_tokens":105},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"f01","trial":2,"input":{"filter":{"notes":"anything"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":60},"repair":{"input":{"filter":{"notes":{"$eq":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":765,"output_tokens":134},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"f01","trial":5,"input":{"filter":{"notes":"anything"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":55},"repair":{"input":{"filter":{"notes":{"$eq":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":765,"output_tokens":112},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"f02","trial":1,"text":"I can’t run a true free-text search across all pet records with the available search tool. It only supports structured field filters, not global text search for “ada.”","stopReason":"stop","usage":{"input_tokens":704,"output_tokens":234}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"f02","trial":2,"text":"I can’t express a free-text search across all pet records with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":704,"output_tokens":186}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"f02","trial":3,"text":"I can’t run a true free-text search across all pet records with the available tool. It only supports structured field filters, not general text search.","stopReason":"stop","usage":{"input_tokens":704,"output_tokens":179}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m01","trial":1,"input":{"filter":{"tags":{"$some":{"$eq":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m01","trial":2,"input":{"filter":{"tags":{"$some":{"$eq":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m01","trial":3,"input":{"filter":{"tags":{"$some":{"$eq":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m01","trial":4,"input":{"filter":{"tags":{"$some":{"$eq":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":62}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m01","trial":5,"input":{"filter":{"tags":{"$some":{"$eq":"indoor"}}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":69}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"f02","trial":4,"text":"I can’t run a free-text search with the available pet search tool. It only supports structured filters on specific fields, not searching for arbitrary text like “ada” across all pet records.","stopReason":"stop","usage":{"input_tokens":704,"output_tokens":162}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"f02","trial":5,"text":"I can’t express a free-text search across all pet records with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":704,"output_tokens":124}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m02","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":719,"output_tokens":82}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m02","trial":3,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":719,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m02","trial":2,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":719,"output_tokens":110}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m02","trial":5,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":719,"output_tokens":84}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a01","trial":1,"input":{"filter":{"status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":20}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m02","trial":4,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":719,"output_tokens":122}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a01","trial":3,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m03","trial":5,"input":{"filter":{"status":"Available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":25},"repair":{"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":769,"output_tokens":48},"problemShown":{"type":"invalid-operand","detail":"/status must be one of: \"available\", \"pending\", \"sold\".","pointer":"/filter/status"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m03","trial":3,"input":{"filter":{"status":"Available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":55},"repair":{"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":769,"output_tokens":48},"problemShown":{"type":"invalid-operand","detail":"/status must be one of: \"available\", \"pending\", \"sold\".","pointer":"/filter/status"}}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m03","trial":2,"input":{"filter":{"status":"Available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":54},"repair":{"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":769,"output_tokens":53},"problemShown":{"type":"invalid-operand","detail":"/status must be one of: \"available\", \"pending\", \"sold\".","pointer":"/filter/status"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a01","trial":4,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a01","trial":2,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m03","trial":4,"input":{"filter":{"status":"Available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":67},"repair":{"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":769,"output_tokens":67},"problemShown":{"type":"invalid-operand","detail":"/status must be one of: \"available\", \"pending\", \"sold\".","pointer":"/filter/status"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a01","trial":5,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a02","trial":1,"input":{"filter":{"species":"cat","status":"available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a02","trial":4,"input":{"filter":{"species":"cat","status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a02","trial":2,"input":{"filter":{"species":"cat","status":"available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a02","trial":3,"input":{"filter":{"species":"cat","status":"available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-raw","caseId":"m03","trial":1,"input":{"filter":{"status":"Available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":96},"repair":{"input":{"filter":{"status":"available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":769,"output_tokens":73},"problemShown":{"type":"invalid-operand","detail":"/status must be one of: \"available\", \"pending\", \"sold\".","pointer":"/filter/status"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a02","trial":5,"input":{"filter":{"species":"cat","status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a03","trial":1,"input":{"filter":{"status":{"$in":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a03","trial":2,"input":{"filter":{"status":{"$in":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a04","trial":1,"input":{"filter":{"weightKg":{"$between":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14099,"output_tokens":32}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a03","trial":5,"input":{"filter":{"status":{"$in":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a03","trial":3,"input":{"filter":{"status":{"$in":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":62}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a04","trial":2,"input":{"filter":{"weightKg":{"$between":[1,5]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14099,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a03","trial":4,"input":{"filter":{"status":{"$in":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":53}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a04","trial":3,"input":{"filter":{"weightKg":{"$between":[1,5]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14099,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a04","trial":4,"input":{"filter":{"weightKg":{"$between":[1,5]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14099,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a05","trial":1,"input":{"filter":{"born":{"$gte":"2020-01-01"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a05","trial":2,"input":{"filter":{"born":{"$gte":"2020-01-01"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a04","trial":5,"input":{"filter":{"weightKg":{"$between":[1,5]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14099,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a05","trial":3,"input":{"filter":{"born":{"$gte":"2020-01-01"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a05","trial":5,"input":{"filter":{"born":{"$gte":"2020-01-01"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a05","trial":4,"input":{"filter":{"born":{"$gte":"2020-01-01"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a06","trial":1,"input":{"filter":{"name":{"$startsWith":"b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14094,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a06","trial":2,"input":{"filter":{"name":{"$startsWith":"b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14094,"output_tokens":60}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a06","trial":3,"input":{"filter":{"name":{"$startsWith":"b"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14094,"output_tokens":53}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a07","trial":2,"input":{"filter":{"name":{"$ilike":"%o%"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a07","trial":3,"input":{"filter":{"name":{"$ilike":"%o%"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a07","trial":1,"input":{"filter":{"name":{"$contains":"o"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a06","trial":5,"input":{"filter":{"name":{"$startsWith":"b"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14094,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a06","trial":4,"input":{"filter":{"name":{"$startsWith":"b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14094,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a07","trial":4,"input":{"filter":{"name":{"$ilike":"%o%"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a07","trial":5,"input":{"filter":{"name":{"$ilike":"%o%"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a08","trial":2,"input":{"filter":{"name":{"$contains":"50%"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a08","trial":4,"input":{"filter":{"name":{"$contains":"50%"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a08","trial":5,"input":{"filter":{"name":{"$contains":"50%"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a08","trial":1,"input":{"filter":{"name":{"$contains":"50%"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a08","trial":3,"input":{"filter":{"name":{"$contains":"50%"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":76}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a09","trial":1,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":127}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a09","trial":2,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":97}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a09","trial":3,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":84}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a10","trial":2,"input":{"filter":{"tags":{"$hasAll":["trained","outdoor"]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14082,"output_tokens":47}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a10","trial":4,"input":{"filter":{"tags":{"$hasAll":["trained","outdoor"]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14082,"output_tokens":47}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a10","trial":1,"input":{"filter":{"tags":{"$hasAll":["trained","outdoor"]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14082,"output_tokens":92}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a09","trial":4,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a09","trial":5,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":120}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a10","trial":3,"input":{"filter":{"tags":{"$hasAll":["trained","outdoor"]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14082,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a10","trial":5,"input":{"filter":{"tags":{"$hasAll":["trained","outdoor"]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14082,"output_tokens":59}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a11","trial":1,"input":{"filter":{"tags":{"$size":{"$gte":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a11","trial":2,"input":{"filter":{"tags":{"$size":{"$gte":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":93}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a11","trial":4,"input":{"filter":{"tags":{"$size":{"$gte":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":89}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a11","trial":3,"input":{"filter":{"tags":{"$size":{"$gte":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":101}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a11","trial":5,"input":{"filter":{"tags":{"$size":{"$gte":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":90}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a13","trial":1,"input":{"filter":{"shelter.city":"Athens"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14080,"output_tokens":50}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a13","trial":3,"input":{"filter":{"shelter.city":"Athens"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14080,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a13","trial":2,"input":{"filter":{"shelter.city":"Athens"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14080,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a12","trial":1,"input":{"filter":{"$or":[{"tags":{"$exists":false}},{"tags":{"$size":{"$eq":0}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":266}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a12","trial":4,"input":{"filter":{"$or":[{"tags":{"$size":{"$eq":0}}},{"tags":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":283}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a13","trial":5,"input":{"filter":{"shelter.city":"Athens"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14080,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a13","trial":4,"input":{"filter":{"shelter.city":"Athens"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14080,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a14","trial":1,"input":{"filter":{"shelter.capacity":{"$lte":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":58}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a12","trial":3,"input":{"filter":{"$or":[{"tags":{"$exists":false}},{"tags":{"$size":{"$eq":0}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":349}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a14","trial":2,"input":{"filter":{"shelter.capacity":{"$lte":12}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":45}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a14","trial":4,"input":{"filter":{"shelter.capacity":{"$lte":12}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a14","trial":3,"input":{"filter":{"shelter.capacity":{"$lte":12}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":49}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a14","trial":5,"input":{"filter":{"shelter.capacity":{"$lte":12}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a12","trial":2,"input":{"filter":{"$or":[{"tags":{"$exists":false}},{"tags":{"$size":{"$eq":0}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":380}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a12","trial":5,"input":{"filter":{"$or":[{"tags":{"$size":{"$eq":0}}},{"tags":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":310}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a15","trial":1,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":46}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a15","trial":5,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":81}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a15","trial":4,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":42}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a15","trial":3,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":92}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a15","trial":2,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":93}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a16","trial":2,"input":{"filter":{"status":"available","$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$eq":"trained"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":137}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a16","trial":1,"input":{"filter":{"status":"available","$or":[{"shelter.city":"Patras"},{"tags":{"$hasAll":["trained"]}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":180}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a16","trial":5,"input":{"filter":{"status":"available","$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$eq":"trained"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":110}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a16","trial":3,"input":{"filter":{"status":"available","$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$in":["trained"]}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":199}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a17","trial":1,"input":{"filter":{"status":{"$nin":["sold","pending"]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":121}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a17","trial":2,"input":{"filter":{"status":{"$nin":["sold","pending"]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":141}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a16","trial":4,"input":{"filter":{"status":"available","$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$eq":"trained"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":164}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a18","trial":1,"input":{"filter":{"neutered":true}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a18","trial":2,"input":{"filter":{"neutered":true}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a18","trial":4,"input":{"filter":{"neutered":true}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a17","trial":4,"input":{"filter":{"status":{"$nin":["sold","pending"]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":148}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a18","trial":5,"input":{"filter":{"neutered":true}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a17","trial":3,"input":{"filter":{"status":{"$nin":["sold","pending"]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":225}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a18","trial":3,"input":{"filter":{"neutered":true}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a17","trial":5,"input":{"filter":{"status":{"$nin":["sold","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":158}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a19","trial":2,"text":"I can’t express “priceCents is greater than costCents” with the available pet search tool, because it doesn’t support comparing one field to another.","stopReason":"stop","usage":{"input_tokens":14081,"output_tokens":165}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a19","trial":1,"text":"I can’t express “price is greater than cost” with the pet search tool, because it doesn’t support comparing one field (`priceCents`) to another field (`costCents`).","stopReason":"stop","usage":{"input_tokens":14081,"output_tokens":269}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a19","trial":5,"text":"I can’t express “priceCents is greater than costCents” with the available pet-search tool, because it doesn’t support comparing one field to another.","stopReason":"stop","usage":{"input_tokens":14081,"output_tokens":192}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a20","trial":4,"input":{"filter":{"status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":20}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a20","trial":2,"input":{"filter":{"status":"available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":151}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a19","trial":4,"text":"I can’t express “priceCents is greater than costCents” with the pet search tool, because it supports comparisons against fixed values but not field-to-field comparisons.","stopReason":"stop","usage":{"input_tokens":14081,"output_tokens":225}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a20","trial":3,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":221}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a20","trial":1,"input":{"filter":{"status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":269}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a19","trial":3,"text":"I can’t express that search with the pet-store tool: it supports comparing fields to fixed values, but not comparing `priceCents` against `costCents` for each pet.","stopReason":"stop","usage":{"input_tokens":14081,"output_tokens":331}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"a20","trial":5,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":177}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b01","trial":1,"input":{"filter":{"vaccinations":{"$some":{"vaccine":"rabies","boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14111,"output_tokens":244}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b01","trial":3,"input":{"filter":{"vaccinations":{"$some":{"vaccine":"rabies","boosterDue":{"$lt":"2025-01-01"}}}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14111,"output_tokens":206}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b01","trial":5,"input":{"filter":{"vaccinations":{"$some":{"vaccine":"rabies","boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14111,"output_tokens":351}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b01","trial":2,"input":{"filter":{"vaccinations":{"$some":{"$and":[{"name":"rabies"},{"boosterDue":{"$lt":"2025-01-01"}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14111,"output_tokens":324},"repair":{"input":{"filter":{"vaccinations":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14219,"output_tokens":147},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/$and/0/name"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b02","trial":3,"input":{"filter":{"$and":[{"vaccinations":{"$some":{"vaccine":"rabies"}}},{"vaccinations":{"$some":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14116,"output_tokens":397}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b02","trial":2,"input":{"filter":{"vaccinations":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14116,"output_tokens":547}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b03","trial":1,"input":{"filter":{"vaccinations":{"$some":{"date":{"$gte":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":207},"repair":{"text":"I can’t express that search with the available pet-store tool: vaccination dates are not a queryable field.","stopReason":"stop","usage":{"input_tokens":14176,"output_tokens":124},"problemShown":{"type":"unknown-field","detail":"\"date\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/date"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b02","trial":5,"input":{"filter":{"$and":[{"vaccinations":{"$some":{"vaccine":"rabies"}}},{"vaccinations":{"$some":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14116,"output_tokens":578}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b01","trial":4,"input":{"filter":{"vaccinations":{"$some":{"name":"rabies","boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14111,"output_tokens":283},"repair":{"text":"I can’t express that query with the available pet-search tool. It would require filtering within the same `vaccinations` array element, e.g. “some vaccination has type/name rabies AND that same vaccination’s boosterDue < 2025-01-01,” but the tool does not expose queryable fields inside `vaccinations` such as the shot name or booster due date.","stopReason":"stop","usage":{"input_tokens":14210,"output_tokens":565},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b02","trial":1,"input":{"filter":{"$and":[{"vaccinations":{"$some":{"name":"rabies"}}},{"vaccinations":{"$some":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14116,"output_tokens":524},"repair":{"input":{"filter":{"$and":[{"vaccinations":{"$some":{"shot":"rabies"}}},{"vaccinations":{"$some":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14231,"output_tokens":389},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/$and/0/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b03","trial":2,"input":{"filter":{"vaccinations":{"$some":{"date":{"$gte":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":170},"repair":{"text":"I can’t express that search with the available pet-store tool: vaccination dates inside `vaccinations` are not a queryable field/path.","stopReason":"stop","usage":{"input_tokens":14176,"output_tokens":188},"problemShown":{"type":"unknown-field","detail":"\"date\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/date"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b03","trial":3,"input":{"filter":{"vaccinations":{"$some":{"date":{"$gte":"2025-01-01"}}}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":151},"repair":{"text":"I’m sorry, but this search can’t be expressed with the available pet-search tool because vaccination dates inside `vaccinations` are not a queryable field.","stopReason":"stop","usage":{"input_tokens":14172,"output_tokens":171},"problemShown":{"type":"unknown-field","detail":"\"date\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/date"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b04","trial":1,"text":"I can’t express “parvo as the very first entry” with the pet search tool, because it doesn’t support filtering vaccination history by array position/order.","stopReason":"stop","usage":{"input_tokens":14085,"output_tokens":224}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b04","trial":2,"text":"I can’t express “parvo as the very first entry in vaccination history” with the available pet-search tool. It supports checking whether some/all vaccination entries match, but not querying the position of an array entry.","stopReason":"stop","usage":{"input_tokens":14085,"output_tokens":202}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b02","trial":4,"input":{"filter":{"$and":[{"vaccinations":{"$some":{"name":"rabies"}}},{"vaccinations":{"$some":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14116,"output_tokens":443},"repair":{"text":"I can’t express that search with the available pet-search tool: the vaccination records’ internal fields like shot name/type and booster due date are not queryable paths.","stopReason":"stop","usage":{"input_tokens":14231,"output_tokens":399},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/$and/0/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b04","trial":4,"text":"I can’t express “the very first entry in vaccination history” with the available pet search tool, because it supports matching array elements generally but not querying by array position/order.","stopReason":"stop","usage":{"input_tokens":14085,"output_tokens":160}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b04","trial":5,"text":"I can’t express “the very first entry” in the vaccination history with the available pet search tool, since it doesn’t support positional array queries.","stopReason":"stop","usage":{"input_tokens":14085,"output_tokens":154}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b04","trial":3,"text":"I can’t express “parvo as the very first entry” with the available pet search tool. It can search whether vaccination history contains something, but not by array position/order.","stopReason":"stop","usage":{"input_tokens":14085,"output_tokens":195}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b03","trial":5,"input":{"filter":{"vaccinations":{"$some":{"date":{"$gte":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":256},"repair":{"text":"I can’t express that search with the available pet-store tool: vaccination dates aren’t a queryable field/path.","stopReason":"stop","usage":{"input_tokens":14176,"output_tokens":153},"problemShown":{"type":"unknown-field","detail":"\"date\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/date"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b03","trial":4,"input":{"filter":{"vaccinations":{"$some":{"date":{"$gte":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":435},"repair":{"text":"I can’t express that search with the available pet-store tool: vaccination dates aren’t a queryable field.","stopReason":"stop","usage":{"input_tokens":14176,"output_tokens":119},"problemShown":{"type":"unknown-field","detail":"\"date\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/date"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b06","trial":1,"input":{"filter":{"$or":[{"tags":{"$exists":false}},{"tags":{"$not":{"$some":{"$eq":"indoor"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":250}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b06","trial":2,"input":{"filter":{"tags":{"$not":{"$some":{"$eq":"indoor"}},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":344}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b06","trial":3,"input":{"filter":{"tags":{"$not":{"$some":{"$eq":"indoor"}},"$unknownAs":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":304}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b05","trial":1,"input":{"filter":{"vaccinations":{"$some":{"name":{"$ilike":"parvo"}}}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":371},"repair":{"input":{"filter":{"vaccinations":{"$some":{"$ilike":"%parvo%"}}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14164,"output_tokens":157},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b05","trial":3,"input":{"filter":{"vaccinations":{"$some":{"vaccine":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":551}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b06","trial":4,"input":{"filter":{"tags":{"$not":{"$some":{"$eq":"indoor"}},"$unknownAs":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":299}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b07","trial":2,"input":{"filter":{"tags":{"$some":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":146}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b07","trial":1,"input":{"filter":{"tags":{"$some":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":182}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b06","trial":5,"input":{"filter":{"tags":{"$not":{"$some":{"$eq":"indoor"}},"$unknownAs":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":274}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b07","trial":3,"input":{"filter":{"tags":{"$some":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":147}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b07","trial":4,"input":{"filter":{"tags":{"$some":{"$ne":"indoor"}}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":145}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b07","trial":5,"input":{"filter":{"tags":{"$some":{"$ne":"indoor"}}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":158}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b08","trial":1,"text":"I can’t express “indoor as the first label” with the available pet search tool, because it supports array membership/size-style filters but not matching an array element by position.","stopReason":"stop","usage":{"input_tokens":14082,"output_tokens":153}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b05","trial":4,"input":{"filter":{"vaccinations":{"$some":{"name":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":546},"repair":{"input":{"filter":{"vaccinations":{"$some":{"$eq":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14163,"output_tokens":339},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b05","trial":5,"input":{"filter":{"vaccinations":{"$some":{"name":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":504},"repair":{"text":"I can’t express “had a parvo vaccination” with the available pet-search tool because the specific vaccination name/type field isn’t queryable.","stopReason":"stop","usage":{"input_tokens":14163,"output_tokens":358},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/name"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b08","trial":2,"text":"I can’t express “indoor as the first label” with the available pet search tool, because it supports array membership/size checks but not checking an array element by position.","stopReason":"stop","usage":{"input_tokens":14082,"output_tokens":146}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b08","trial":3,"text":"I can’t express “indoor as the first label” with the available pet search tool, because it supports array membership/quantifiers but not positional array queries like “first element.”","stopReason":"stop","usage":{"input_tokens":14082,"output_tokens":179}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b05","trial":2,"input":{"filter":{"vaccinations":{"$some":{"$or":[{"name":{"$ilike":"%parvo%"}},{"vaccine":{"$ilike":"%parvo%"}},{"disease":{"$ilike":"%parvo%"}},{"type":{"$ilike":"%parvo%"}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":600},"repair":{"input":{"filter":{"vaccinations":{"$some":{"$contains":"parvo"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14217,"output_tokens":291},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/$or/0/name"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b08","trial":4,"text":"I can’t express “indoor as the first label” with the available pet search tool, because it doesn’t support filtering array fields by position/order.","stopReason":"stop","usage":{"input_tokens":14082,"output_tokens":158}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b08","trial":5,"text":"I can’t express “indoor as the first label” with the available pet search tool, because it supports checking labels in the `tags` array but not their position/order.","stopReason":"stop","usage":{"input_tokens":14082,"output_tokens":133}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b09","trial":2,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":{"$exists":true}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":186},"repair":{"text":"I can’t express that search with the available pet-store tool: `boosterDate` is not a queryable field/path for vaccinations.","stopReason":"stop","usage":{"input_tokens":14177,"output_tokens":121},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b09","trial":4,"input":{"filter":{"vaccinations":{"$some":{"booster":{"$exists":true}}}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":150},"repair":{"text":"I can’t express “vaccination with a booster date recorded” with the available pet search tool, because `booster` is not a queryable field within `vaccinations`.","stopReason":"stop","usage":{"input_tokens":14170,"output_tokens":168},"problemShown":{"type":"unknown-field","detail":"\"booster\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/booster"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b09","trial":1,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":{"$exists":true}}}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":380},"repair":{"text":"I can’t answer that with the available pet-search tool: vaccination booster-date fields are not queryable.","stopReason":"stop","usage":{"input_tokens":14173,"output_tokens":95},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b10","trial":2,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":{"$exists":false}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":332},"repair":{"text":"I can’t answer that with the available pet search tool: `boosterDate` within `vaccinations` is not a queryable field on this resource.","stopReason":"stop","usage":{"input_tokens":14177,"output_tokens":144},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b09","trial":5,"input":{"filter":{"vaccinations":{"$some":{"booster":{"$exists":true,"$isNull":false}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":560},"repair":{"text":"I can’t express that search with the available pet-store tool: the vaccination “booster date” field isn’t a queryable path.","stopReason":"stop","usage":{"input_tokens":14180,"output_tokens":160},"problemShown":{"type":"unknown-field","detail":"\"booster\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/booster"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b10","trial":1,"input":{"filter":{"vaccinations":{"$some":{"$or":[{"boosterDate":{"$exists":false}},{"boosterDate":{"$isNull":true}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":544},"repair":{"text":"I can’t express that search with the available pet-store tool: `boosterDate` inside `vaccinations` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":14197,"output_tokens":124},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/$or/0/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b09","trial":3,"input":{"filter":{"vaccinations":{"$some":{"boosterDate":{"$exists":true,"$isNull":false}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":561},"repair":{"text":"I can’t express that search with the available pet-store tool: `boosterDate` is not a queryable field within `vaccinations`.","stopReason":"stop","usage":{"input_tokens":14183,"output_tokens":155},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b10","trial":3,"input":{"filter":{"vaccinations":{"$some":{"boosterDue":{"$isNull":true,"$unknownAs":true}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":501},"repair":{"input":{"filter":{"vaccinations":{"$some":{"boosterDue":{"$exists":false}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14194,"output_tokens":140},"problemShown":{"type":"unsupported-operator","detail":"$isNull is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$some/boosterDue/$isNull"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b10","trial":5,"input":{"filter":{"vaccinations":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":547}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b10","trial":4,"input":{"filter":{"vaccinations":{"$some":{"$or":[{"boosterDate":{"$exists":false}},{"boosterDate":{"$isNull":true}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":571},"repair":{"text":"I can’t express that search with the available tool: vaccination subfields like `boosterDate` are not queryable.","stopReason":"stop","usage":{"input_tokens":14197,"output_tokens":109},"problemShown":{"type":"unknown-field","detail":"\"boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/$or/0/boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b11","trial":2,"input":{"filter":{"status":{"$in":["available","pending","sold"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14094,"output_tokens":460}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b12","trial":1,"input":{"filter":{"tags":{"$every":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14093,"output_tokens":273}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b12","trial":2,"input":{"filter":{"tags":{"$not":{"$some":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14093,"output_tokens":276}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b11","trial":1,"input":{"filter":{"vaccinations":{"$not":{"$some":{"$ne":"rabies"}},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14094,"output_tokens":561},"repair":{"input":{"filter":{"vaccinations":{"$every":{"$eq":"rabies"},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14200,"output_tokens":173},"problemShown":{"type":"unsupported-operator","detail":"$ne is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$not/$some/$ne"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b12","trial":4,"input":{"filter":{"tags":{"$every":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14093,"output_tokens":239}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b12","trial":3,"input":{"filter":{"tags":{"$not":{"$some":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14093,"output_tokens":300}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b12","trial":5,"input":{"filter":{"tags":{"$every":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14093,"output_tokens":259}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b11","trial":3,"input":{"filter":{"vaccinations":{"$not":{"$some":{"$ne":"rabies"}},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14094,"output_tokens":561},"repair":{"input":{"filter":{"vaccinations":{"$every":{"$eq":"rabies"},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14200,"output_tokens":170},"problemShown":{"type":"unsupported-operator","detail":"$ne is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$not/$some/$ne"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b13","trial":1,"input":{"filter":{"$or":[{"tags":{"$every":{"$eq":"indoor"}}},{"tags":{"$size":{"$eq":0}}},{"tags":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":382}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b11","trial":4,"input":{"filter":{"$or":[{"vaccinations":{"$exists":false}},{"vaccinations":{"$every":{"$eq":"rabies"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14094,"output_tokens":552},"repair":{"input":{"filter":{"vaccinations":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14207,"output_tokens":293},"problemShown":{"type":"unsupported-operator","detail":"$eq is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$or/1/vaccinations/$every/$eq"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b13","trial":2,"input":{"filter":{"$or":[{"tags":{"$every":{"$eq":"indoor"}}},{"tags":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":367}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b11","trial":5,"input":{"filter":{"vaccinations":{"$not":{"$some":{"$ne":"rabies"}},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14094,"output_tokens":561},"repair":{"input":{"filter":{"vaccinations":{"$every":{"$eq":"rabies"},"$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14200,"output_tokens":302},"problemShown":{"type":"unsupported-operator","detail":"$ne is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$not/$some/$ne"}}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c01","trial":2,"input":{"filter":{"microchip":{"$exists":true,"$isNull":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":149}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b13","trial":5,"input":{"filter":{"$or":[{"tags":{"$every":{"$eq":"indoor"}}},{"tags":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":400}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b13","trial":4,"input":{"filter":{"$or":[{"tags":{"$every":{"$eq":"indoor"}}},{"tags":{"$size":{"$eq":0}}},{"tags":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":380}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"b13","trial":3,"input":{"filter":{"$or":[{"tags":{"$some":{"$eq":"indoor"},"$every":{"$eq":"indoor"}}},{"tags":{"$size":{"$eq":0}}},{"tags":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":503}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c01","trial":4,"input":{"filter":{"microchip":{"$exists":true,"$isNull":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":173}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c02","trial":1,"input":{"filter":{"microchip":{"$exists":false}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c02","trial":2,"input":{"filter":{"microchip":{"$exists":false}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c02","trial":3,"input":{"filter":{"microchip":{"$exists":false}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":49}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c02","trial":4,"input":{"filter":{"microchip":{"$exists":false}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c02","trial":5,"input":{"filter":{"microchip":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":60}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c03","trial":1,"input":{"filter":{"notes":{"$exists":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c03","trial":2,"input":{"filter":{"notes":{"$exists":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c03","trial":3,"input":{"filter":{"notes":{"$exists":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c03","trial":4,"input":{"filter":{"notes":{"$exists":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c01","trial":3,"input":{"filter":{"microchip":{"$exists":true,"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":339}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c01","trial":1,"input":{"filter":{"microchip":{"$exists":true,"$isNull":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":553}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c03","trial":5,"input":{"filter":{"notes":{"$exists":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c04","trial":1,"input":{"filter":{"shelter":{"$exists":true},"shelter.capacity":{"$exists":false}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":106}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c04","trial":2,"input":{"filter":{"shelter":{"$exists":true},"shelter.capacity":{"$exists":false}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":111}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c04","trial":3,"input":{"filter":{"shelter":{"$exists":true},"shelter.capacity":{"$exists":false}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":128}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c04","trial":4,"input":{"filter":{"shelter":{"$exists":true},"shelter.capacity":{"$exists":false}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":151}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c04","trial":5,"input":{"filter":{"shelter.capacity":{"$exists":false}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c01","trial":5,"input":{"filter":{"$and":[{"microchip":{"$exists":true}},{"$or":[{"microchip":{"$isNull":true}},{"microchip":""}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":521}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c05","trial":2,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14097,"output_tokens":246}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c05","trial":3,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14097,"output_tokens":172}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c05","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$isNull":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14097,"output_tokens":304}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c05","trial":5,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14097,"output_tokens":202}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c05","trial":4,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14097,"output_tokens":214}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c07","trial":1,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$isNull":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14103,"output_tokens":172}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c08","trial":2,"input":{"filter":{"status":{"$ne":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14076,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c08","trial":1,"input":{"filter":{"status":{"$ne":"sold"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14076,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c08","trial":3,"input":{"filter":{"status":{"$ne":"sold"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14076,"output_tokens":84}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c07","trial":5,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$isNull":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14103,"output_tokens":229}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c07","trial":2,"input":{"filter":{"$and":[{"microchip":{"$exists":true}},{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$isNull":true}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14103,"output_tokens":356}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c07","trial":3,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$isNull":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14103,"output_tokens":229}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c08","trial":4,"input":{"filter":{"status":{"$ne":"sold"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14076,"output_tokens":110}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c07","trial":4,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$isNull":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14103,"output_tokens":281}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c11","trial":1,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c11","trial":2,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":89}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c08","trial":5,"input":{"filter":{"status":{"$ne":"sold"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14076,"output_tokens":106}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c11","trial":3,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":109}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c11","trial":4,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":124}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c11","trial":5,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":112}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c12","trial":2,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14106,"output_tokens":227}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c12","trial":3,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14106,"output_tokens":245}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c12","trial":4,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14106,"output_tokens":213}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c13","trial":1,"input":{"filter":{"born":{"$lt":"2020-01-01","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14101,"output_tokens":187}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c12","trial":5,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14106,"output_tokens":236}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c12","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14106,"output_tokens":220}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c13","trial":3,"input":{"filter":{"born":{"$lt":"2020-01-01","$unknownAs":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14101,"output_tokens":203}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d01","trial":1,"input":{"filter":{"notes":{"$type":"number"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d01","trial":2,"input":{"filter":{"notes":{"$type":"number"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d01","trial":3,"input":{"filter":{"notes":{"$type":"number"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d01","trial":4,"input":{"filter":{"notes":{"$type":"number"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d01","trial":5,"input":{"filter":{"notes":{"$type":"number"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c13","trial":4,"input":{"filter":{"born":{"$lt":"2020-01-01","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14101,"output_tokens":220}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d02","trial":2,"input":{"filter":{"notes":{"$type":"integer"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d02","trial":1,"input":{"filter":{"notes":{"$type":"integer"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":84}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d02","trial":4,"input":{"filter":{"notes":{"$type":"integer"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":77}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c13","trial":5,"input":{"filter":{"born":{"$lt":"2020-01-01","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14101,"output_tokens":274}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d03","trial":1,"input":{"filter":{"notes":{"$type":"array"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":39}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d02","trial":3,"input":{"filter":{"notes":{"$type":"integer"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":84}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d03","trial":2,"input":{"filter":{"notes":{"$type":"array"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d03","trial":4,"input":{"filter":{"notes":{"$type":"array"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d02","trial":5,"input":{"filter":{"notes":{"$type":"integer"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":72}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d03","trial":3,"input":{"filter":{"notes":{"$type":"array"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":51}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"c13","trial":2,"input":{"filter":{"born":{"$lt":"2020-01-01","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14101,"output_tokens":261}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d03","trial":5,"input":{"filter":{"notes":{"$type":"array"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14081,"output_tokens":52}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d04","trial":1,"input":{"filter":{"notes":{"$isNull":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":92}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d04","trial":3,"input":{"filter":{"notes":{"$isNull":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":108}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d05","trial":1,"input":{"filter":{"notes":{"$eq":3}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d05","trial":3,"input":{"filter":{"notes":{"$eq":3}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":58}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d05","trial":2,"input":{"filter":{"notes":{"$eq":3}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":85}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d04","trial":5,"input":{"filter":{"notes":{"$isNull":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":102}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d04","trial":4,"input":{"filter":{"notes":{"$isNull":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":132}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d04","trial":2,"input":{"filter":{"notes":{"$isNull":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":138}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d05","trial":4,"input":{"filter":{"notes":{"$eq":3}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d05","trial":5,"input":{"filter":{"notes":{"$eq":3}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d06","trial":1,"input":{"filter":{"notes":{"$type":"number","$gt":2}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":121}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d06","trial":4,"input":{"filter":{"notes":{"$type":"number","$gt":2}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":82}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d06","trial":5,"input":{"filter":{"notes":{"$type":"number","$gt":2}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d06","trial":2,"input":{"filter":{"notes":{"$type":"number","$gt":2}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":112}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d06","trial":3,"input":{"filter":{"notes":{"$type":"number","$gt":2}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":163}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d07","trial":1,"input":{"filter":{"notes":{"$type":"string","$gt":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":113}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d07","trial":2,"input":{"filter":{"notes":{"$type":"string","$gt":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d07","trial":3,"input":{"filter":{"notes":{"$type":"string","$gt":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":102}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d07","trial":4,"input":{"filter":{"notes":{"$type":"string","$gt":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":100}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d07","trial":5,"input":{"filter":{"notes":{"$type":"string","$gt":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":119}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d09","trial":1,"input":{"filter":{"notes":{"$lte":2}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":99}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d08","trial":1,"input":{"filter":{"notes":{"$ne":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":229}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d08","trial":3,"input":{"filter":{"notes":{"$ne":3}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":177}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d08","trial":2,"input":{"filter":{"notes":{"$ne":3}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":225}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d08","trial":4,"input":{"filter":{"notes":{"$ne":3}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":206}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d09","trial":2,"input":{"filter":{"notes":{"$lte":2}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":178}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d09","trial":3,"input":{"filter":{"notes":{"$lte":2}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":157}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d08","trial":5,"input":{"filter":{"notes":{"$ne":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14085,"output_tokens":292}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e01","trial":3,"input":{"filter":{"$$rate":{"$gt":5}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":50}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e01","trial":2,"input":{"filter":{"$$rate":{"$gt":5}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e01","trial":4,"input":{"filter":{"$$rate":{"$gt":5}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d09","trial":5,"input":{"filter":{"notes":{"$lte":2}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":138}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e02","trial":1,"input":{"filter":{"size\\.raw":"XS"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14101,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"d09","trial":4,"input":{"filter":{"notes":{"$lte":2}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":217}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e01","trial":5,"input":{"filter":{"$$rate":{"$gt":5}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e01","trial":1,"input":{"filter":{"$$rate":{"$gt":5}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14090,"output_tokens":55}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e02","trial":2,"input":{"filter":{"size\\.raw":"XS"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14101,"output_tokens":52}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e02","trial":4,"input":{"filter":{"size\\.raw":"XS"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14101,"output_tokens":58}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e03","trial":2,"input":{"filter":{"name":{"$regex":"^C.*o$"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e03","trial":4,"input":{"filter":{"name":{"$regex":"^C.*o$"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e03","trial":1,"input":{"filter":{"name":{"$startsWith":"C","$endsWith":"o"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":67}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e03","trial":3,"input":{"filter":{"name":{"$startsWith":"C","$endsWith":"o"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e02","trial":3,"input":{"filter":{"size\\.raw":"XS"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14101,"output_tokens":175}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e02","trial":5,"input":{"filter":{"size\\.raw":"XS"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14101,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e03","trial":5,"input":{"filter":{"name":{"$regex":"^C.*o$"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e04","trial":2,"input":{"filter":{"$or":[{"name":{"$startsWith":"A"}},{"name":{"$startsWith":"B"}},{"name":{"$startsWith":"C"}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":121}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e04","trial":1,"input":{"filter":{"$or":[{"name":{"$startsWith":"A"}},{"name":{"$startsWith":"B"}},{"name":{"$startsWith":"C"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":90}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e04","trial":3,"input":{"filter":{"$or":[{"name":{"$startsWith":"A"}},{"name":{"$startsWith":"B"}},{"name":{"$startsWith":"C"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":117}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e04","trial":4,"input":{"filter":{"$or":[{"name":{"$startsWith":"A"}},{"name":{"$startsWith":"B"}},{"name":{"$startsWith":"C"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":93}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e04","trial":5,"input":{"filter":{"$or":[{"name":{"$startsWith":"A"}},{"name":{"$startsWith":"B"}},{"name":{"$startsWith":"C"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14086,"output_tokens":119}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e05","trial":1,"input":{"filter":{"name":{"$regex":"^[abc]","$flags":"i"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14087,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e06","trial":2,"input":{"filter":{"name":{"$endsWith":"s"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e05","trial":2,"input":{"filter":{"name":{"$regex":"^[abc]","$flags":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14087,"output_tokens":90}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e05","trial":3,"input":{"filter":{"name":{"$regex":"^[abc]","$flags":"i"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14087,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e05","trial":5,"input":{"filter":{"name":{"$regex":"^[abc]","$flags":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14087,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e05","trial":4,"input":{"filter":{"name":{"$regex":"^[abc]","$flags":"i"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14087,"output_tokens":89}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e06","trial":1,"input":{"filter":{"name":{"$endsWith":"s"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e06","trial":4,"input":{"filter":{"name":{"$endsWith":"s"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e06","trial":5,"input":{"filter":{"name":{"$endsWith":"s"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"e06","trial":3,"input":{"filter":{"name":{"$endsWith":"s"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"f01","trial":1,"input":{"filter":{"notes":{"$eq":"anything"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":90}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"f01","trial":3,"input":{"filter":{"notes":{"$eq":"anything"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":104}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"f01","trial":5,"input":{"filter":{"notes":{"$eq":"anything"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":86}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"f01","trial":2,"input":{"filter":{"notes":{"$eq":"anything"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":123}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"f01","trial":4,"input":{"filter":{"notes":{"$eq":"anything"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":156}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"f02","trial":1,"text":"I can’t express a free-text search across all pet record fields with the available pet search tool. It only supports structured filters on specific fields.","stopReason":"stop","usage":{"input_tokens":14089,"output_tokens":86}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"f02","trial":5,"text":"I can’t run a true free-text search across all pet record fields with the available pet-search tool. The tool only supports structured filters on specific fields, so I shouldn’t approximate it.","stopReason":"stop","usage":{"input_tokens":14089,"output_tokens":117}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m01","trial":2,"input":{"filter":{"tags":{"$some":{"$eq":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":105}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"f02","trial":4,"text":"I can’t run a free-text search across all pet records with the available tool. It only supports structured field-specific filters, not global free-text search.","stopReason":"stop","usage":{"input_tokens":14089,"output_tokens":127}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"f02","trial":3,"text":"I can’t run a free-text search across all pet records with this tool. It only supports structured filters on specific fields, not a global text search for “ada.”","stopReason":"stop","usage":{"input_tokens":14089,"output_tokens":117}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m01","trial":1,"input":{"filter":{"tags":{"$hasAll":["indoor"]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":105}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m01","trial":3,"input":{"filter":{"tags":{"$some":{"$eq":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":62}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m01","trial":4,"input":{"filter":{"tags":{"$hasAll":["indoor"]}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":96}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"f02","trial":2,"text":"I can’t run a true free-text search across entire pet records with this tool. It only supports structured filters on specific fields, so I can’t express “search everywhere for `ada`” without approximating.","stopReason":"stop","usage":{"input_tokens":14089,"output_tokens":314}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m02","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14104,"output_tokens":155}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m02","trial":2,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14104,"output_tokens":209}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m02","trial":4,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14104,"output_tokens":184}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m01","trial":5,"input":{"filter":{"tags":{"$some":{"$eq":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":130}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m02","trial":3,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14104,"output_tokens":197}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m02","trial":5,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14104,"output_tokens":190}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m03","trial":2,"input":{"filter":{"status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":36}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m03","trial":3,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m03","trial":1,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":42}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m03","trial":5,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":42}} +{"provider":"gpt-5.5","arm":"jql-generated","caseId":"m03","trial":4,"input":{"filter":{"status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":44}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a01","trial":1,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":99}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a01","trial":2,"input":{"filter":{"status":{"$eq":"available"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":84}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a01","trial":3,"input":{"filter":{"status":{"$eq":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":88}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a01","trial":5,"input":{"filter":{"status":{"$eq":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a01","trial":4,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a02","trial":1,"input":{"filter":{"species":{"$eq":"cat"},"status":{"$eq":"available"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":91}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a02","trial":2,"input":{"filter":{"$and":[{"species":{"$eq":"cat"}},{"status":{"$eq":"available"}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":106}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a02","trial":5,"input":{"filter":{"species":{"$eq":"cat"},"status":{"$eq":"available"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":128}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a03","trial":1,"input":{"filter":{"$or":[{"status":{"$eq":"available"}},{"status":{"$eq":"pending"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":133}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a03","trial":4,"input":{"filter":{"$or":[{"status":{"$eq":"available"}},{"status":{"$eq":"pending"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":126}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a02","trial":4,"input":{"filter":{"species":{"$eq":"cat"},"status":{"$eq":"available"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":110}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a03","trial":2,"input":{"filter":{"$or":[{"status":"available"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":131}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a03","trial":3,"input":{"filter":{"$or":[{"status":{"$eq":"available"}},{"status":{"$eq":"pending"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":174}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a02","trial":3,"input":{"filter":{"species":{"$eq":"cat"},"status":{"$eq":"available"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":208}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a03","trial":5,"input":{"filter":{"$or":[{"status":"available"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":209}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a04","trial":2,"input":{"filter":{"weightKg":{"$gte":1,"$lte":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":697,"output_tokens":142}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a04","trial":1,"input":{"filter":{"weightKg":{"$gte":1,"$lte":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":697,"output_tokens":152}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a04","trial":5,"input":{"filter":{"weightKg":{"$gte":1,"$lte":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":697,"output_tokens":106}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a04","trial":3,"input":{"filter":{"weightKg":{"$gte":1,"$lte":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":697,"output_tokens":144}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a04","trial":4,"input":{"filter":{"weightKg":{"$gte":1,"$lte":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":697,"output_tokens":242}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a05","trial":2,"input":{"filter":{"born":{"$gte":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":203}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a05","trial":1,"input":{"filter":{"born":{"$gte":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":285}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a05","trial":5,"input":{"filter":{"born":{"$gte":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":223}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a06","trial":2,"input":{"filter":{"name":{"$regex":"^b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":215}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a05","trial":4,"input":{"filter":{"born":{"$gte":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":200}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a06","trial":1,"input":{"filter":{"name":{"$regex":"^b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":296}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a06","trial":3,"input":{"filter":{"name":{"$regex":"^b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":212}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a06","trial":4,"input":{"filter":{"name":{"$regex":"^b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":177}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a06","trial":5,"input":{"filter":{"name":{"$like":"b%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":277}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a05","trial":3,"input":{"filter":{"birthDate":{"$gte":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":305},"repair":{"text":"I can’t answer that with the available pet search tool because birth date is not a queryable field.","stopReason":"stop","usage":{"input_tokens":766,"output_tokens":155},"problemShown":{"type":"unknown-field","detail":"\"birthDate\" is not a queryable path on this resource.","pointer":"/filter/birthDate"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a07","trial":1,"input":{"filter":{"name":{"$ilike":"%o%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":338}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a07","trial":3,"input":{"filter":{"name":{"$regex":"o","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":208},"repair":{"input":{"filter":{"name":{"$regex":"(?i)o"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":773,"output_tokens":104},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a07","trial":2,"input":{"filter":{"name":{"$regex":"o","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":258},"repair":{"input":{"filter":{"name":{"$regex":"(?i)o"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":773,"output_tokens":97},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a07","trial":5,"input":{"filter":{"name":{"$regex":"o","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":211},"repair":{"input":{"filter":{"name":{"$regex":"[oO]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":773,"output_tokens":116},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a07","trial":4,"input":{"filter":{"name":{"$regex":"o","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":196},"repair":{"input":{"filter":{"name":{"$regex":"(?i)o"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":773,"output_tokens":126},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a08","trial":1,"input":{"filter":{"name":{"$like":"%50\\%%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":520}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a08","trial":4,"input":{"filter":{"name":{"$regex":"50%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":281}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a08","trial":2,"input":{"filter":{"name":{"$like":"%50\\%%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":461}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a08","trial":3,"input":{"filter":{"name":{"$like":"%50\\%%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":533}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a09","trial":1,"input":{"filter":{"labels":{"$in":["indoor","small"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":305},"repair":{"text":"I can’t answer that with the available pet search tool because `labels` is not a queryable pet field.","stopReason":"stop","usage":{"input_tokens":761,"output_tokens":113},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a08","trial":5,"input":{"filter":{"name":{"$regex":"50%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":492}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a09","trial":2,"input":{"filter":{"labels":{"$in":["indoor","small"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":268},"repair":{"text":"I can’t answer that with the available pet search tool because “labels” is not a queryable field for pets.","stopReason":"stop","usage":{"input_tokens":761,"output_tokens":141},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a09","trial":3,"input":{"filter":{"labels":{"$in":["indoor","small"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":330},"repair":{"text":"I can’t search by “labels” because that is not a queryable field in the pet-store search tool.","stopReason":"stop","usage":{"input_tokens":761,"output_tokens":114},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a10","trial":1,"input":{"filter":{"labels":{"$all":["trained","outdoor"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":680,"output_tokens":247},"repair":{"input":{"filter":{"$and":[{"labels":"trained"},{"labels":"outdoor"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":770,"output_tokens":170},"problemShown":{"type":"unsupported-operator","detail":"$all is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/labels/$all"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a09","trial":5,"input":{"filter":{"labels":{"$in":["indoor","small"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":414},"repair":{"text":"I can’t express that search with the available pet-store tool because “labels” is not a queryable field.","stopReason":"stop","usage":{"input_tokens":761,"output_tokens":111},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a09","trial":4,"input":{"filter":{"labels":{"$in":["indoor","small"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":219},"repair":{"text":"I can’t answer that with the available pet search tool because “labels” is not a queryable field for this resource.","stopReason":"stop","usage":{"input_tokens":761,"output_tokens":125},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a10","trial":3,"input":{"filter":{"labels":{"$all":["trained","outdoor"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":680,"output_tokens":201},"repair":{"input":{"filter":{"$and":[{"labels":"trained"},{"labels":"outdoor"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":770,"output_tokens":151},"problemShown":{"type":"unsupported-operator","detail":"$all is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/labels/$all"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a10","trial":5,"input":{"filter":{"labels":{"$all":["trained","outdoor"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":680,"output_tokens":182},"repair":{"input":{"filter":{"$and":[{"labels":{"$eq":"trained"}},{"labels":{"$eq":"outdoor"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":770,"output_tokens":132},"problemShown":{"type":"unsupported-operator","detail":"$all is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/labels/$all"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a10","trial":4,"input":{"filter":{"labels":{"$all":["trained","outdoor"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":680,"output_tokens":130},"repair":{"input":{"filter":{"$and":[{"labels":"trained"},{"labels":"outdoor"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":770,"output_tokens":257},"problemShown":{"type":"unsupported-operator","detail":"$all is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/labels/$all"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a11","trial":3,"text":"I can’t express “pets with three or more labels” with the available pet-search filter, because it doesn’t provide an array-length/count operator for labels.","stopReason":"stop","usage":{"input_tokens":676,"output_tokens":270}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a11","trial":1,"text":"I can’t express “three or more labels” with the pet-search tool as described, because it doesn’t provide an array-length/count operator for labels.","stopReason":"stop","usage":{"input_tokens":676,"output_tokens":514}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a11","trial":4,"text":"I can’t express “pets with three or more labels” with the search tool as described, because it doesn’t provide an array length/cardinality operator for the `labels` field.","stopReason":"stop","usage":{"input_tokens":676,"output_tokens":356}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a11","trial":2,"text":"I can’t express “pets with three or more labels” with the available pet-search filter syntax, since no array-length/count operator is described.","stopReason":"stop","usage":{"input_tokens":676,"output_tokens":417}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a10","trial":2,"input":{"filter":{"$and":[{"labels":{"$contains":"trained"}},{"labels":{"$contains":"outdoor"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":680,"output_tokens":437},"repair":{"input":{"filter":{"$and":[{"tags":{"$contains":"trained"}},{"tags":{"$contains":"outdoor"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":774,"output_tokens":390},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$and/0/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a12","trial":1,"input":{"filter":{"labels":{"$eq":[]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":296},"repair":{"text":"I can’t answer that with the available pet search tool: “labels” is not a queryable field, so I can’t search for pets with no labels.","stopReason":"stop","usage":{"input_tokens":751,"output_tokens":73},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a11","trial":5,"text":"I can’t express “three or more labels” with the available pet search filter as described.","stopReason":"stop","usage":{"input_tokens":676,"output_tokens":540}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a12","trial":3,"input":{"filter":{"labels":[]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":241},"repair":{"text":"I can’t determine which pets have no labels because `labels` is not a queryable field in the pet search tool.","stopReason":"stop","usage":{"input_tokens":748,"output_tokens":66},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a12","trial":4,"input":{"filter":{"labels":[]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":194},"repair":{"text":"I can’t determine that with the available pet search tool because `labels` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":748,"output_tokens":80},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a12","trial":2,"input":{"filter":{"labels":{"$eq":[]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":354},"repair":{"text":"I can’t search for pets with no labels because `labels` is not a queryable field in the pet search tool.","stopReason":"stop","usage":{"input_tokens":751,"output_tokens":103},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a12","trial":5,"input":{"filter":{"$or":[{"labels":{"$eq":[]}},{"labels":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":380},"repair":{"text":"I can’t determine which pets have no labels because `labels` is not a queryable field in the pet search tool.","stopReason":"stop","usage":{"input_tokens":768,"output_tokens":85},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$or/0/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a14","trial":2,"input":{"filter":{"shelter.kennels":{"$lte":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":118},"repair":{"text":"I can’t express that search with the available pet-store tool: `shelter.kennels` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":772,"output_tokens":95},"problemShown":{"type":"unknown-field","detail":"\"shelter.kennels\" is not a queryable path on this resource.","pointer":"/filter/shelter.kennels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a13","trial":1,"input":{"filter":{"shelter":{"city":{"$eq":"Athens"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":678,"output_tokens":335},"repair":{"text":"I can’t express that search with the available pet-store tool: the shelter city field isn’t queryable, so I can’t reliably find pets housed at a shelter in Athens.","stopReason":"stop","usage":{"input_tokens":762,"output_tokens":294},"problemShown":{"type":"unknown-field","detail":"\"city\" is not a queryable path on this resource.","pointer":"/filter/shelter/city"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a13","trial":4,"input":{"filter":{"shelter":{"city":{"$eq":"Athens"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":678,"output_tokens":241},"repair":{"input":{"filter":{"shelter":{"$eq":"Athens"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":762,"output_tokens":156},"problemShown":{"type":"unknown-field","detail":"\"city\" is not a queryable path on this resource.","pointer":"/filter/shelter/city"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a13","trial":2,"input":{"filter":{"shelter":{"city":{"$eq":"Athens"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":678,"output_tokens":238},"repair":{"text":"I can’t search for pets by a shelter’s city with the available tool: “city” under “shelter” is not a queryable field.","stopReason":"stop","usage":{"input_tokens":762,"output_tokens":278},"problemShown":{"type":"unknown-field","detail":"\"city\" is not a queryable path on this resource.","pointer":"/filter/shelter/city"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a14","trial":1,"input":{"filter":{"shelter":{"kennels":{"$lte":12}}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":243},"repair":{"text":"I can’t answer that with the available pet search tool because “shelter kennels” is not a queryable field.","stopReason":"stop","usage":{"input_tokens":768,"output_tokens":119},"problemShown":{"type":"unknown-field","detail":"\"kennels\" is not a queryable path on this resource.","pointer":"/filter/shelter/kennels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a13","trial":3,"input":{"filter":{"shelter":{"city":{"$eq":"Athens"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":678,"output_tokens":343},"repair":{"text":"I can’t search for pets by shelter city/location: `shelter.city` isn’t a queryable field in the pet search tool.","stopReason":"stop","usage":{"input_tokens":762,"output_tokens":344},"problemShown":{"type":"unknown-field","detail":"\"city\" is not a queryable path on this resource.","pointer":"/filter/shelter/city"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a15","trial":1,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":231}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a15","trial":3,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":155}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a14","trial":4,"input":{"filter":{"shelter.kennels":{"$lte":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":185},"repair":{"text":"I can’t answer that with the available pet-search tool because “shelter kennels” is not a queryable field for this resource.","stopReason":"stop","usage":{"input_tokens":772,"output_tokens":89},"problemShown":{"type":"unknown-field","detail":"\"shelter.kennels\" is not a queryable path on this resource.","pointer":"/filter/shelter.kennels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a15","trial":2,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":250}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a14","trial":3,"input":{"filter":{"shelter.kennels":{"$lte":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":258},"repair":{"input":{"filter":{"shelter":{"kennels":{"$lte":12}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":772,"output_tokens":160},"problemShown":{"type":"unknown-field","detail":"\"shelter.kennels\" is not a queryable path on this resource.","pointer":"/filter/shelter.kennels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a13","trial":5,"input":{"filter":{"shelter":{"city":"Athens"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":678,"output_tokens":409},"repair":{"text":"I can’t answer that with the available pet-search tool because shelter city (`Athens`) is not a queryable field.","stopReason":"stop","usage":{"input_tokens":759,"output_tokens":258},"problemShown":{"type":"unknown-field","detail":"\"city\" is not a queryable path on this resource.","pointer":"/filter/shelter/city"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a14","trial":5,"input":{"filter":{"shelter.kennels":{"$lte":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":201},"repair":{"text":"I can’t search for pets by a shelter’s kennel count because that field isn’t queryable in the pet search tool.","stopReason":"stop","usage":{"input_tokens":772,"output_tokens":125},"problemShown":{"type":"unknown-field","detail":"\"shelter.kennels\" is not a queryable path on this resource.","pointer":"/filter/shelter.kennels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a15","trial":4,"input":{"filter":{"$or":[{"species":{"$eq":"cat"},"status":{"$eq":"available"}},{"species":{"$eq":"dog"},"weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":265}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a15","trial":5,"input":{"filter":{"$or":[{"species":{"$eq":"cat"},"status":{"$eq":"available"}},{"species":{"$eq":"dog"},"weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":234}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a17","trial":1,"input":{"filter":{"$nor":[{"status":"sold"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":187}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a17","trial":2,"input":{"filter":{"$nor":[{"status":"sold"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":128}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a17","trial":3,"input":{"filter":{"$nor":[{"status":"sold"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":177}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a17","trial":4,"input":{"filter":{"$nor":[{"status":"sold"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":121}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a18","trial":1,"input":{"filter":{"neutered":true},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a17","trial":5,"input":{"filter":{"$nor":[{"status":"sold"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":153}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a18","trial":3,"input":{"filter":{"neutered":true},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":69}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a18","trial":2,"input":{"filter":{"neutered":true},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a16","trial":1,"input":{"filter":{"status":"available","$or":[{"shelter":{"city":"Patras"}},{"labels":"trained"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":332},"repair":{"input":{"filter":{"status":"available","$or":[{"shelter":"Patras"},{"labels":"trained"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":787,"output_tokens":261},"problemShown":{"type":"unknown-field","detail":"\"city\" is not a queryable path on this resource.","pointer":"/filter/$or/0/shelter/city"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a18","trial":4,"input":{"filter":{"neutered":true},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a16","trial":5,"input":{"filter":{"$and":[{"status":"available"},{"$or":[{"shelter.city":"Patras"},{"labels":"trained"}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":402},"repair":{"text":"I can’t answer that with the pet search tool as described, because “trained label”/`labels` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":791,"output_tokens":185},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$and/1/$or/1/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a18","trial":5,"input":{"filter":{"neutered":true},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":91}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a16","trial":2,"input":{"filter":{"$and":[{"status":"available"},{"$or":[{"shelter":{"city":"Patras"}},{"labels":"trained"}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":567},"repair":{"input":{"filter":{"$and":[{"status":"available"},{"$or":[{"shelter":"Patras"},{"labels":"trained"}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":796,"output_tokens":175},"problemShown":{"type":"unknown-field","detail":"\"city\" is not a queryable path on this resource.","pointer":"/filter/$and/1/$or/0/shelter/city"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a16","trial":4,"input":{"filter":{"$or":[{"shelter":{"city":{"$eq":"Patras"}}},{"labels":{"$contains":"trained"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":564},"repair":{"input":{"filter":{"$or":[{"shelter":{"$eq":"Patras"}},{"labels":{"$contains":"trained"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":789,"output_tokens":239},"problemShown":{"type":"unknown-field","detail":"\"city\" is not a queryable path on this resource.","pointer":"/filter/$or/0/shelter/city"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a20","trial":2,"input":{"filter":{"status":{"$regex":"^a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":129}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a19","trial":3,"text":"I can’t express “listed price is greater than cost” with the available pet-search filter, because it requires comparing one field to another rather than comparing a field to a fixed value.","stopReason":"stop","usage":{"input_tokens":679,"output_tokens":357}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a20","trial":1,"input":{"filter":{"status":{"$like":"a%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":192}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a19","trial":2,"text":"I can’t express “listed price is greater than cost” with the available pet search tool, because it requires comparing one field to another field.","stopReason":"stop","usage":{"input_tokens":679,"output_tokens":467}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a16","trial":3,"input":{"filter":{"$or":[{"shelter":{"city":{"$eq":"Patras"}}},{"labels":{"$contains":"trained"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":468},"repair":{"input":{"filter":{"status":{"$eq":"available"}},"limit":5},"text":"","stopReason":"tool_calls","usage":{"input_tokens":789,"output_tokens":388},"problemShown":{"type":"unknown-field","detail":"\"city\" is not a queryable path on this resource.","pointer":"/filter/$or/0/shelter/city"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a20","trial":3,"input":{"filter":{"status":{"$like":"a%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":148}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a19","trial":5,"text":"I can’t express that with the available pet search tool, because it only supports filtering fields against fixed values—not comparing one field, like listed price, to another field, like our cost.","stopReason":"stop","usage":{"input_tokens":679,"output_tokens":559}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a20","trial":5,"input":{"filter":{"status":{"$like":"a%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":242}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a20","trial":4,"input":{"filter":{"status":{"$like":"a%"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":267}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a19","trial":4,"input":{"filter":{"listed_price":{"$gt":{"$field":"cost"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":550},"repair":{"input":{"filter":{"price":{"$gt":{"$field":"cost"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":761,"output_tokens":237},"problemShown":{"type":"unknown-field","detail":"\"listed_price\" is not a queryable path on this resource.","pointer":"/filter/listed_price"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"a19","trial":1,"input":{"filter":{"price":{"$gt":{"$field":"cost"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":549},"repair":{"text":"I can’t express that search with the available pet-search tool: the listing/price field needed for “listed for more than they cost us” isn’t queryable, so I can’t reliably return matching pets.","stopReason":"stop","usage":{"input_tokens":758,"output_tokens":319},"problemShown":{"type":"unknown-field","detail":"\"price\" is not a queryable path on this resource.","pointer":"/filter/price"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b01","trial":1,"input":{"filter":{"vaccinations":{"$elemMatch":{"name":{"$eq":"rabies"},"boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":227},"repair":{"text":"I can’t express that correctly with the available pet-search tool.\n\nThe query requires matching two conditions on the same vaccination entry: `name = rabies` and `boosterDue < 2025-01-01`. The tool does not support an array “same element” operator like `$elemMatch`, and using separate conditions could incorrectly match a rabies shot and a different shot’s overdue booster.","stopReason":"stop","usage":{"input_tokens":822,"output_tokens":202},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b01","trial":3,"input":{"filter":{"shots":{"$elemMatch":{"name":{"$eq":"rabies"},"boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":538},"repair":{"text":"I can’t express that query with the available pet-search filter. It would require a same-array-element condition such as “a shot where `name = rabies` and that same shot’s `boosterDue < 2025-01-01`,” but the tool does not support that kind of correlated array matching.","stopReason":"stop","usage":{"input_tokens":820,"output_tokens":154},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/shots/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b01","trial":4,"input":{"filter":{"vaccinations":{"$elemMatch":{"name":{"$eq":"rabies"},"booster_due":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":412},"repair":{"text":"I can’t express that query with the available pet-search tool, because it requires matching two conditions within the same vaccination array item: `name = rabies` and `booster_due < 2025-01-01`. The needed same-element array operator (`$elemMatch`) is not supported, and approximating it could incorrectly match a rabies shot plus an overdue booster from a different shot.","stopReason":"stop","usage":{"input_tokens":822,"output_tokens":179},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b02","trial":1,"input":{"filter":{"$and":[{"shots":{"$elemMatch":{"type":{"$eq":"rabies"}}}},{"shots":{"$elemMatch":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":714,"output_tokens":329},"repair":{"input":{"filter":{"$and":[{"shots.type":{"$eq":"rabies"}},{"shots.boosterDue":{"$lt":"2025-01-01"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":841,"output_tokens":249},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$and/0/shots/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b01","trial":2,"input":{"filter":{"shots":{"$elemMatch":{"type":{"$eq":"rabies"},"boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":567},"repair":{"text":"I can’t express that query with the available pet-search tool: it does not support an array-element match operator needed to ensure the rabies shot and its overdue booster date belong to the same shot record.","stopReason":"stop","usage":{"input_tokens":820,"output_tokens":136},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/shots/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b01","trial":5,"input":{"filter":{"shots":{"$elemMatch":{"name":{"$eq":"rabies"},"boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":569},"repair":{"text":"I can’t express that search with the available pet-store tool: it doesn’t support an array element match operator like `$elemMatch`, which is needed to ensure the `rabies` shot and the overdue `boosterDue` date belong to the same shot record rather than different shots.","stopReason":"stop","usage":{"input_tokens":820,"output_tokens":197},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/shots/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b02","trial":2,"input":{"filter":{"$and":[{"shots":{"$elemMatch":{"name":{"$eq":"rabies"}}}},{"shots":{"$elemMatch":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":714,"output_tokens":449},"repair":{"input":{"filter":{"$and":[{"shots.name":{"$eq":"rabies"}},{"shots.boosterDue":{"$lt":"2025-01-01"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":841,"output_tokens":288},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$and/0/shots/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b03","trial":3,"input":{"filter":{"vaccinated_on":{"$gte":"2025-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":316},"repair":{"text":"I’m sorry, but this search can’t be expressed with the available pet-search fields: vaccination date is not a queryable field.","stopReason":"stop","usage":{"input_tokens":769,"output_tokens":99},"problemShown":{"type":"unknown-field","detail":"\"vaccinated_on\" is not a queryable path on this resource.","pointer":"/filter/vaccinated_on"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b03","trial":2,"input":{"filter":{"vaccinated_on":{"$gte":"2025-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":358},"repair":{"text":"I can’t answer that with the available pet-search tool because vaccination date is not a queryable field.","stopReason":"stop","usage":{"input_tokens":769,"output_tokens":102},"problemShown":{"type":"unknown-field","detail":"\"vaccinated_on\" is not a queryable path on this resource.","pointer":"/filter/vaccinated_on"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b02","trial":3,"input":{"filter":{"$and":[{"shots":{"$elemMatch":{"name":{"$eq":"rabies"}}}},{"shots":{"$elemMatch":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":714,"output_tokens":528},"repair":{"input":{"filter":{"$and":[{"shots.name":{"$eq":"rabies"}},{"shots.boosterDue":{"$lt":"2025-01-01"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":841,"output_tokens":327},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$and/0/shots/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b02","trial":4,"input":{"filter":{"$and":[{"shots":{"$elemMatch":{"name":{"$eq":"rabies"}}}},{"shots":{"$elemMatch":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":714,"output_tokens":581},"repair":{"input":{"filter":{"$and":[{"shots.name":{"$eq":"rabies"}},{"shots.boosterDue":{"$lt":"2025-01-01"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":841,"output_tokens":280},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$and/0/shots/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b03","trial":1,"input":{"filter":{"vaccinated_on":{"$gte":"2025-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":553},"repair":{"text":"I can’t search for pets by vaccination date because the pet search tool does not expose a queryable vaccination-date field.","stopReason":"stop","usage":{"input_tokens":769,"output_tokens":55},"problemShown":{"type":"unknown-field","detail":"\"vaccinated_on\" is not a queryable path on this resource.","pointer":"/filter/vaccinated_on"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b03","trial":4,"input":{"filter":{"vaccinated_on":{"$gte":"2025-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":401},"repair":{"text":"I can’t answer that with the available pet search tool because vaccination date (`vaccinated_on`) is not a queryable field.","stopReason":"stop","usage":{"input_tokens":769,"output_tokens":84},"problemShown":{"type":"unknown-field","detail":"\"vaccinated_on\" is not a queryable path on this resource.","pointer":"/filter/vaccinated_on"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b03","trial":5,"input":{"filter":{"vaccinatedDate":{"$gte":"2025-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":458},"repair":{"text":"I can’t answer that with the available pet search tool because vaccination date is not a queryable field.","stopReason":"stop","usage":{"input_tokens":769,"output_tokens":132},"problemShown":{"type":"unknown-field","detail":"\"vaccinatedDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinatedDate"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b02","trial":5,"input":{"filter":{"$and":[{"vaccinations":{"$elemMatch":{"vaccine":{"$eq":"rabies"}}}},{"vaccinations":{"$elemMatch":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":714,"output_tokens":584},"repair":{"input":{"filter":{"vaccinations.vaccine":{"$eq":"rabies"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":845,"output_tokens":600},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$and/0/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b04","trial":1,"input":{"filter":{"vaccination_history.0":"parvo"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":485},"repair":{"text":"I can’t express “parvo as the very first entry in vaccination history” with the available pet search tool, because array indexing like `vaccination_history.0` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":769,"output_tokens":175},"problemShown":{"type":"unknown-field","detail":"\"vaccination_history.0\" is not a queryable path on this resource.","pointer":"/filter/vaccination_history.0"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b04","trial":4,"input":{"filter":{"vaccination_history.0":{"$eq":"parvo"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":374},"repair":{"text":"I can’t express “parvo as the very first entry in vaccination history” with the available pet search tool, because indexed access like `vaccination_history.0` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":772,"output_tokens":148},"problemShown":{"type":"unknown-field","detail":"\"vaccination_history.0\" is not a queryable path on this resource.","pointer":"/filter/vaccination_history.0"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b04","trial":5,"text":"I can’t express “parvo as the very first entry in vaccination history” with the search tool as described, because it doesn’t specify support for filtering by an array position/first element.","stopReason":"stop","usage":{"input_tokens":683,"output_tokens":560}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b04","trial":2,"input":{"filter":{"vaccination_history.0":"parvo"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":521},"repair":{"text":"I can’t express that query with the pet search tool: `vaccination_history.0` (the first entry of the vaccination history) is not a queryable path.","stopReason":"stop","usage":{"input_tokens":769,"output_tokens":136},"problemShown":{"type":"unknown-field","detail":"\"vaccination_history.0\" is not a queryable path on this resource.","pointer":"/filter/vaccination_history.0"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b05","trial":4,"input":{"filter":{"vaccinations":{"$contains":"parvo"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":204}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b04","trial":3,"input":{"filter":{"vaccinationHistory.0":{"$eq":"parvo"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":551},"repair":{"text":"I can’t express “parvo as the very first entry in vaccination history” with the available pet search tool because indexing into `vaccinationHistory` (e.g. `vaccinationHistory.0`) is not a queryable path.","stopReason":"stop","usage":{"input_tokens":772,"output_tokens":158},"problemShown":{"type":"unknown-field","detail":"\"vaccinationHistory.0\" is not a queryable path on this resource.","pointer":"/filter/vaccinationHistory.0"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b05","trial":2,"input":{"filter":{"vaccinations":{"$contains":"parvo"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":446}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b05","trial":5,"input":{"filter":{"vaccinations":{"$eq":"parvo"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":238}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b06","trial":1,"input":{"filter":{"$not":{"labels":"indoor"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":259},"repair":{"text":"I can’t answer that with the available pet search tool because “labels” is not a queryable field for pets.","stopReason":"stop","usage":{"input_tokens":756,"output_tokens":108},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$not/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b05","trial":1,"input":{"filter":{"vaccinations":{"$elemMatch":{"$or":[{"name":{"$eq":"parvo"}},{"type":{"$eq":"parvo"}},{"vaccine":{"$eq":"parvo"}},{"$eq":"parvo"}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":586},"repair":{"input":{"filter":{"vaccinations":"parvo"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":806,"output_tokens":165},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b05","trial":3,"input":{"filter":{"vaccinations":{"$contains":"parvo"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":548}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b06","trial":2,"input":{"filter":{"labels":{"$nin":["indoor"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":331},"repair":{"input":{"filter":{"tags":{"$nin":["indoor"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":755,"output_tokens":276},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b07","trial":1,"input":{"filter":{"labels":{"$elemMatch":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":280},"repair":{"input":{"filter":{"labels":{"$ne":"indoor"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":775,"output_tokens":191},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/labels/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b06","trial":4,"input":{"filter":{"$not":{"labels":{"$contains":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":297},"repair":{"input":{"filter":{"$not":{"tags":{"$contains":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":759,"output_tokens":274},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$not/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b06","trial":3,"input":{"filter":{"$not":{"labels":{"$contains":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":493},"repair":{"text":"I can’t answer that with the available pet search tool because “labels” is not a queryable field for this resource.","stopReason":"stop","usage":{"input_tokens":759,"output_tokens":156},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$not/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b06","trial":5,"input":{"filter":{"$not":{"labels":{"$contains":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":337},"repair":{"text":"I can’t answer that with the available pet search tool because the pet “labels” field is not queryable.","stopReason":"stop","usage":{"input_tokens":759,"output_tokens":285},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$not/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b07","trial":2,"input":{"filter":{"labels":{"$elemMatch":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":353},"repair":{"input":{"filter":{"labels":{"$ne":"indoor"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":775,"output_tokens":165},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/labels/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b07","trial":3,"input":{"filter":{"labels":{"$elemMatch":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":334},"repair":{"input":{"filter":{"labels":{"$ne":"indoor"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":775,"output_tokens":178},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/labels/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b07","trial":4,"input":{"filter":{"labels":{"$elemMatch":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":355},"repair":{"input":{"filter":{"labels":{"$ne":"indoor"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":775,"output_tokens":293},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/labels/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b08","trial":2,"text":"I can’t express “the first label in their list” with the available pet search filter syntax, because it doesn’t describe positional array access.","stopReason":"stop","usage":{"input_tokens":680,"output_tokens":550}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b08","trial":1,"input":{"filter":{"labels.0":"indoor"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":680,"output_tokens":273},"repair":{"text":"I can’t express “indoor as the first label” with the available pet search tool: the positional path `labels.0` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":760,"output_tokens":216},"problemShown":{"type":"unknown-field","detail":"\"labels.0\" is not a queryable path on this resource.","pointer":"/filter/labels.0"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b08","trial":5,"input":{"filter":{"labels.0":"indoor"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":680,"output_tokens":226},"repair":{"text":"I can’t express “indoor as the first label in their list” with the available pet search tool, because the first-label path (`labels.0`) is not queryable.","stopReason":"stop","usage":{"input_tokens":760,"output_tokens":115},"problemShown":{"type":"unknown-field","detail":"\"labels.0\" is not a queryable path on this resource.","pointer":"/filter/labels.0"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b08","trial":4,"input":{"filter":{"labels.0":"indoor"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":680,"output_tokens":352},"repair":{"text":"I can’t express “indoor as the first label in their list” with the available pet search tool, because the tool does not support querying by list position like `labels.0`.","stopReason":"stop","usage":{"input_tokens":760,"output_tokens":148},"problemShown":{"type":"unknown-field","detail":"\"labels.0\" is not a queryable path on this resource.","pointer":"/filter/labels.0"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b08","trial":3,"input":{"filter":{"labels.0":"indoor"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":680,"output_tokens":432},"repair":{"text":"I can’t express “indoor as the first label in their list” with the available pet search tool, because indexing into `labels` (e.g. `labels.0`) is not a queryable path.","stopReason":"stop","usage":{"input_tokens":760,"output_tokens":148},"problemShown":{"type":"unknown-field","detail":"\"labels.0\" is not a queryable path on this resource.","pointer":"/filter/labels.0"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b07","trial":5,"input":{"filter":{"labels":{"$elemMatch":{"$ne":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":309},"repair":{"text":"I can’t express that exact query with the available pet-search filter operators because the needed “array has at least one element not equal to indoor” operator isn’t available for `labels`.","stopReason":"stop","usage":{"input_tokens":775,"output_tokens":558},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/labels/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b09","trial":1,"input":{"filter":{"vaccinations":{"$elemMatch":{"boosterDate":{"$exists":true,"$ne":null}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":320},"repair":{"input":{"filter":{"vaccinations.boosterDate":{"$exists":true,"$ne":null}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":785,"output_tokens":301},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b09","trial":2,"input":{"filter":{"vaccinations":{"$elemMatch":{"booster_date":{"$exists":true,"$ne":null}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":243},"repair":{"input":{"filter":{"vaccinations.booster_date":{"$exists":true,"$ne":null}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":785,"output_tokens":244},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b09","trial":4,"input":{"filter":{"vaccinations":{"$elemMatch":{"boosterDate":{"$exists":true,"$ne":null}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":432},"repair":{"text":"I can’t express “at least one vaccination with a booster date recorded” with the available pet search tool, because it does not support the needed array/element matching operator for vaccinations.","stopReason":"stop","usage":{"input_tokens":785,"output_tokens":225},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b10","trial":1,"input":{"filter":{"vaccinations":{"$elemMatch":{"$or":[{"boosterDate":{"$exists":false}},{"boosterDate":null}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":393},"repair":{"text":"I can’t express that query with the available pet-search tool: it does not support an array-element operator like `$elemMatch` needed to find pets where at least one vaccination has no booster date recorded.","stopReason":"stop","usage":{"input_tokens":791,"output_tokens":217},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b10","trial":4,"input":{"filter":{"vaccinations":{"$elemMatch":{"$or":[{"boosterDate":{"$eq":null}},{"boosterDate":{"$exists":false}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":330},"repair":{"text":"I can’t express “pets with at least one vaccination with no booster date recorded” using the available pet search tool, because it does not support matching individual elements inside the `vaccinations` array (e.g., no `$elemMatch`).","stopReason":"stop","usage":{"input_tokens":795,"output_tokens":164},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b09","trial":3,"input":{"filter":{"vaccinations.boosterDate":{"$exists":true,"$ne":null}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":430},"repair":{"input":{"filter":{"vaccinations":{"$elemMatch":{"boosterDate":{"$exists":true,"$ne":null}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":774,"output_tokens":404},"problemShown":{"type":"unknown-field","detail":"\"vaccinations.boosterDate\" is not a queryable path on this resource.","pointer":"/filter/vaccinations.boosterDate"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b10","trial":3,"input":{"filter":{"vaccinations":{"$elemMatch":{"$or":[{"boosterDate":null},{"boosterDate":{"$exists":false}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":519},"repair":{"input":{"filter":{"$or":[{"vaccinations.boosterDate":null},{"vaccinations.boosterDate":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":791,"output_tokens":296},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b09","trial":5,"input":{"filter":{"vaccinations":{"$elemMatch":{"boosterDate":{"$exists":true,"$ne":null}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":462},"repair":{"text":"I can’t express that query with the available pet search tool: it does not support an array element predicate for `vaccinations` to require “at least one vaccination with a recorded booster date.”","stopReason":"stop","usage":{"input_tokens":785,"output_tokens":447},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b10","trial":2,"input":{"filter":{"vaccinations":{"$elemMatch":{"$or":[{"boosterDate":null},{"boosterDate":{"$exists":false}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":490},"repair":{"text":"I can’t express this query with the available pet-search tool: it does not support an array “at least one vaccination”/`$elemMatch`-style filter needed to find vaccinations with no booster date recorded.","stopReason":"stop","usage":{"input_tokens":791,"output_tokens":406},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b10","trial":5,"input":{"filter":{"vaccinations":{"$elemMatch":{"boosterDate":{"$eq":null}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":521},"repair":{"text":"I can’t express that search with the available pet-search tool: it does not support an array “at least one vaccination” / `$elemMatch` condition needed to find vaccinations with no booster date recorded.","stopReason":"stop","usage":{"input_tokens":780,"output_tokens":291},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b11","trial":1,"text":"I can’t express that query with the pet-search tool as described. It requires checking every item in a pet’s vaccination history, including treating missing/empty vaccination records as a match, but the available filter schema doesn’t describe array/list operators or vaccination-specific fields needed to do that reliably.","stopReason":"stop","usage":{"input_tokens":692,"output_tokens":579}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b11","trial":3,"text":"I can’t express this query with the available pet-search tool as described, because it requires checking that no vaccination record has a value other than “rabies” while also including pets with no vaccination records.","stopReason":"stop","usage":{"input_tokens":692,"output_tokens":562}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b11","trial":2,"input":{"filter":{"vaccinations":{"$not":{"$elemMatch":{"$ne":"rabies"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":359},"repair":{"input":{"filter":{"vaccinations":{"$not":{"$contains":"rabies"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":793,"output_tokens":389},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/vaccinations/$not/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b12","trial":1,"input":{"filter":{"$nor":[{"labels":"indoor"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":691,"output_tokens":305},"repair":{"text":"I can’t answer that with the available pet search tool: `labels` is not a queryable field, so the requested condition (“no label is indoor,” including empty label lists) cannot be expressed.","stopReason":"stop","usage":{"input_tokens":774,"output_tokens":166},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$nor/0/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b12","trial":3,"input":{"filter":{"labels":{"$not":{"$elemMatch":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":691,"output_tokens":306},"repair":{"input":{"filter":{"labels":{"$nin":["indoor"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":790,"output_tokens":308},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/labels/$not/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b12","trial":2,"input":{"filter":{"$not":{"labels":{"$elemMatch":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":691,"output_tokens":555},"repair":{"input":{"filter":{"labels":{"$nin":["indoor"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":790,"output_tokens":218},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$not/labels/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b11","trial":4,"input":{"filter":{"$not":{"vaccinations":{"$elemMatch":{"$ne":"rabies"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":556},"repair":{"input":{"filter":{"vaccinations":{"$ne":"rabies"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":793,"output_tokens":466},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$not/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b12","trial":4,"input":{"filter":{"labels":{"$not":{"$elemMatch":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":691,"output_tokens":414},"repair":{"input":{"filter":{"labels":{"$nin":["indoor"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":790,"output_tokens":251},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/labels/$not/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b11","trial":5,"input":{"filter":{"$or":[{"vaccinations":{"$exists":false}},{"vaccinations":{"$size":0}},{"$not":{"vaccinations":{"$elemMatch":{"$ne":"rabies"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":525},"repair":{"input":{"filter":{"$not":{"vaccinations":{"$ne":"rabies"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":820,"output_tokens":488},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$or/2/$not/vaccinations/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b12","trial":5,"input":{"filter":{"labels":{"$not":{"$elemMatch":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":691,"output_tokens":530},"repair":{"input":{"filter":{"labels":{"$nin":["indoor"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":790,"output_tokens":202},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/labels/$not/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c01","trial":1,"input":{"filter":{"microchip":{"$exists":true,"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":120}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c01","trial":2,"input":{"filter":{"microchip":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":123}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c01","trial":3,"input":{"filter":{"microchip":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":105}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c01","trial":4,"input":{"filter":{"microchip":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":145}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c01","trial":5,"input":{"filter":{"microchip":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":162}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c02","trial":1,"input":{"filter":{"microchip":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":161}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b13","trial":4,"input":{"filter":{"$or":[{"labels":{"$eq":["indoor"]}},{"labels":{"$eq":[]}},{"labels":{"$exists":false}},{"labels":null}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":493},"repair":{"text":"I can’t answer that with the available pet search tool because “labels” is not a queryable field for this resource.","stopReason":"stop","usage":{"input_tokens":789,"output_tokens":96},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$or/0/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b13","trial":1,"input":{"filter":{"$or":[{"labels":{"$exists":false}},{"labels":{"$eq":null}},{"labels":{"$size":0}},{"labels":{"$not":{"$elemMatch":{"$ne":"indoor"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":584},"repair":{"input":{"filter":{"$or":[{"labels":{"$exists":false}},{"labels":{"$eq":null}},{"labels":{"$eq":[]}},{"labels":{"$subset":["indoor"]}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":814,"output_tokens":435},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$or/3/labels/$not/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c02","trial":3,"input":{"filter":{"microchip":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":141}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b13","trial":2,"input":{"filter":{"$or":[{"labels":{"$eq":[]}},{"labels":{"$eq":null}},{"labels":{"$exists":false}},{"labels":{"$not":{"$elemMatch":{"$ne":"indoor"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":582},"repair":{"input":{"filter":{"$or":[{"labels":{"$eq":["indoor"]}},{"labels":{"$eq":[]}},{"labels":{"$eq":null}},{"labels":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":813,"output_tokens":449},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$or/3/labels/$not/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b13","trial":3,"input":{"filter":{"$or":[{"labels":{"$eq":[]}},{"labels":{"$not":{"$elemMatch":{"$ne":"indoor"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":399},"repair":{"input":{"filter":{"$or":[{"labels":{"$eq":["indoor"]}},{"labels":{"$eq":[]}},{"labels":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":797,"output_tokens":267},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$or/1/labels/$not/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c02","trial":2,"input":{"filter":{"microchip":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":161}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c03","trial":1,"input":{"filter":{"notes":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":130}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c03","trial":2,"input":{"filter":{"notes":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":153}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c02","trial":4,"input":{"filter":{"microchip":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":159}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c03","trial":3,"input":{"filter":{"notes":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":125}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c02","trial":5,"input":{"filter":{"microchip":{"$exists":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":198}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c03","trial":5,"input":{"filter":{"notes":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":141}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c03","trial":4,"input":{"filter":{"notes":{"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":212}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"b13","trial":5,"input":{"filter":{"$or":[{"labels":{"$exists":false}},{"labels":{"$size":0}},{"$nor":[{"labels":{"$elemMatch":{"$ne":"indoor"}}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":578},"repair":{"input":{"filter":{"$or":[{"labels":{"$exists":false}},{"labels":{"$size":0}},{"labels":{"$eq":["indoor"]}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":810,"output_tokens":497},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$or/2/$nor/0/labels/$elemMatch"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c05","trial":2,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":695,"output_tokens":305}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c05","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":695,"output_tokens":410}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c04","trial":4,"input":{"filter":{"shelter.kennel_capacity":null},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":400},"repair":{"text":"I can’t answer that with the available pet search tool because `shelter.kennel_capacity` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":771,"output_tokens":105},"problemShown":{"type":"unknown-field","detail":"\"shelter.kennel_capacity\" is not a queryable path on this resource.","pointer":"/filter/shelter.kennel_capacity"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c04","trial":5,"input":{"filter":{"shelter":{"kennel_capacity":{"$eq":null}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":510},"repair":{"text":"I can’t answer that with the available pet search tool because the shelter’s kennel capacity is not a queryable field.","stopReason":"stop","usage":{"input_tokens":771,"output_tokens":108},"problemShown":{"type":"unknown-field","detail":"\"kennel_capacity\" is not a queryable path on this resource.","pointer":"/filter/shelter/kennel_capacity"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c05","trial":5,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":695,"output_tokens":243}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c04","trial":2,"input":{"filter":{"shelter.kennelCapacity":{"$exists":false}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":548},"repair":{"text":"I can’t express that search with the available pet-search tool: `shelter.kennelCapacity` is not a queryable field on this resource.","stopReason":"stop","usage":{"input_tokens":771,"output_tokens":174},"problemShown":{"type":"unknown-field","detail":"\"shelter.kennelCapacity\" is not a queryable path on this resource.","pointer":"/filter/shelter.kennelCapacity"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c04","trial":1,"input":{"filter":{"$or":[{"shelter.kennelCapacity":{"$eq":null}},{"shelter.kennelCapacity":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":450},"repair":{"text":"I can’t express that search with the available pet-search fields: the shelter kennel capacity field is not queryable on this resource.","stopReason":"stop","usage":{"input_tokens":798,"output_tokens":317},"problemShown":{"type":"unknown-field","detail":"\"shelter.kennelCapacity\" is not a queryable path on this resource.","pointer":"/filter/$or/0/shelter.kennelCapacity"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c05","trial":3,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":695,"output_tokens":520}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c05","trial":4,"input":{"filter":{"$and":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$ne":null}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":695,"output_tokens":564}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c04","trial":3,"input":{"filter":{"$or":[{"shelter.kennelCapacity":{"$eq":null}},{"shelter.kennelCapacity":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":559},"repair":{"text":"I can’t answer that with the available pet-search tool because “shelter kennel capacity” is not a queryable field on the pet records.","stopReason":"stop","usage":{"input_tokens":798,"output_tokens":312},"problemShown":{"type":"unknown-field","detail":"\"shelter.kennelCapacity\" is not a queryable path on this resource.","pointer":"/filter/$or/0/shelter.kennelCapacity"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c08","trial":2,"input":{"filter":{"status":{"$ne":"sold"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":674,"output_tokens":110}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c07","trial":1,"input":{"filter":{"$and":[{"chip":{"$exists":true}},{"$or":[{"chip":{"$ne":"CHIP-001"}},{"chip":{"$eq":null}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":360},"repair":{"text":"I can’t answer that with the available pet search tool: `chip` is not a queryable field on this resource.","stopReason":"stop","usage":{"input_tokens":810,"output_tokens":113},"problemShown":{"type":"unknown-field","detail":"\"chip\" is not a queryable path on this resource.","pointer":"/filter/$and/0/chip"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c08","trial":1,"input":{"filter":{"status":{"$ne":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":674,"output_tokens":140}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c08","trial":3,"input":{"filter":{"status":{"$ne":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":674,"output_tokens":108}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c07","trial":2,"input":{"filter":{"$and":[{"chip":{"$exists":true}},{"$or":[{"chip":{"$ne":"CHIP-001"}},{"chip":{"$eq":null}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":335},"repair":{"text":"I can’t answer that with the available pet search tool: the chip field is not a queryable path on this resource.","stopReason":"stop","usage":{"input_tokens":810,"output_tokens":87},"problemShown":{"type":"unknown-field","detail":"\"chip\" is not a queryable path on this resource.","pointer":"/filter/$and/0/chip"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c07","trial":5,"input":{"filter":{"$and":[{"chip":{"$exists":true}},{"$or":[{"chip":{"$ne":"CHIP-001"}},{"chip":{"$eq":null}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":301},"repair":{"text":"I can’t express that search with the available pet-store tool because `chip` is not a queryable field on this resource.","stopReason":"stop","usage":{"input_tokens":810,"output_tokens":102},"problemShown":{"type":"unknown-field","detail":"\"chip\" is not a queryable path on this resource.","pointer":"/filter/$and/0/chip"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c08","trial":5,"input":{"filter":{"status":{"$ne":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":674,"output_tokens":96}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c07","trial":3,"input":{"filter":{"$and":[{"chip":{"$exists":true}},{"$or":[{"chip":{"$ne":"CHIP-001"}},{"chip":{"$eq":null}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":346},"repair":{"text":"I can’t express that search with the available pet-search tool because `chip` is not a queryable field on this resource.","stopReason":"stop","usage":{"input_tokens":810,"output_tokens":124},"problemShown":{"type":"unknown-field","detail":"\"chip\" is not a queryable path on this resource.","pointer":"/filter/$and/0/chip"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c07","trial":4,"input":{"filter":{"$and":[{"chip":{"$exists":true}},{"$or":[{"chip":{"$ne":"CHIP-001"}},{"chip":{"$eq":null}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":701,"output_tokens":360},"repair":{"text":"I can’t answer that with the available pet search tool: `chip` is not a queryable field on this resource.","stopReason":"stop","usage":{"input_tokens":810,"output_tokens":107},"problemShown":{"type":"unknown-field","detail":"\"chip\" is not a queryable path on this resource.","pointer":"/filter/$and/0/chip"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c08","trial":4,"input":{"filter":{"status":{"$ne":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":674,"output_tokens":170}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c11","trial":2,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":264}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c11","trial":3,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":259}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c11","trial":1,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":329}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c11","trial":4,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":215}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c11","trial":5,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":311}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c12","trial":1,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$exists":false}},{"microchip":{"$eq":null}},{"microchip":{"$eq":""}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":704,"output_tokens":508}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c12","trial":3,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}},{"microchip":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":704,"output_tokens":485}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c12","trial":2,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}},{"microchip":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":704,"output_tokens":573}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c12","trial":4,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}},{"microchip":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":704,"output_tokens":522}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c12","trial":5,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}},{"microchip":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":704,"output_tokens":556}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c13","trial":1,"input":{"filter":{"$or":[{"birthDate":{"$lt":"2020-01-01"}},{"birthDate":{"$eq":null}},{"birthDate":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":575},"repair":{"text":"I can’t express this search with the available pet-store tool because birth date is not a queryable field on the pet records.","stopReason":"stop","usage":{"input_tokens":810,"output_tokens":99},"problemShown":{"type":"unknown-field","detail":"\"birthDate\" is not a queryable path on this resource.","pointer":"/filter/$or/0/birthDate"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c13","trial":3,"input":{"filter":{"$or":[{"birthDate":{"$lt":"2020-01-01"}},{"birthDate":{"$eq":null}},{"birthDate":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":313},"repair":{"text":"I can’t express this search with the available pet-store tool because birth date is not a queryable field on this resource.","stopReason":"stop","usage":{"input_tokens":810,"output_tokens":178},"problemShown":{"type":"unknown-field","detail":"\"birthDate\" is not a queryable path on this resource.","pointer":"/filter/$or/0/birthDate"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c13","trial":2,"input":{"filter":{"$or":[{"birthDate":{"$lt":"2020-01-01"}},{"birthDate":{"$eq":null}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":460},"repair":{"text":"I can’t answer this with the available search tool because birth date is not a queryable field for pets.","stopReason":"stop","usage":{"input_tokens":801,"output_tokens":157},"problemShown":{"type":"unknown-field","detail":"\"birthDate\" is not a queryable path on this resource.","pointer":"/filter/$or/0/birthDate"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d01","trial":3,"input":{"filter":{"notes":{"$type":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":305}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d01","trial":1,"text":"I can’t express “notes value is a number” with the pet search filter as described, because it doesn’t provide a type-check operator.","stopReason":"stop","usage":{"input_tokens":679,"output_tokens":405}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c13","trial":4,"input":{"filter":{"$or":[{"birthDate":{"$lt":"2020-01-01"}},{"birthDate":{"$eq":null}},{"birthDate":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":509},"repair":{"text":"I can’t answer this with the pet-store search tool as available: birth date is not a queryable field on this resource, so I can’t filter pets by “not born on or after 1 January 2020” or include missing birth dates.","stopReason":"stop","usage":{"input_tokens":810,"output_tokens":163},"problemShown":{"type":"unknown-field","detail":"\"birthDate\" is not a queryable path on this resource.","pointer":"/filter/$or/0/birthDate"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d01","trial":2,"input":{"filter":{"notes":{"$type":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":509}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d01","trial":4,"text":"I can’t express “notes value is a number” with the pet search tool as described, because it doesn’t provide a type-check operator for fields.","stopReason":"stop","usage":{"input_tokens":679,"output_tokens":307}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d01","trial":5,"text":"I can’t express “notes value is a number” with the pet search tool as described, because it does not provide a type-check operator for fields.","stopReason":"stop","usage":{"input_tokens":679,"output_tokens":325}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"c13","trial":5,"input":{"filter":{"$or":[{"birthDate":{"$lt":"2020-01-01"}},{"birthDate":{"$eq":null}},{"birthDate":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":402},"repair":{"text":"I can’t answer this with the available pet search tool because the birth date field is not queryable; the tool rejected the filter for birth date.","stopReason":"stop","usage":{"input_tokens":810,"output_tokens":282},"problemShown":{"type":"unknown-field","detail":"\"birthDate\" is not a queryable path on this resource.","pointer":"/filter/$or/0/birthDate"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d02","trial":3,"text":"I can’t express “notes value is a whole number, treating 3.0 as whole” with the available pet-search filter operators.","stopReason":"stop","usage":{"input_tokens":688,"output_tokens":260}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d03","trial":3,"input":{"filter":{"notes":{"$type":"array"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":679,"output_tokens":167}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d03","trial":2,"text":"I can’t express “notes value is a list/array” with the available pet search filter.","stopReason":"stop","usage":{"input_tokens":679,"output_tokens":333}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d02","trial":1,"input":{"filter":{"notes":{"$type":"number","$mod":[1,0]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":391},"repair":{"input":{"filter":{"notes":{"$type":"integer"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":782,"output_tokens":435},"problemShown":{"type":"unsupported-operator","detail":"$mod is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/notes/$mod"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d04","trial":1,"input":{"filter":{"notes":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d02","trial":5,"text":"I can’t express “notes value is a whole number, treating 3.0 as whole” with the available pet-search filter operators.","stopReason":"stop","usage":{"input_tokens":688,"output_tokens":549}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d04","trial":2,"input":{"filter":{"notes":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":111}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d03","trial":1,"text":"I can’t express “notes value is a list” with the pet-search filter as described, because it doesn’t provide a type-check operator for fields.","stopReason":"stop","usage":{"input_tokens":679,"output_tokens":552}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d03","trial":4,"text":"I can’t express “notes value is a list/array” with the available pet search filter as described, because it doesn’t include a type-check operator.","stopReason":"stop","usage":{"input_tokens":679,"output_tokens":397}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d03","trial":5,"text":"I can’t express “notes value is a list” with the available pet-search filter language as described, because it doesn’t provide a type/is-array predicate.","stopReason":"stop","usage":{"input_tokens":679,"output_tokens":315}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d04","trial":3,"input":{"filter":{"notes":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":108}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d04","trial":5,"input":{"filter":{"notes":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":126}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d02","trial":4,"input":{"filter":{"notes":{"$mod":[1,0]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":521},"repair":{"text":"I can’t express “notes value is a whole number, treating 3.0 as whole” with the available pet-search filter operators. The needed numeric-integer/modulo check is not supported for `notes`.","stopReason":"stop","usage":{"input_tokens":777,"output_tokens":233},"problemShown":{"type":"unsupported-operator","detail":"$mod is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/notes/$mod"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d05","trial":3,"input":{"filter":{"notes":{"$eq":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":95}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d02","trial":2,"input":{"filter":{"notes":{"$mod":[1,0]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":549},"repair":{"text":"I can’t express “notes value is a whole number, treating 3.0 as whole” with the available pet search filter. The needed numeric-integer/modulo test isn’t supported.","stopReason":"stop","usage":{"input_tokens":777,"output_tokens":267},"problemShown":{"type":"unsupported-operator","detail":"$mod is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/notes/$mod"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d04","trial":4,"input":{"filter":{"notes":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":147}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d05","trial":2,"input":{"filter":{"notes":{"$eq":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":86}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d05","trial":4,"input":{"filter":{"notes":{"$eq":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d05","trial":1,"input":{"filter":{"notes":{"$eq":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":100}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d05","trial":5,"input":{"filter":{"notes":{"$eq":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":114}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d06","trial":4,"input":{"filter":{"notes":{"$gt":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":142}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d07","trial":1,"input":{"filter":{"notes":{"$gt":"a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":100}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d06","trial":2,"input":{"filter":{"notes":{"$gt":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":132}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d07","trial":2,"input":{"filter":{"notes":{"$gt":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":104}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d06","trial":5,"input":{"filter":{"notes":{"$gt":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":228}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d07","trial":3,"input":{"filter":{"notes":{"$gt":"a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":97}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d07","trial":4,"input":{"filter":{"notes":{"$gt":"a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":126}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d07","trial":5,"input":{"filter":{"notes":{"$gt":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d08","trial":2,"input":{"filter":{"notes":{"$ne":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":146}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d06","trial":1,"input":{"filter":{"$and":[{"notes":{"$type":"number"}},{"notes":{"$gt":2}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":324}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d08","trial":1,"input":{"filter":{"notes":{"$ne":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":146}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d06","trial":3,"input":{"filter":{"notes":{"$gt":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":317}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d09","trial":2,"input":{"filter":{"notes":{"$lte":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":124}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d09","trial":1,"input":{"filter":{"notes":{"$lte":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":148}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d08","trial":5,"input":{"filter":{"notes":{"$ne":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":158}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d08","trial":4,"input":{"filter":{"notes":{"$ne":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":156}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d09","trial":5,"input":{"filter":{"notes":{"$lte":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":117}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d08","trial":3,"input":{"filter":{"notes":{"$ne":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":683,"output_tokens":231}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d09","trial":4,"input":{"filter":{"notes":{"$lte":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":133}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"d09","trial":3,"input":{"filter":{"notes":{"$lte":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":218}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e01","trial":2,"input":{"filter":{"$$rate":{"$gt":5}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":120}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e01","trial":1,"input":{"filter":{"$$rate":{"$gt":5}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":165}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e01","trial":3,"input":{"filter":{"$$rate":{"$gt":5}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":164}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e01","trial":5,"input":{"filter":{"$$rate":{"$gt":5}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":169}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e01","trial":4,"input":{"filter":{"$$rate":{"$gt":5}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":688,"output_tokens":204}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e02","trial":2,"input":{"filter":{"size.raw":{"$eq":"XS"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":113},"repair":{"text":"I can’t express that search with the available pet search tool: it treats `\"size.raw\"` as a query path, and the tool reports that it is not a queryable field/path.","stopReason":"stop","usage":{"input_tokens":778,"output_tokens":137},"problemShown":{"type":"unknown-field","detail":"\"size.raw\" is not a queryable path on this resource.","pointer":"/filter/size.raw"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e03","trial":2,"input":{"filter":{"name":{"$regex":"^C.*o$"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":169}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e02","trial":4,"input":{"filter":{"size.raw":{"$eq":"XS"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":76},"repair":{"text":"I can’t express that search with the available tool. It treats `size.raw` as a query path, and reports that `\"size.raw\"` is not a queryable path; it doesn’t provide a way to query a literal field name containing a dot.","stopReason":"stop","usage":{"input_tokens":778,"output_tokens":162},"problemShown":{"type":"unknown-field","detail":"\"size.raw\" is not a queryable path on this resource.","pointer":"/filter/size.raw"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e03","trial":1,"input":{"filter":{"name":{"$regex":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":196}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e02","trial":5,"input":{"filter":{"size.raw":"XS"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":106},"repair":{"text":"I can’t express that with the available pet search tool: it treats `size.raw` as an unqueryable path/field, so the literal field name `size.raw` is not searchable through this API.","stopReason":"stop","usage":{"input_tokens":775,"output_tokens":46},"problemShown":{"type":"unknown-field","detail":"\"size.raw\" is not a queryable path on this resource.","pointer":"/filter/size.raw"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e03","trial":3,"input":{"filter":{"name":{"$regex":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":163}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e02","trial":1,"input":{"filter":{"size.raw":"XS"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":97},"repair":{"text":"I can’t express that query with the available pet search tool: the tool treats `size.raw` as a query path and reports that `\"size.raw\"` is not a queryable field/path.","stopReason":"stop","usage":{"input_tokens":775,"output_tokens":188},"problemShown":{"type":"unknown-field","detail":"\"size.raw\" is not a queryable path on this resource.","pointer":"/filter/size.raw"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e02","trial":3,"input":{"filter":{"size.raw":{"$eq":"XS"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":118},"repair":{"text":"I can’t express that query with the available pet-search tool: the literal field name `size.raw` is not a queryable field/path on this resource.","stopReason":"stop","usage":{"input_tokens":778,"output_tokens":149},"problemShown":{"type":"unknown-field","detail":"\"size.raw\" is not a queryable path on this resource.","pointer":"/filter/size.raw"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e03","trial":4,"input":{"filter":{"name":{"$regex":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":147}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e04","trial":3,"input":{"filter":{"name":{"$regex":"^[ABC]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":129}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e04","trial":1,"input":{"filter":{"name":{"$regex":"^[ABC]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":184}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e04","trial":5,"input":{"filter":{"name":{"$regex":"^[ABC]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":197}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e03","trial":5,"input":{"filter":{"name":{"$like":"C%o"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":254}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e04","trial":2,"input":{"filter":{"name":{"$regex":"^[ABC]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":280}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e05","trial":1,"input":{"filter":{"name":{"$regex":"^[abcABC]"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":685,"output_tokens":272}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e04","trial":4,"input":{"filter":{"name":{"$regex":"^[ABC]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":684,"output_tokens":284}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e06","trial":2,"input":{"filter":{"name":{"$regex":"s$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":188}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e05","trial":2,"input":{"filter":{"$or":[{"name":{"$ilike":"a%"}},{"name":{"$ilike":"b%"}},{"name":{"$ilike":"c%"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":685,"output_tokens":374}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e06","trial":3,"input":{"filter":{"name":{"$regex":"s$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":196}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e05","trial":3,"input":{"filter":{"name":{"$regex":"^[abc]","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":685,"output_tokens":204},"repair":{"input":{"filter":{"name":{"$regex":"(?i)^[abc]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":777,"output_tokens":98},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e06","trial":4,"input":{"filter":{"name":{"$regex":"s$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":146}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e05","trial":4,"input":{"filter":{"name":{"$regex":"^[abc]","$options":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":685,"output_tokens":221},"repair":{"input":{"filter":{"name":{"$regex":"^[a-cA-C]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":777,"output_tokens":120},"problemShown":{"type":"unsupported-operator","detail":"$options is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/name/$options"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e06","trial":5,"input":{"filter":{"name":{"$regex":"s$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":147}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e05","trial":5,"input":{"filter":{"$or":[{"name":{"$like":"a%"}},{"name":{"$like":"A%"}},{"name":{"$like":"b%"}},{"name":{"$like":"B%"}},{"name":{"$like":"c%"}},{"name":{"$like":"C%"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":685,"output_tokens":446}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"f01","trial":1,"input":{"filter":{"internal_staff_notes":{"$eq":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":168},"repair":{"text":"I can’t answer that with the available pet search tool: `internal_staff_notes` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":762,"output_tokens":27},"problemShown":{"type":"unknown-field","detail":"\"internal_staff_notes\" is not a queryable path on this resource.","pointer":"/filter/internal_staff_notes"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"f01","trial":3,"input":{"filter":{"internal_staff_notes":{"$eq":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":148},"repair":{"text":"I can’t answer that with the pet search tool: `internal_staff_notes` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":762,"output_tokens":26},"problemShown":{"type":"unknown-field","detail":"\"internal_staff_notes\" is not a queryable path on this resource.","pointer":"/filter/internal_staff_notes"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"e06","trial":1,"input":{"filter":{"name":{"$regex":"s$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":497}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"f01","trial":4,"input":{"filter":{"internal_staff_notes":{"$eq":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":189},"repair":{"text":"I can’t answer that with the pet search tool: `internal_staff_notes` is not a queryable field on this resource.","stopReason":"stop","usage":{"input_tokens":762,"output_tokens":29},"problemShown":{"type":"unknown-field","detail":"\"internal_staff_notes\" is not a queryable path on this resource.","pointer":"/filter/internal_staff_notes"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"f02","trial":1,"text":"I can’t express a free-text search for “ada” with the available pet search tool. It only supports structured filter predicates, not free-text search across records.","stopReason":"stop","usage":{"input_tokens":687,"output_tokens":104}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"f02","trial":2,"text":"I can’t express a free-text search across all pet records with the available pet search tool. It only supports structured filters over pet record fields, not general text search.","stopReason":"stop","usage":{"input_tokens":687,"output_tokens":113}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"f01","trial":2,"input":{"filter":{"internal_staff_notes":{"$eq":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":287},"repair":{"text":"I can’t answer that with the available pet search tool because `internal_staff_notes` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":762,"output_tokens":27},"problemShown":{"type":"unknown-field","detail":"\"internal_staff_notes\" is not a queryable path on this resource.","pointer":"/filter/internal_staff_notes"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"f02","trial":4,"text":"I can’t express a free-text search across all pet records with the available pet search tool. It only supports structured filters over pet fields, not a general text query for “ada.”","stopReason":"stop","usage":{"input_tokens":687,"output_tokens":135}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"f01","trial":5,"input":{"filter":{"internalStaffNotes":{"$eq":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":232},"repair":{"text":"I can’t answer that with the available pet-search tool because `internalStaffNotes` is not a queryable field.","stopReason":"stop","usage":{"input_tokens":762,"output_tokens":27},"problemShown":{"type":"unknown-field","detail":"\"internalStaffNotes\" is not a queryable path on this resource.","pointer":"/filter/internalStaffNotes"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"f02","trial":3,"text":"I can’t run a free-text search with the available pet search tool. It only supports structured filter predicates, not searching all record text for “ada.”","stopReason":"stop","usage":{"input_tokens":687,"output_tokens":129}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"f02","trial":5,"text":"I can’t express a free-text search with the available pet search tool. It only supports structured filter predicates over record fields, not full-text search for a term like “ada.”","stopReason":"stop","usage":{"input_tokens":687,"output_tokens":130}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m02","trial":3,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}},{"microchip":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":702,"output_tokens":354}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m01","trial":2,"input":{"filter":{"labels":"indoor"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":218},"repair":{"input":{"filter":{"tags":"indoor"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":749,"output_tokens":245},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m01","trial":4,"input":{"filter":{"labels":"indoor"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":192},"repair":{"input":{"filter":{"tags":"indoor"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":749,"output_tokens":237},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m02","trial":2,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}},{"microchip":{"$eq":""}},{"microchip":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":702,"output_tokens":479}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m03","trial":1,"input":{"filter":{"status":"Available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m03","trial":2,"input":{"filter":{"status":"Available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m03","trial":3,"input":{"filter":{"status":"Available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m02","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":702,"output_tokens":550}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m01","trial":1,"input":{"filter":{"labels":{"$eq":"indoor"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":422},"repair":{"text":"I can’t search by “indoor label” because `labels` is not a queryable field for pets in this tool.","stopReason":"stop","usage":{"input_tokens":752,"output_tokens":191},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m01","trial":3,"input":{"filter":{"labels":{"$contains":"indoor"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":418},"repair":{"text":"I can’t answer that with the available pet search tool because “labels” is not a queryable field.","stopReason":"stop","usage":{"input_tokens":752,"output_tokens":210},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m01","trial":5,"input":{"filter":{"labels":{"$contains":"indoor"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":428},"repair":{"text":"I can’t search for pets by the “indoor” label because labels are not a queryable field in the pet-store search tool.","stopReason":"stop","usage":{"input_tokens":752,"output_tokens":138},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/labels"}}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a01","trial":2,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a01","trial":1,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a01","trial":3,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":44}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m03","trial":5,"input":{"filter":{"status":"Available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":48}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m03","trial":4,"input":{"filter":{"status":"Available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":46}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a01","trial":4,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":41}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a02","trial":1,"input":{"q":"species:cat AND status:available","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a02","trial":2,"input":{"q":"species:cat AND status:available","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a02","trial":3,"input":{"q":"species:cat AND status:available","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a01","trial":5,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":45}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a02","trial":4,"input":{"q":"species:cat AND status:available","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a02","trial":5,"input":{"q":"species:cat AND status:available","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m02","trial":4,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}},{"microchip":{"$eq":""}},{"microchip":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":702,"output_tokens":454}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a03","trial":5,"input":{"q":"status:(available OR pending)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a03","trial":1,"input":{"q":"status:(available OR pending)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a03","trial":3,"input":{"q":"status:(available OR pending)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a03","trial":4,"input":{"q":"status:(available OR pending)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a03","trial":2,"input":{"q":"status:(available OR pending)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a04","trial":1,"input":{"q":"weightKg:[1 TO 5]","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":953,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a04","trial":2,"input":{"q":"weightKg:[1 TO 5]","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":953,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a04","trial":3,"input":{"q":"weightKg:[1 TO 5]","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":953,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a04","trial":4,"input":{"q":"weightKg:[1 TO 5]","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":953,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a05","trial":2,"input":{"q":"born:>=2020-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":30}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a05","trial":3,"input":{"q":"born:>=2020-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":30}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a05","trial":1,"input":{"q":"born:>=2020-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":30}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a05","trial":4,"input":{"q":"born:>=2020-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":30}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a04","trial":5,"input":{"q":"weightKg:[1 TO 5]","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":953,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a05","trial":5,"input":{"q":"born:>=2020-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":30}} +{"provider":"gpt-5.5","arm":"jql-published","caseId":"m02","trial":5,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}},{"microchip":{"$exists":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":702,"output_tokens":573}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a06","trial":1,"input":{"q":"name:b*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":948,"output_tokens":49}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a06","trial":3,"input":{"q":"name:b*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":948,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a06","trial":4,"input":{"q":"name:b*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":948,"output_tokens":49}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a06","trial":2,"input":{"q":"name:b*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":948,"output_tokens":49}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a06","trial":5,"input":{"q":"name:b*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":948,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a07","trial":4,"input":{"q":"name:/o/i","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a07","trial":1,"input":{"q":"name:/o/i","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":89}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a07","trial":3,"input":{"q":"name:/o/i","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a07","trial":2,"input":{"q":"name:/o/i","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":63}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a07","trial":5,"input":{"q":"name:/o/i","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":88}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a08","trial":1,"input":{"q":"name:*50%*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a08","trial":2,"input":{"q":"name:*50%*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a08","trial":4,"input":{"q":"name:*50%*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":64}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a09","trial":1,"input":{"q":"tags:(indoor OR small)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a09","trial":3,"input":{"q":"tags:(indoor OR small)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a09","trial":2,"input":{"q":"tags:(indoor OR small)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a08","trial":5,"input":{"q":"name:*50%*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":76}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a09","trial":4,"input":{"q":"tags:(indoor OR small)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a09","trial":5,"input":{"q":"tags:(indoor OR small)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a08","trial":3,"input":{"q":"name:*50%*","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":87}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a10","trial":1,"input":{"q":"tags:trained AND tags:outdoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":936,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a11","trial":1,"input":{"q":"count(tags):>=3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a10","trial":2,"input":{"q":"tags:trained AND tags:outdoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":936,"output_tokens":70}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a10","trial":3,"input":{"q":"tags:trained AND tags:outdoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":936,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a10","trial":5,"input":{"q":"tags:trained AND tags:outdoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":936,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a11","trial":3,"input":{"q":"count(tags):>=3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a11","trial":2,"input":{"q":"count(tags):>=3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":48}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a10","trial":4,"input":{"q":"tags:trained AND tags:outdoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":936,"output_tokens":62}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a11","trial":4,"input":{"q":"count(tags):>=3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":48}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a11","trial":5,"input":{"q":"count(tags):>=3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":46}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a13","trial":1,"input":{"q":"shelter.city:Athens","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":934,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a13","trial":2,"input":{"q":"shelter.city:Athens","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":934,"output_tokens":49}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a12","trial":2,"input":{"q":"count(tags):0","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":175}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a13","trial":3,"input":{"q":"shelter.city:Athens","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":934,"output_tokens":47}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a13","trial":4,"input":{"q":"shelter.city:Athens","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":934,"output_tokens":51}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a13","trial":5,"input":{"q":"shelter.city:Athens","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":934,"output_tokens":47}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a12","trial":5,"input":{"q":"count(tags):0 OR tags:null OR _missing_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":202}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a12","trial":1,"input":{"q":"count(tags):0 OR tags:null OR _missing_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":246}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a14","trial":2,"input":{"q":"shelter.capacity:<=12","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a12","trial":3,"input":{"q":"count(tags):0 OR tags:null OR _missing_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":221}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a14","trial":3,"input":{"q":"shelter.capacity:<=12","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a12","trial":4,"input":{"q":"count(tags):0 OR tags:null OR _missing_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":273}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a14","trial":4,"input":{"q":"shelter.capacity:<=12","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a14","trial":1,"input":{"q":"shelter.capacity:<=12","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":85}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a14","trial":5,"input":{"q":"shelter.capacity:<=12","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":55}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a15","trial":1,"input":{"q":"(status:available AND species:cat) OR (species:dog AND weightKg:<20)","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":41}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a15","trial":2,"input":{"q":"(status:available AND species:cat) OR (species:dog AND weightKg:<20)","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":41}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a15","trial":3,"input":{"q":"(status:available AND species:cat) OR (species:dog AND weightKg:<20)","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":41}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a15","trial":4,"input":{"q":"(status:available AND species:cat) OR (species:dog AND weightKg:<20)","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":41}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a15","trial":5,"input":{"q":"(status:available AND species:cat) OR (species:dog AND weightKg:<20)","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":41}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a16","trial":2,"input":{"q":"status:available AND (shelter.city:Patras OR tags:trained)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a16","trial":1,"input":{"q":"status:available AND (shelter.city:Patras OR tags:trained)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":72}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a16","trial":4,"input":{"q":"status:available AND (shelter.city:Patras OR tags:trained)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a16","trial":5,"input":{"q":"status:available AND (shelter.city:Patras OR tags:trained)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":68}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a16","trial":3,"input":{"q":"status:available AND (shelter.city:Patras OR tags:trained)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":76}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a18","trial":1,"input":{"q":"neutered:true","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a17","trial":1,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":91}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a18","trial":3,"input":{"q":"neutered:true","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a18","trial":2,"input":{"q":"neutered:true","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":49}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a17","trial":3,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a17","trial":2,"input":{"q":"-status:sold AND -status:pending","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":103}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a18","trial":4,"input":{"q":"neutered:true","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a18","trial":5,"input":{"q":"neutered:true","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a17","trial":5,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":97}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a17","trial":4,"input":{"q":"-status:(sold OR pending)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":104}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a20","trial":2,"input":{"q":"status:a*","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":935,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a20","trial":1,"input":{"q":"status:a*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":935,"output_tokens":55}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a20","trial":3,"input":{"q":"status:a*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":935,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a19","trial":5,"text":"I can’t express “priceCents is greater than costCents” with the pet search tool, because it doesn’t support comparing one field to another.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":131}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a19","trial":3,"text":"I can’t express “priceCents is greater than costCents” with the available pet-store search tool, because it doesn’t support comparing one field to another.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":169}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a20","trial":5,"input":{"q":"status:a*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":935,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a19","trial":4,"text":"I can’t express “priceCents is greater than costCents” with the pet search tool, because it supports comparisons only against fixed values, not field-to-field comparisons.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":195}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a20","trial":4,"input":{"q":"status:a*","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":935,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a19","trial":1,"text":"I can’t express “priceCents is greater than costCents” with the available pet-search tool, because it doesn’t support comparing one field to another.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":271}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"a19","trial":2,"text":"I can’t express “priceCents is greater than costCents” with the pet search tool, because it doesn’t support comparing one field to another.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":284}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b02","trial":1,"input":{"q":"vaccinations.vaccine:rabies AND vaccinations.boosterDue:<2025-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":970,"output_tokens":119}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b02","trial":2,"input":{"q":"vaccinations.vaccine:rabies AND vaccinations.boosterDue:<2025-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":970,"output_tokens":137}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b02","trial":3,"input":{"q":"vaccinations.vaccine:rabies AND vaccinations.boosterDue:<2025-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":970,"output_tokens":114}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b01","trial":1,"text":"I can’t express that requirement with the available search tool.\n\nIt can search `vaccinations.vaccine:rabies` and `vaccinations.boosterDue:<2025-01-01`, but separate clauses on vaccination lists may match different vaccination entries. The tool doesn’t provide a way to require that the overdue booster date belongs to the same rabies vaccination record.","stopReason":"stop","usage":{"input_tokens":965,"output_tokens":196}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b01","trial":2,"text":"I can’t express that safely with the available search tool. It supports searching `vaccinations.vaccine` and `vaccinations.boosterDue`, but separate clauses on vaccination fields may match different vaccination entries, so it can’t guarantee the overdue booster belongs to the rabies shot itself.","stopReason":"stop","usage":{"input_tokens":965,"output_tokens":191}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b01","trial":4,"text":"I can’t express that accurately with the available search tool.\n\nIt supports searching `vaccinations.vaccine:rabies` and `vaccinations.boosterDue:<2025-01-01`, but separate clauses on vaccination fields may match different vaccination entries for the same pet. Since you need the booster date to belong to the rabies shot itself, the tool doesn’t provide a way to enforce that nested same-shot relationship.","stopReason":"stop","usage":{"input_tokens":965,"output_tokens":200}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b01","trial":5,"text":"I can’t express that with the available search tool. It can search `vaccinations.vaccine:rabies` and `vaccinations.boosterDue:<2025-01-01`, but it cannot ensure both conditions apply to the same vaccination record rather than different shots for the same pet.","stopReason":"stop","usage":{"input_tokens":965,"output_tokens":185}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b02","trial":4,"input":{"q":"vaccinations.vaccine:rabies AND vaccinations.boosterDue:<2025-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":970,"output_tokens":90}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b02","trial":5,"input":{"q":"vaccinations.vaccine:rabies AND vaccinations.boosterDue:<2025-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":970,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b03","trial":1,"input":{"q":"vaccinations.administeredAt:>=2025-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":82}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b03","trial":5,"input":{"q":"vaccinations.administeredAt:>=2025-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":35}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b03","trial":4,"input":{"q":"vaccinations.administeredAt:>=2025-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b03","trial":2,"input":{"q":"vaccinations.administeredAt:>=2025-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":81}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b03","trial":3,"input":{"q":"vaccinations.administeredAt:>=2025-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b01","trial":3,"text":"I can’t express that query with the available search tool. It can search `vaccinations.vaccine:rabies` and `vaccinations.boosterDue:<2025-01-01`, but it cannot require both conditions to apply to the same vaccination record.","stopReason":"stop","usage":{"input_tokens":965,"output_tokens":345}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b04","trial":1,"input":{"q":"first(vaccinations.vaccine):parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b05","trial":1,"input":{"q":"vaccinations.vaccine:parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b05","trial":2,"input":{"q":"vaccinations.vaccine:parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b04","trial":2,"input":{"q":"first(vaccinations.vaccine):parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b04","trial":3,"input":{"q":"first(vaccinations.vaccine):parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":89}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b04","trial":4,"input":{"q":"first(vaccinations.vaccine):parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":95}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b05","trial":3,"input":{"q":"vaccinations.vaccine:parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b04","trial":5,"input":{"q":"first(vaccinations.vaccine):parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":95}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b05","trial":4,"input":{"q":"vaccinations.vaccine:parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b05","trial":5,"input":{"q":"vaccinations.vaccine:parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b06","trial":1,"input":{"q":"-tags:indoor OR tags:null OR -_exists_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":132}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b06","trial":4,"input":{"q":"-tags:indoor OR tags:null OR -_exists_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":175}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b06","trial":5,"input":{"q":"-tags:indoor OR tags:null OR -_exists_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":173}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b06","trial":2,"input":{"q":"(-tags:indoor OR tags:null OR -_exists_:tags)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":198}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b07","trial":2,"input":{"q":"tags:/^(?!indoor$).+/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":202}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b06","trial":3,"input":{"q":"-tags:indoor OR tags:null OR -_exists_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":252}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b08","trial":1,"input":{"q":"first(tags):indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":936,"output_tokens":47}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b08","trial":2,"input":{"q":"first(tags):indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":936,"output_tokens":46}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b07","trial":3,"input":{"q":"tags:/^(?!indoor$).+/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":242}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b07","trial":1,"input":{"q":"tags:/^(?!indoor$).+/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":290}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b08","trial":3,"input":{"q":"first(tags):indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":936,"output_tokens":48}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b08","trial":4,"input":{"q":"first(tags):indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":936,"output_tokens":46}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b08","trial":5,"input":{"q":"first(tags):indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":936,"output_tokens":46}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b07","trial":4,"input":{"q":"tags:/^(?!indoor$).+/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":230}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b09","trial":1,"input":{"q":"_exists_:vaccinations.boosterDue","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":77}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b09","trial":3,"input":{"q":"_exists_:vaccinations.boosterDue","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":76}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b09","trial":2,"input":{"q":"_exists_:vaccinations.boosterDue","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b09","trial":5,"input":{"q":"_exists_:vaccinations.boosterDue","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b09","trial":4,"input":{"q":"_exists_:vaccinations.boosterDue","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b07","trial":5,"input":{"q":"tags:/^(?!indoor$).+/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":292}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b10","trial":2,"input":{"q":"vaccinations.boosterDue:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":126}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b11","trial":1,"input":{"q":"(-vaccinations.vaccine:(distemper OR parvo) OR vaccinations.vaccine:null OR -_exists_:vaccinations.vaccine)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":948,"output_tokens":394}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b10","trial":1,"input":{"q":"vaccinations.boosterDue:null OR _missing_:vaccinations.boosterDue","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":554}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b11","trial":2,"input":{"q":"-vaccinations.vaccine:(distemper OR parvo) OR -_exists_:vaccinations.vaccine","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":948,"output_tokens":448}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b10","trial":4,"input":{"q":"vaccinations.boosterDue:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":545}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b10","trial":5,"text":"I’m sorry, but that query can’t be expressed precisely with the available search tool.","stopReason":"stop","usage":{"input_tokens":937,"output_tokens":539}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b11","trial":3,"input":{"q":"(-(vaccinations.vaccine:distemper OR vaccinations.vaccine:parvo) OR _missing_:vaccinations.vaccine OR vaccinations.vaccine:null)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":948,"output_tokens":569}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b10","trial":3,"text":"I can’t express that query exactly with the available search syntax.\n\nThe tool can find pets where the entire `vaccinations.boosterDue` field is missing or blank, but it can’t reliably find pets where **one vaccination entry** lacks a booster date while another vaccination entry may have one.","stopReason":"stop","usage":{"input_tokens":937,"output_tokens":580}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b11","trial":4,"input":{"q":"(-vaccinations.vaccine:(distemper OR parvo)) OR _missing_:vaccinations.vaccine","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":948,"output_tokens":559}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b12","trial":3,"input":{"q":"-tags:indoor OR count(tags):0","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":947,"output_tokens":287}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b12","trial":2,"input":{"q":"-tags:indoor OR count(tags):0","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":947,"output_tokens":339}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b12","trial":1,"input":{"q":"-tags:indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":947,"output_tokens":362}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b12","trial":4,"input":{"q":"count(tags):0 OR -tags:indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":947,"output_tokens":338}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b12","trial":5,"input":{"q":"count(tags):0 OR -tags:indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":947,"output_tokens":307}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c01","trial":1,"input":{"q":"microchip:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b11","trial":5,"input":{"q":"(-vaccinations.vaccine:distemper AND -vaccinations.vaccine:parvo) OR _missing_:vaccinations.vaccine","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":948,"output_tokens":565}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c01","trial":2,"input":{"q":"microchip:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":42}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c01","trial":3,"input":{"q":"microchip:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c01","trial":4,"input":{"q":"microchip:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c01","trial":5,"input":{"q":"microchip:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b13","trial":1,"input":{"q":"-tags:/^(?!indoor$).+/ OR _missing_:tags OR tags:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":557}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c02","trial":1,"input":{"q":"_missing_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":935,"output_tokens":49}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c02","trial":2,"input":{"q":"_missing_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":935,"output_tokens":45}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c02","trial":3,"input":{"q":"_missing_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":935,"output_tokens":47}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c02","trial":5,"input":{"q":"_missing_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":935,"output_tokens":52}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c03","trial":1,"input":{"q":"_exists_:notes","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c03","trial":2,"input":{"q":"_exists_:notes","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c02","trial":4,"input":{"q":"_missing_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":935,"output_tokens":59}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b13","trial":2,"input":{"q":"(tags:indoor AND -tags:/^(?!indoor$).+/) OR count(tags):0 OR tags:null OR _missing_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":542}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c03","trial":5,"input":{"q":"_exists_:notes","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c03","trial":3,"input":{"q":"_exists_:notes","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c03","trial":4,"input":{"q":"_exists_:notes","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":42}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c04","trial":1,"input":{"q":"_missing_:shelter.capacity","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":72}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b13","trial":4,"input":{"q":"(tags:indoor AND -tags:/^(?!indoor$).+/) OR count(tags):0 OR tags:null OR _missing_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":542}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c04","trial":2,"input":{"q":"_missing_:shelter.capacity","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b13","trial":3,"input":{"q":"(tags:indoor AND -tags:/^(?!indoor$).+/) OR count(tags):0 OR tags:null OR _missing_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":569}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c04","trial":4,"input":{"q":"_missing_:shelter.capacity","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c04","trial":5,"input":{"q":"_missing_:shelter.capacity","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":77}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c04","trial":3,"input":{"q":"_missing_:shelter.capacity","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"b13","trial":5,"input":{"q":"(tags:indoor AND count(tags):1) OR count(tags):0 OR tags:null OR _missing_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":563}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c05","trial":3,"input":{"q":"-microchip:CHIP-001","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":951,"output_tokens":214}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c07","trial":2,"input":{"q":"-microchip:CHIP-001 OR microchip:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":957,"output_tokens":130}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c05","trial":1,"input":{"q":"_exists_:microchip AND -microchip:CHIP-001","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":951,"output_tokens":279}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c07","trial":1,"input":{"q":"(-microchip:CHIP-001 OR microchip:null)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":957,"output_tokens":185}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c05","trial":2,"input":{"q":"_exists_:microchip AND -microchip:null AND -microchip:CHIP-001","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":951,"output_tokens":317}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c05","trial":4,"input":{"q":"-microchip:CHIP-001","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":951,"output_tokens":242}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c05","trial":5,"input":{"q":"-microchip:CHIP-001 AND -microchip:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":951,"output_tokens":333}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c07","trial":3,"input":{"q":"-microchip:CHIP-001 OR microchip:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":957,"output_tokens":337}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c08","trial":2,"input":{"q":"status:(available OR pending)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":930,"output_tokens":97}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c07","trial":4,"input":{"q":"(-microchip:CHIP-001 OR microchip:null)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":957,"output_tokens":189}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c08","trial":3,"input":{"q":"status:(available OR pending)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":930,"output_tokens":93}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c08","trial":1,"input":{"q":"status:(available OR pending)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":930,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c08","trial":4,"input":{"q":"-status:sold","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":930,"output_tokens":68}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c08","trial":5,"input":{"q":"status:(available OR pending)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":930,"output_tokens":96}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c07","trial":5,"input":{"q":"-microchip:CHIP-001 OR microchip:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":957,"output_tokens":277}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c11","trial":1,"input":{"q":"weightKg:<1 OR weightKg:>5","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":103}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c11","trial":2,"input":{"q":"weightKg:<1 OR weightKg:>5","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c11","trial":3,"input":{"q":"weightKg:<1 OR weightKg:>5","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c11","trial":4,"input":{"q":"weightKg:<1 OR weightKg:>5","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":92}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c11","trial":5,"input":{"q":"weightKg:<1 OR weightKg:>5","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":76}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c12","trial":2,"input":{"q":"-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":960,"output_tokens":121}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c12","trial":5,"input":{"q":"-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":960,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c12","trial":3,"input":{"q":"(-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":960,"output_tokens":120}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c12","trial":4,"input":{"q":"-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":960,"output_tokens":155}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c12","trial":1,"input":{"q":"-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":960,"output_tokens":205}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c13","trial":2,"input":{"q":"born:<2020-01-01 OR born:null OR -_exists_:born","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":955,"output_tokens":170}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c13","trial":1,"input":{"q":"born:<2020-01-01 OR born:null OR _missing_:born","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":955,"output_tokens":170}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c13","trial":3,"input":{"q":"born:<2020-01-01 OR born:null OR -_exists_:born","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":955,"output_tokens":147}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c13","trial":4,"input":{"q":"born:<2020-01-01 OR born:null OR _missing_:born","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":955,"output_tokens":152}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"c13","trial":5,"input":{"q":"born:<2020-01-01 OR born:null OR _missing_:born","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":955,"output_tokens":171}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d01","trial":1,"text":"I can’t express “notes value is a number” with the available pet-search syntax; it doesn’t provide a type-check operator for unstructured fields like `notes`.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":173}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d02","trial":1,"input":{"q":"notes:/^-?\\d+(?:\\.0+)?$/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":215}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d01","trial":5,"text":"I can’t express “notes value is a number” with the available pet search tool, because it doesn’t provide a type-check predicate for fields.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":204}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d02","trial":2,"input":{"q":"notes:/^-?\\d+(?:\\.0+)?$/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":167}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d01","trial":3,"text":"I can’t express “notes value is a number” with the pet-search tool as described; it doesn’t provide a type-check predicate for fields like `notes`.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":325}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d01","trial":4,"text":"I can’t express “notes value is a number” with the available pet search syntax, since it doesn’t provide a type-check operator for fields.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":265}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d01","trial":2,"text":"I can’t express “notes value is a number” with the available pet-search query syntax, because it doesn’t provide a way to test a field’s data type.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":318}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d02","trial":4,"input":{"q":"notes:/^-?\\d+(?:\\.0+)?$/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":163}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d03","trial":1,"text":"I can’t express “notes value is a list” with the available pet search tool.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":119}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d04","trial":1,"input":{"q":"notes:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":40}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d03","trial":2,"text":"I can’t express “notes value is a list” with the available pet search tool. It supports field matching, presence/missing/null checks, comparisons, and text patterns, but not type checks for unstructured fields like `notes`.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":130}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d03","trial":3,"text":"I can’t express “notes value is a list” with the available pet-search tool, because it doesn’t support querying by field value type.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":109}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d04","trial":2,"input":{"q":"notes:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":44}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d02","trial":5,"input":{"q":"notes:/^-?\\d+(?:\\.0+)?$/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":206}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d04","trial":5,"input":{"q":"notes:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":47}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d03","trial":5,"text":"I can’t express “notes value is a list” with the available pet-search tool, since it doesn’t support querying by the data type of a field.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":129}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d03","trial":4,"text":"I can’t express “notes value is a list” with the available pet-store search tool. It supports field matching, ranges, presence/missing/null checks, and list counts only for specific multi-valued fields like `tags` and `vaccinations`, not type checks on `notes`.","stopReason":"stop","usage":{"input_tokens":935,"output_tokens":168}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d04","trial":3,"input":{"q":"notes:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":44}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d02","trial":3,"input":{"q":"notes:/^-?\\d+(?:\\.0+)?$/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":429}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d04","trial":4,"input":{"q":"notes:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d05","trial":1,"input":{"q":"notes:3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d05","trial":4,"input":{"q":"notes:3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d05","trial":2,"input":{"q":"notes:3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":88}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d05","trial":5,"input":{"q":"notes:3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d05","trial":3,"input":{"q":"notes:3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":113}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d06","trial":1,"input":{"q":"notes:>2","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":91}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d06","trial":2,"input":{"q":"notes:>2","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":97}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d06","trial":3,"input":{"q":"notes:>2","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d06","trial":4,"input":{"q":"notes:>2","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":92}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d07","trial":3,"input":{"q":"notes:>a","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":95}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d07","trial":1,"input":{"q":"notes:>a","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":138}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d07","trial":2,"input":{"q":"notes:>a","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":147}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d07","trial":4,"input":{"q":"notes:>a","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":132}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d07","trial":5,"input":{"q":"notes:>a","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":128}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d06","trial":5,"input":{"q":"notes:>2","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":172}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d08","trial":1,"input":{"q":"-notes:3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":170}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d08","trial":2,"input":{"q":"-notes:3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":153}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d08","trial":4,"input":{"q":"-notes:3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":107}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d09","trial":1,"input":{"q":"notes:<=2","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":132}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d09","trial":3,"input":{"q":"notes:<=2","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":138}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d09","trial":2,"input":{"q":"notes:<=2","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":159}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e01","trial":1,"input":{"q":"$rate:>5","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d09","trial":4,"input":{"q":"notes:<=2","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":148}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d08","trial":5,"input":{"q":"-notes:3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":240}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d08","trial":3,"input":{"q":"-notes:3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":939,"output_tokens":296}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"d09","trial":5,"input":{"q":"notes:<=2","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":158}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e01","trial":3,"input":{"q":"$rate:>5","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e02","trial":1,"input":{"q":"\"size.raw\":XS","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":955,"output_tokens":50}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e02","trial":2,"input":{"q":"\"size.raw\":XS","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":955,"output_tokens":42}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e02","trial":3,"input":{"q":"\"size.raw\":XS","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":955,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e01","trial":5,"input":{"q":"$rate:>5","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":86}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e02","trial":4,"input":{"q":"\"size.raw\":XS","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":955,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e01","trial":4,"input":{"q":"$rate:>5","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":127}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e02","trial":5,"input":{"q":"\"size.raw\":XS","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":955,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e01","trial":2,"input":{"q":"\"$rate\":>5","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":944,"output_tokens":159}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e03","trial":1,"input":{"q":"name:C*o","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e03","trial":3,"input":{"q":"name:C*o","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e03","trial":2,"input":{"q":"name:C*o","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":69}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e03","trial":5,"input":{"q":"name:C*o","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":70}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e04","trial":2,"input":{"q":"name:/^[ABC]/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e04","trial":1,"input":{"q":"name:/^[ABC]/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e04","trial":3,"input":{"q":"name:/^[ABC]/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e03","trial":4,"input":{"q":"name:/^C.*o$/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":93}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e04","trial":4,"input":{"q":"name:/^[ABC]/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e04","trial":5,"input":{"q":"name:/^[ABC]/","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":940,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e05","trial":2,"input":{"q":"name:/^[a-c]/i","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":941,"output_tokens":49}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e05","trial":3,"input":{"q":"name:/^[a-c]/i","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":941,"output_tokens":62}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e05","trial":1,"input":{"q":"name:/^[a-c]/i","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":941,"output_tokens":70}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e05","trial":4,"input":{"q":"name:/^[a-c]/i","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":941,"output_tokens":50}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e05","trial":5,"input":{"q":"name:/^[a-c]/i","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":941,"output_tokens":50}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e06","trial":1,"input":{"q":"name:*s","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":72}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e06","trial":2,"input":{"q":"name:*s","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e06","trial":3,"input":{"q":"name:*s","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e06","trial":4,"input":{"q":"name:*s","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"e06","trial":5,"input":{"q":"name:/s$/i","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":91}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"f01","trial":1,"input":{"q":"notes:anything","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":84}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"f01","trial":2,"input":{"q":"notes:anything","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"f01","trial":4,"input":{"q":"notes:anything","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":90}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"f01","trial":3,"input":{"q":"notes:anything","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":116}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"f01","trial":5,"input":{"q":"notes:anything","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":104}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m01","trial":1,"input":{"q":"tags:indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"f02","trial":3,"input":{"q":"ada","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":943,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m01","trial":2,"input":{"q":"tags:indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m01","trial":3,"input":{"q":"tags:indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":44}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m01","trial":4,"input":{"q":"tags:indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":44}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m01","trial":5,"input":{"q":"tags:indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":25}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m02","trial":2,"input":{"q":"(-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":958,"output_tokens":93}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m02","trial":1,"input":{"q":"-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":958,"output_tokens":91}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m02","trial":3,"input":{"q":"-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":958,"output_tokens":87}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m02","trial":4,"input":{"q":"(-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":958,"output_tokens":119}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m03","trial":1,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m02","trial":5,"input":{"q":"(-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":958,"output_tokens":96}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"f02","trial":1,"text":"I can’t express a true free-text search across all pet records with the available tool. It only documents fielded queries like `name:*ada*` or `notes:*ada*`, not an all-fields search for `ada`.","stopReason":"stop","usage":{"input_tokens":943,"output_tokens":393}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"f02","trial":2,"input":{"q":"ada","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":943,"output_tokens":389}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m03","trial":2,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m03","trial":3,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":50}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a01","trial":1,"input":{"where":{"status":{"op":"is","value":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m03","trial":4,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":51}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a01","trial":2,"input":{"where":{"status":{"op":"is","value":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"m03","trial":5,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":59}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a01","trial":4,"input":{"where":{"status":{"op":"is","value":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a01","trial":3,"input":{"where":{"status":{"op":"is","value":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":48}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"f02","trial":4,"input":{"q":"ada","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":943,"output_tokens":509}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a02","trial":1,"input":{"where":{"species":{"op":"is","value":"cat"},"status":{"op":"is","value":"available"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":38}} +{"provider":"gpt-5.5","arm":"search-string","caseId":"f02","trial":5,"input":{"q":"ada","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":943,"output_tokens":467}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a01","trial":5,"input":{"where":{"status":{"op":"is","value":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":51}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a02","trial":4,"input":{"where":{"species":{"op":"is","value":"cat"},"status":{"op":"is","value":"available"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":38}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a02","trial":3,"input":{"where":{"species":{"op":"is","value":"cat"},"status":{"op":"is","value":"available"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":42}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a02","trial":2,"input":{"where":{"species":{"op":"is","value":"cat"},"status":{"op":"is","value":"available"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":42}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a02","trial":5,"input":{"where":{"species":{"op":"is","value":"cat"},"status":{"op":"is","value":"available"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":38}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a03","trial":1,"input":{"where":{"status":{"op":"anyof","value":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":67}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a03","trial":2,"input":{"where":{"status":{"op":"anyof","value":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":63}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a03","trial":3,"input":{"where":{"status":{"op":"anyof","value":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":67}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a03","trial":4,"input":{"where":{"status":{"op":"anyof","value":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":63}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a04","trial":2,"input":{"where":{"weightKg":{"op":"between","value":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1135,"output_tokens":35}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a04","trial":3,"input":{"where":{"weightKg":{"op":"between","value":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1135,"output_tokens":35}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a04","trial":1,"input":{"where":{"weightKg":{"op":"between","value":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1135,"output_tokens":35}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a03","trial":5,"input":{"where":{"status":{"op":"anyof","value":["available","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a04","trial":5,"input":{"where":{"weightKg":{"op":"between","value":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1135,"output_tokens":35}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a04","trial":4,"input":{"where":{"weightKg":{"op":"between","value":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1135,"output_tokens":35}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a05","trial":5,"input":{"where":{"born":{"op":"gte","value":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":36}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a05","trial":3,"input":{"where":{"born":{"op":"gte","value":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":36}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a05","trial":1,"input":{"where":{"born":{"op":"gte","value":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":36}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a05","trial":4,"input":{"where":{"born":{"op":"gte","value":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":36}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a05","trial":2,"input":{"where":{"born":{"op":"gte","value":"2020-01-01"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":36}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a06","trial":1,"input":{"where":{"name":{"op":"prefix","value":"b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1130,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a06","trial":3,"input":{"where":{"name":{"op":"prefix","value":"b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1130,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a06","trial":4,"input":{"where":{"name":{"op":"prefix","value":"b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1130,"output_tokens":64}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a07","trial":1,"input":{"where":{"name":{"op":"ilike","value":"%o%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":34}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a07","trial":3,"input":{"where":{"name":{"op":"ilike","value":"%o%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":55}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a07","trial":2,"input":{"where":{"name":{"op":"ilike","value":"%o%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":55}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a06","trial":2,"input":{"where":{"name":{"op":"prefix","value":"b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1130,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a06","trial":5,"input":{"where":{"name":{"op":"prefix","value":"b"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1130,"output_tokens":68}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a07","trial":4,"input":{"where":{"name":{"op":"ilike","value":"%o%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":60}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a07","trial":5,"input":{"where":{"name":{"op":"ilike","value":"%o%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a08","trial":3,"input":{"where":{"name":{"op":"substring","value":"50%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":63}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a08","trial":4,"input":{"where":{"name":{"op":"substring","value":"50%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":64}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a08","trial":1,"input":{"where":{"name":{"op":"substring","value":"50%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a08","trial":2,"input":{"where":{"name":{"op":"substring","value":"50%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a09","trial":1,"input":{"where":{"tags":{"op":"some","value":{"op":"anyof","value":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":67}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a08","trial":5,"input":{"where":{"name":{"op":"substring","value":"50%"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":64}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a09","trial":2,"input":{"where":{"tags":{"op":"some","value":{"op":"anyof","value":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":42}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a09","trial":3,"input":{"where":{"tags":{"op":"some","value":{"op":"anyof","value":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":76}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a09","trial":4,"input":{"where":{"tags":{"op":"some","value":{"op":"anyof","value":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":60}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a10","trial":1,"input":{"where":{"all":[{"tags":{"op":"some","value":{"op":"is","value":"trained"}}},{"tags":{"op":"some","value":{"op":"is","value":"outdoor"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1118,"output_tokens":104}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a10","trial":3,"input":{"where":{"all":[{"tags":{"op":"some","value":{"op":"is","value":"trained"}}},{"tags":{"op":"some","value":{"op":"is","value":"outdoor"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1118,"output_tokens":107}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a09","trial":5,"input":{"where":{"tags":{"op":"some","value":{"op":"anyof","value":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":86}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a10","trial":4,"input":{"where":{"all":[{"tags":{"op":"some","value":{"op":"is","value":"trained"}}},{"tags":{"op":"some","value":{"op":"is","value":"outdoor"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1118,"output_tokens":108}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a10","trial":5,"input":{"where":{"all":[{"tags":{"op":"some","value":{"op":"is","value":"trained"}}},{"tags":{"op":"some","value":{"op":"is","value":"outdoor"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1118,"output_tokens":111}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a10","trial":2,"input":{"where":{"all":[{"tags":{"op":"some","value":{"op":"is","value":"trained"}}},{"tags":{"op":"some","value":{"op":"is","value":"outdoor"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1118,"output_tokens":120}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a11","trial":2,"input":{"where":{"tags":{"op":"count","value":{"op":"gte","value":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":70}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a11","trial":1,"input":{"where":{"tags":{"op":"count","value":{"op":"gte","value":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":86}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a11","trial":3,"input":{"where":{"tags":{"op":"count","value":{"op":"gte","value":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a11","trial":5,"input":{"where":{"tags":{"op":"count","value":{"op":"gte","value":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a11","trial":4,"input":{"where":{"tags":{"op":"count","value":{"op":"gte","value":3}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":70}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a12","trial":1,"input":{"where":{"tags":{"op":"count","value":{"op":"is","value":0}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a13","trial":1,"input":{"where":{"shelter.city":{"op":"is","value":"Athens"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1116,"output_tokens":52}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a13","trial":3,"input":{"where":{"shelter.city":{"op":"is","value":"Athens"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1116,"output_tokens":35}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a12","trial":4,"input":{"where":{"tags":{"op":"count","value":{"op":"is","value":0}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":85}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a13","trial":2,"input":{"where":{"shelter.city":{"op":"is","value":"Athens"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1116,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a13","trial":4,"input":{"where":{"shelter.city":{"op":"is","value":"Athens"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1116,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a14","trial":1,"input":{"where":{"shelter.capacity":{"op":"lte","value":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a13","trial":5,"input":{"where":{"shelter.city":{"op":"is","value":"Athens"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1116,"output_tokens":58}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a12","trial":5,"input":{"where":{"any":[{"tags":{"op":"count","value":{"op":"is","value":0}}},{"tags":{"op":"exists","value":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":207}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a14","trial":3,"input":{"where":{"shelter.capacity":{"op":"lte","value":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a14","trial":2,"input":{"where":{"shelter.capacity":{"op":"lte","value":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":64}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a12","trial":3,"input":{"where":{"tags":{"op":"count","value":{"op":"is","value":0}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":220}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a14","trial":4,"input":{"where":{"shelter.capacity":{"op":"lte","value":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a14","trial":5,"input":{"where":{"shelter.capacity":{"op":"lte","value":12}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":72}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a15","trial":1,"input":{"where":{"any":[{"species":{"op":"is","value":"cat"},"status":{"op":"is","value":"available"}},{"species":{"op":"is","value":"dog"},"weightKg":{"op":"lt","value":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":69}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a15","trial":3,"input":{"where":{"any":[{"species":{"op":"is","value":"cat"},"status":{"op":"is","value":"available"}},{"species":{"op":"is","value":"dog"},"weightKg":{"op":"lt","value":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":69}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a15","trial":4,"input":{"where":{"any":[{"species":{"op":"is","value":"cat"},"status":{"op":"is","value":"available"}},{"species":{"op":"is","value":"dog"},"weightKg":{"op":"lt","value":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":69}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a15","trial":5,"input":{"where":{"any":[{"species":{"op":"is","value":"cat"},"status":{"op":"is","value":"available"}},{"species":{"op":"is","value":"dog"},"weightKg":{"op":"lt","value":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":69}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a15","trial":2,"input":{"where":{"any":[{"species":{"op":"is","value":"cat"},"status":{"op":"is","value":"available"}},{"species":{"op":"is","value":"dog"},"weightKg":{"op":"lt","value":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":69}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a12","trial":2,"input":{"where":{"any":[{"tags":{"op":"count","value":{"op":"is","value":0}}},{"tags":{"op":"exists","value":false}},{"tags":{"op":"isblank"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":376}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a16","trial":1,"input":{"where":{"status":{"op":"is","value":"available"},"any":[{"shelter.city":{"op":"is","value":"Patras"}},{"tags":{"op":"some","value":{"op":"is","value":"trained"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1126,"output_tokens":97}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a16","trial":2,"input":{"where":{"status":{"op":"is","value":"available"},"any":[{"shelter.city":{"op":"is","value":"Patras"}},{"tags":{"op":"some","value":{"op":"is","value":"trained"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1126,"output_tokens":101}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a16","trial":3,"input":{"where":{"status":{"op":"is","value":"available"},"any":[{"shelter.city":{"op":"is","value":"Patras"}},{"tags":{"op":"some","value":{"op":"is","value":"trained"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1126,"output_tokens":100}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a16","trial":5,"input":{"where":{"status":{"op":"is","value":"available"},"any":[{"shelter.city":{"op":"is","value":"Patras"}},{"tags":{"op":"some","value":{"op":"is","value":"trained"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1126,"output_tokens":100}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a16","trial":4,"input":{"where":{"status":{"op":"is","value":"available"},"any":[{"shelter.city":{"op":"is","value":"Patras"}},{"tags":{"op":"some","value":{"op":"is","value":"trained"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1126,"output_tokens":101}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a17","trial":2,"input":{"where":{"status":{"op":"noneof","value":["sold","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":72}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a17","trial":1,"input":{"where":{"status":{"op":"noneof","value":["sold","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":85}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a17","trial":3,"input":{"where":{"status":{"op":"noneof","value":["sold","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a17","trial":4,"input":{"where":{"status":{"op":"noneof","value":["sold","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":82}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a18","trial":2,"input":{"where":{"neutered":{"op":"is","value":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":33}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a18","trial":3,"input":{"where":{"neutered":{"op":"is","value":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":33}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a18","trial":5,"input":{"where":{"neutered":{"op":"is","value":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":33}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a18","trial":1,"input":{"where":{"neutered":{"op":"is","value":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":33}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a17","trial":5,"input":{"where":{"status":{"op":"noneof","value":["sold","pending"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a18","trial":4,"input":{"where":{"neutered":{"op":"is","value":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":33}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a19","trial":1,"input":{"where":{"priceCents":{"op":"gt","value":{"op":"fieldref","value":"costCents"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a19","trial":2,"input":{"where":{"priceCents":{"op":"gt","value":{"op":"fieldref","value":"costCents"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":64}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a20","trial":2,"input":{"where":{"status":{"op":"prefix","value":"a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a20","trial":1,"input":{"where":{"status":{"op":"prefix","value":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a20","trial":3,"input":{"where":{"status":{"op":"prefix","value":"a"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a19","trial":5,"input":{"where":{"priceCents":{"op":"gt","value":{"op":"fieldref","value":"costCents"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":64}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a19","trial":4,"input":{"where":{"priceCents":{"op":"gt","value":{"op":"fieldref","value":"costCents"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a19","trial":3,"input":{"where":{"priceCents":{"op":"gt","value":{"op":"fieldref","value":"costCents"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":84}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a20","trial":4,"input":{"where":{"status":{"op":"prefix","value":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"a20","trial":5,"input":{"where":{"status":{"op":"prefix","value":"a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b01","trial":3,"input":{"where":{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"rabies"},"boosterDue":{"op":"lt","value":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1147,"output_tokens":91}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b01","trial":2,"input":{"where":{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"rabies"},"boosterDue":{"op":"lt","value":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1147,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b01","trial":5,"input":{"where":{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"rabies"},"boosterDue":{"op":"lt","value":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1147,"output_tokens":90}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b01","trial":4,"input":{"where":{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"rabies"},"boosterDue":{"op":"lt","value":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1147,"output_tokens":99}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b01","trial":1,"input":{"where":{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"rabies"},"boosterDue":{"op":"lt","value":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1147,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b02","trial":1,"input":{"where":{"all":[{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"rabies"}}}},{"vaccinations":{"op":"some","value":{"boosterDue":{"op":"lt","value":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1152,"output_tokens":142}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b02","trial":2,"input":{"where":{"all":[{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"rabies"}}}},{"vaccinations":{"op":"some","value":{"boosterDue":{"op":"lt","value":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1152,"output_tokens":133}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b02","trial":3,"input":{"where":{"all":[{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"rabies"}}}},{"vaccinations":{"op":"some","value":{"boosterDue":{"op":"lt","value":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1152,"output_tokens":128}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b02","trial":4,"input":{"where":{"all":[{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"rabies"}}}},{"vaccinations":{"op":"some","value":{"boosterDue":{"op":"lt","value":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1152,"output_tokens":139}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b02","trial":5,"input":{"where":{"all":[{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"rabies"}}}},{"vaccinations":{"op":"some","value":{"boosterDue":{"op":"lt","value":"2025-01-01"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1152,"output_tokens":136}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b03","trial":1,"input":{"where":{"vaccinations":{"op":"some","value":{"administeredAt":{"op":"gte","value":"2025-01-01T00:00:00"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":126}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b03","trial":2,"input":{"where":{"vaccinations":{"op":"some","value":{"administeredAt":{"op":"gte","value":"2025-01-01T00:00:00"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":130}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b03","trial":3,"input":{"where":{"vaccinations":{"op":"some","value":{"administeredAt":{"op":"gte","value":"2025-01-01T00:00:00"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":134}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b04","trial":1,"input":{"where":{"vaccinations":{"op":"first","value":{"vaccine":{"op":"is","value":"parvo"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b03","trial":4,"input":{"where":{"vaccinations":{"op":"some","value":{"administeredAt":{"op":"gte","value":"2025-01-01T00:00:00"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":119}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b04","trial":4,"input":{"where":{"vaccinations":{"op":"first","value":{"vaccine":{"op":"is","value":"parvo"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b04","trial":2,"input":{"where":{"vaccinations":{"op":"first","value":{"vaccine":{"op":"is","value":"parvo"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b04","trial":3,"input":{"where":{"vaccinations":{"op":"first","value":{"vaccine":{"op":"is","value":"parvo"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b04","trial":5,"input":{"where":{"vaccinations":{"op":"first","value":{"vaccine":{"op":"is","value":"parvo"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b03","trial":5,"input":{"where":{"vaccinations":{"op":"some","value":{"administeredAt":{"op":"gte","value":"2025-01-01T00:00:00"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":137}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b05","trial":1,"input":{"where":{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"parvo"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b05","trial":2,"input":{"where":{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"parvo"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b05","trial":3,"input":{"where":{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"parvo"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b05","trial":5,"input":{"where":{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"parvo"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b05","trial":4,"input":{"where":{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"parvo"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b07","trial":1,"input":{"where":{"tags":{"op":"some","value":{"op":"isnot","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":130}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b07","trial":2,"input":{"where":{"tags":{"op":"some","value":{"op":"isnot","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":134}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b07","trial":3,"input":{"where":{"tags":{"op":"some","value":{"op":"isnot","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":119}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b06","trial":3,"input":{"where":{"tags":{"op":"some","value":{"op":"is","value":"indoor"},"negate":true,"orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":267}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b06","trial":5,"input":{"where":{"tags":{"op":"some","value":{"op":"is","value":"indoor"},"negate":true,"orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":257}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b06","trial":4,"input":{"where":{"not":{"tags":{"op":"some","value":{"op":"is","value":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":241}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b08","trial":1,"input":{"where":{"tags":{"op":"first","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1118,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b07","trial":5,"input":{"where":{"tags":{"op":"some","value":{"op":"isnot","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":116}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b06","trial":1,"input":{"where":{"not":{"tags":{"op":"some","value":{"op":"is","value":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":346}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b07","trial":4,"input":{"where":{"tags":{"op":"some","value":{"op":"isnot","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":129}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b08","trial":2,"input":{"where":{"tags":{"op":"first","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1118,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b08","trial":3,"input":{"where":{"tags":{"op":"first","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1118,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b08","trial":4,"input":{"where":{"tags":{"op":"first","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1118,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b08","trial":5,"input":{"where":{"tags":{"op":"first","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1118,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b06","trial":2,"input":{"where":{"tags":{"op":"some","value":{"op":"is","value":"indoor"},"negate":true,"orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":350}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b09","trial":1,"input":{"where":{"vaccinations":{"op":"some","value":{"all":[{"boosterDue":{"op":"exists","value":true}},{"boosterDue":{"op":"isblank","value":true,"negate":true}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":232}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b10","trial":2,"input":{"where":{"vaccinations":{"op":"some","value":{"any":[{"boosterDue":{"op":"exists","value":false}},{"boosterDue":{"op":"isblank","value":true}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":153}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b09","trial":3,"input":{"where":{"vaccinations":{"op":"some","value":{"all":[{"boosterDue":{"op":"exists","value":true}},{"boosterDue":{"op":"isblank","value":false}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":260}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b09","trial":4,"input":{"where":{"vaccinations":{"op":"some","value":{"all":[{"boosterDue":{"op":"exists","value":true}},{"boosterDue":{"op":"isblank","negate":true}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":274}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b10","trial":1,"input":{"where":{"vaccinations":{"op":"some","value":{"any":[{"boosterDue":{"op":"exists","value":false}},{"boosterDue":{"op":"isblank"}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":278}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b10","trial":3,"input":{"where":{"vaccinations":{"op":"some","value":{"any":[{"boosterDue":{"op":"exists","value":false}},{"boosterDue":{"op":"isblank","value":true}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":248}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b09","trial":5,"input":{"where":{"vaccinations":{"op":"some","value":{"all":[{"boosterDue":{"op":"exists","value":true}},{"boosterDue":{"op":"isblank","negate":true}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":380}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b09","trial":2,"input":{"where":{"vaccinations":{"op":"some","value":{"all":[{"boosterDue":{"op":"exists","value":true}},{"boosterDue":{"op":"isblank","negate":true}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":376}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b10","trial":5,"input":{"where":{"vaccinations":{"op":"some","value":{"any":[{"boosterDue":{"op":"exists","value":false}},{"boosterDue":{"op":"isblank","value":true}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":142}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b10","trial":4,"input":{"where":{"vaccinations":{"op":"some","value":{"any":[{"boosterDue":{"op":"exists","value":false}},{"boosterDue":{"op":"isblank","value":true}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":163}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b11","trial":4,"input":{"where":{"vaccinations":{"op":"every","value":{"vaccine":{"op":"is","value":"rabies"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1130,"output_tokens":95}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b11","trial":3,"input":{"where":{"vaccinations":{"op":"every","value":{"vaccine":{"op":"is","value":"rabies"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1130,"output_tokens":96}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b11","trial":1,"input":{"where":{"vaccinations":{"op":"every","value":{"vaccine":{"op":"is","value":"rabies"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1130,"output_tokens":177}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b11","trial":5,"input":{"where":{"vaccinations":{"op":"every","value":{"vaccine":{"op":"is","value":"rabies"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1130,"output_tokens":125}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b11","trial":2,"input":{"where":{"vaccinations":{"op":"every","value":{"vaccine":{"op":"is","value":"rabies"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1130,"output_tokens":196}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b12","trial":2,"input":{"where":{"tags":{"op":"every","value":{"op":"isnot","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1129,"output_tokens":161}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b12","trial":3,"input":{"where":{"not":{"tags":{"op":"some","value":{"op":"is","value":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1129,"output_tokens":260}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b12","trial":4,"input":{"where":{"tags":{"op":"every","value":{"op":"isnot","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1129,"output_tokens":201}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b12","trial":5,"input":{"where":{"tags":{"op":"every","value":{"op":"isnot","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1129,"output_tokens":189}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b12","trial":1,"input":{"where":{"none":[{"tags":{"op":"some","value":{"op":"is","value":"indoor"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1129,"output_tokens":306}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b13","trial":2,"input":{"where":{"tags":{"op":"every","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":213}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b13","trial":3,"input":{"where":{"tags":{"op":"every","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":243}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b13","trial":4,"input":{"where":{"tags":{"op":"every","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":95}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c01","trial":4,"input":{"where":{"microchip":{"op":"isblank"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":29}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b13","trial":1,"input":{"where":{"any":[{"tags":{"op":"every","value":{"op":"is","value":"indoor"}}},{"tags":{"op":"exists","value":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":319}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"b13","trial":5,"input":{"where":{"tags":{"op":"every","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":150}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c01","trial":1,"input":{"where":{"microchip":{"op":"isblank"},"all":[{"microchip":{"op":"exists","value":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":95}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c01","trial":3,"input":{"where":{"all":[{"microchip":{"op":"exists","value":true}},{"microchip":{"op":"isblank","value":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":145}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c02","trial":2,"input":{"where":{"microchip":{"op":"exists","value":false}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c02","trial":1,"input":{"where":{"microchip":{"op":"exists","value":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":32}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c01","trial":5,"input":{"where":{"all":[{"microchip":{"op":"exists","value":true}},{"microchip":{"op":"isblank","value":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":120}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c02","trial":3,"input":{"where":{"microchip":{"op":"exists","value":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":49}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c02","trial":4,"input":{"where":{"microchip":{"op":"exists","value":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":32}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c02","trial":5,"input":{"where":{"microchip":{"op":"exists","value":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":32}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c01","trial":2,"input":{"where":{"all":[{"microchip":{"op":"exists","value":true}},{"microchip":{"op":"isblank","value":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":125}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c03","trial":1,"input":{"where":{"notes":{"op":"exists","value":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c03","trial":3,"input":{"where":{"notes":{"op":"exists","value":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c03","trial":4,"input":{"where":{"notes":{"op":"exists","value":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c03","trial":2,"input":{"where":{"notes":{"op":"exists","value":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c03","trial":5,"input":{"where":{"notes":{"op":"exists","value":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c04","trial":3,"input":{"where":{"shelter.capacity":{"op":"exists","value":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c05","trial":2,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1133,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c04","trial":1,"input":{"where":{"any":[{"shelter.capacity":{"op":"exists","value":false}},{"shelter.capacity":{"op":"isblank","value":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":172}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c05","trial":3,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1133,"output_tokens":96}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c04","trial":4,"input":{"where":{"any":[{"shelter.capacity":{"op":"exists","value":false}},{"shelter.capacity":{"op":"isblank","value":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":136}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c04","trial":2,"input":{"where":{"any":[{"shelter.capacity":{"op":"exists","value":false}},{"shelter.capacity":{"op":"isblank","value":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":163}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c04","trial":5,"input":{"where":{"any":[{"shelter.capacity":{"op":"exists","value":false}},{"shelter.capacity":{"op":"isblank","value":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":181}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c05","trial":1,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1133,"output_tokens":133}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c08","trial":1,"input":{"where":{"status":{"op":"isnot","value":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1112,"output_tokens":32}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c05","trial":5,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1133,"output_tokens":129}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c05","trial":4,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1133,"output_tokens":129}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c08","trial":2,"input":{"where":{"status":{"op":"isnot","value":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1112,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c08","trial":3,"input":{"where":{"status":{"op":"isnot","value":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1112,"output_tokens":32}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c08","trial":4,"input":{"where":{"status":{"op":"isnot","value":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1112,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c07","trial":2,"input":{"where":{"all":[{"microchip":{"op":"exists","value":true}},{"any":[{"microchip":{"op":"isblank","value":true}},{"microchip":{"op":"isnot","value":"CHIP-001"}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1139,"output_tokens":283}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c07","trial":5,"input":{"where":{"all":[{"microchip":{"op":"exists","value":true}},{"any":[{"microchip":{"op":"isnot","value":"CHIP-001"}},{"microchip":{"op":"isblank","value":true}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1139,"output_tokens":261}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c07","trial":4,"input":{"where":{"all":[{"microchip":{"op":"exists","value":true}},{"any":[{"microchip":{"op":"isnot","value":"CHIP-001"}},{"microchip":{"op":"isblank"}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1139,"output_tokens":294}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c07","trial":1,"input":{"where":{"all":[{"microchip":{"op":"exists","value":true}},{"any":[{"microchip":{"op":"isnot","value":"CHIP-001"}},{"microchip":{"op":"isblank","value":true}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1139,"output_tokens":373}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c07","trial":3,"input":{"where":{"all":[{"microchip":{"op":"exists","value":true}},{"any":[{"microchip":{"op":"isnot","value":"CHIP-001"}},{"microchip":{"op":"isblank","value":true}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1139,"output_tokens":322}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c11","trial":1,"input":{"where":{"weightKg":{"op":"notbetween","value":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":101}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c11","trial":2,"input":{"where":{"weightKg":{"op":"notbetween","value":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":89}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c08","trial":5,"input":{"where":{"status":{"op":"isnot","value":"sold"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1112,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c11","trial":3,"input":{"where":{"weightKg":{"op":"notbetween","value":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":92}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c11","trial":4,"input":{"where":{"weightKg":{"op":"notbetween","value":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c11","trial":5,"input":{"where":{"weightKg":{"op":"notbetween","value":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":92}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c12","trial":1,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1142,"output_tokens":176}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c12","trial":2,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1142,"output_tokens":148}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c12","trial":5,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1142,"output_tokens":140}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c12","trial":3,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1142,"output_tokens":157}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c12","trial":4,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1142,"output_tokens":185}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c13","trial":1,"input":{"where":{"born":{"op":"gte","value":"2020-01-01","negate":true,"orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1137,"output_tokens":191}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c13","trial":2,"input":{"where":{"born":{"op":"lt","value":"2020-01-01","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1137,"output_tokens":240}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d01","trial":1,"input":{"where":{"notes":{"op":"type","value":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c13","trial":3,"input":{"where":{"born":{"op":"lt","value":"2020-01-01","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1137,"output_tokens":234}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d01","trial":2,"input":{"where":{"notes":{"op":"type","value":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d01","trial":3,"input":{"where":{"notes":{"op":"type","value":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d01","trial":4,"input":{"where":{"notes":{"op":"type","value":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c13","trial":4,"input":{"where":{"born":{"op":"gte","value":"2020-01-01","negate":true,"orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1137,"output_tokens":181}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"c13","trial":5,"input":{"where":{"born":{"op":"lt","value":"2020-01-01","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1137,"output_tokens":164}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d01","trial":5,"input":{"where":{"notes":{"op":"type","value":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d03","trial":1,"input":{"where":{"notes":{"op":"type","value":"array"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d03","trial":2,"input":{"where":{"notes":{"op":"type","value":"array"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d03","trial":3,"input":{"where":{"notes":{"op":"type","value":"array"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d03","trial":4,"input":{"where":{"notes":{"op":"type","value":"array"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d03","trial":5,"input":{"where":{"notes":{"op":"type","value":"array"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d04","trial":1,"input":{"where":{"notes":{"op":"isblank"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d04","trial":2,"input":{"where":{"notes":{"op":"isblank"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":28}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d02","trial":1,"input":{"where":{"notes":{"op":"type","value":"integer"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1126,"output_tokens":190}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d02","trial":4,"input":{"where":{"notes":{"op":"type","value":"integer"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1126,"output_tokens":227}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d02","trial":3,"input":{"where":{"notes":{"op":"type","value":"integer"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1126,"output_tokens":230}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d02","trial":5,"input":{"where":{"notes":{"op":"type","value":"integer"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1126,"output_tokens":234}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d04","trial":3,"input":{"where":{"all":[{"notes":{"op":"exists","value":true}},{"notes":{"op":"isblank","value":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":155}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d05","trial":2,"input":{"where":{"notes":{"op":"is","value":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d05","trial":3,"input":{"where":{"notes":{"op":"is","value":3}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d05","trial":1,"input":{"where":{"notes":{"op":"is","value":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":68}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d05","trial":4,"input":{"where":{"notes":{"op":"is","value":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d02","trial":2,"input":{"where":{"notes":{"op":"type","value":"integer"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1126,"output_tokens":323}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d04","trial":5,"input":{"where":{"notes":{"op":"isblank"},"all":[{"notes":{"op":"exists","value":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":132}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d04","trial":4,"input":{"where":{"notes":{"op":"isblank"},"all":[{"notes":{"op":"exists","value":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":155}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d05","trial":5,"input":{"where":{"notes":{"op":"is","value":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d06","trial":3,"input":{"where":{"notes":{"op":"gt","value":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d06","trial":5,"input":{"where":{"notes":{"op":"gt","value":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d06","trial":1,"input":{"where":{"all":[{"notes":{"op":"type","value":"number"}},{"notes":{"op":"gt","value":2}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":93}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d06","trial":4,"input":{"where":{"notes":{"op":"gt","value":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d06","trial":2,"input":{"where":{"all":[{"notes":{"op":"type","value":"number"}},{"notes":{"op":"gt","value":2}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":151}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d07","trial":3,"input":{"where":{"notes":{"op":"gt","value":"a"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":112}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d07","trial":4,"input":{"where":{"notes":{"op":"gt","value":"a"},"all":[{"notes":{"op":"type","value":"string"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":103}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d07","trial":1,"input":{"where":{"all":[{"notes":{"op":"type","value":"string"}},{"notes":{"op":"gt","value":"a"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":174}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d07","trial":2,"input":{"where":{"all":[{"notes":{"op":"type","value":"string"}},{"notes":{"op":"gt","value":"a"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":133}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d07","trial":5,"input":{"where":{"all":[{"notes":{"op":"type","value":"string"}},{"notes":{"op":"gt","value":"a"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":140}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d08","trial":1,"input":{"where":{"notes":{"op":"isnot","value":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d08","trial":3,"input":{"where":{"notes":{"op":"isnot","value":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":111}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d08","trial":2,"input":{"where":{"notes":{"op":"isnot","value":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":138}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d08","trial":4,"input":{"where":{"notes":{"op":"isnot","value":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":146}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d08","trial":5,"input":{"where":{"notes":{"op":"isnot","value":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":132}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d09","trial":3,"input":{"where":{"notes":{"op":"lte","value":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":127}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d09","trial":1,"input":{"where":{"notes":{"op":"lte","value":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":169}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e01","trial":1,"text":"I can’t express that with the pet search tool: the available fields include `rate`, but not a field literally named `$rate`.","stopReason":"stop","usage":{"input_tokens":1126,"output_tokens":132}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d09","trial":5,"input":{"where":{"notes":{"op":"lte","value":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":159}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d09","trial":2,"input":{"where":{"notes":{"op":"lte","value":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":206}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e01","trial":3,"text":"I can’t express that with the available search tool: it only supports the field `rate`, not a field literally named `$rate`.","stopReason":"stop","usage":{"input_tokens":1126,"output_tokens":122}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e01","trial":2,"text":"I can’t express that with the pet-store search tool: it only supports a field named `rate`, not a field literally named `$rate`.","stopReason":"stop","usage":{"input_tokens":1126,"output_tokens":176}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"d09","trial":4,"input":{"where":{"notes":{"op":"lte","value":2}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":202}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e01","trial":5,"text":"I can’t express that query with the available pet-search tool because it only supports the field `rate`, not a field literally named `$rate`.","stopReason":"stop","usage":{"input_tokens":1126,"output_tokens":141}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e01","trial":4,"text":"I can’t express that with the pet-search tool, because the available fields include `rate` but not a field literally named `$rate`.","stopReason":"stop","usage":{"input_tokens":1126,"output_tokens":148}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e03","trial":1,"input":{"where":{"name":{"op":"matches","value":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":96}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e02","trial":3,"text":"I can’t express that query with the available pet-search tool: it only exposes `sizeRaw`, not a literal field named `size.raw`.","stopReason":"stop","usage":{"input_tokens":1137,"output_tokens":131}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e02","trial":4,"text":"I can’t express that search with the available tool: it only exposes a field named `sizeRaw`, not a literal field named `size.raw`, and there’s no escaping mechanism for dots in field names.","stopReason":"stop","usage":{"input_tokens":1137,"output_tokens":106}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e02","trial":1,"text":"I can’t express that query with the available pet-search tool: it does not expose a literal field named `size.raw`—only `sizeRaw` is available, and I shouldn’t approximate it.","stopReason":"stop","usage":{"input_tokens":1137,"output_tokens":170}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e02","trial":2,"text":"I can’t express that with the available pet-search tool: it does not expose a literal field named `size.raw`. It only lists `sizeRaw` as a searchable field, and I shouldn’t approximate it.","stopReason":"stop","usage":{"input_tokens":1137,"output_tokens":153}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e02","trial":5,"text":"I can’t express that search with the available tool: it only supports the field `sizeRaw`, not a literal field named `size.raw`.","stopReason":"stop","usage":{"input_tokens":1137,"output_tokens":153}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e03","trial":3,"input":{"where":{"name":{"op":"matches","value":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":81}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e03","trial":5,"input":{"where":{"name":{"op":"matches","value":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":87}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e04","trial":1,"input":{"where":{"any":[{"name":{"op":"prefix","value":"A"}},{"name":{"op":"prefix","value":"B"}},{"name":{"op":"prefix","value":"C"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":97}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e03","trial":4,"input":{"where":{"all":[{"name":{"op":"prefix","value":"C"}},{"name":{"op":"suffix","value":"o"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":86}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e03","trial":2,"input":{"where":{"all":[{"name":{"op":"prefix","value":"C"}},{"name":{"op":"suffix","value":"o"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":132}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e04","trial":4,"input":{"where":{"any":[{"name":{"op":"prefix","value":"A"}},{"name":{"op":"prefix","value":"B"}},{"name":{"op":"prefix","value":"C"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":96}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e04","trial":2,"input":{"where":{"name":{"op":"matches","value":"^[ABC]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":89}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e04","trial":3,"input":{"where":{"any":[{"name":{"op":"prefix","value":"A"}},{"name":{"op":"prefix","value":"B"}},{"name":{"op":"prefix","value":"C"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":91}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e04","trial":5,"input":{"where":{"name":{"op":"matches","value":"^[ABC]"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1122,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e05","trial":1,"input":{"where":{"name":{"op":"matches","value":"^[abc]","flags":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1123,"output_tokens":84}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e05","trial":2,"input":{"where":{"name":{"op":"matches","value":"^[abc]","flags":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1123,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e05","trial":3,"input":{"where":{"name":{"op":"matches","value":"^[abc]","flags":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1123,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e06","trial":1,"input":{"where":{"name":{"op":"suffix","value":"s"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e06","trial":3,"input":{"where":{"name":{"op":"suffix","value":"s"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e06","trial":2,"input":{"where":{"name":{"op":"suffix","value":"s"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e05","trial":4,"input":{"where":{"name":{"op":"matches","value":"^[abc]","flags":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1123,"output_tokens":84}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e05","trial":5,"input":{"where":{"name":{"op":"matches","value":"^[abc]","flags":"i"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1123,"output_tokens":82}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e06","trial":4,"input":{"where":{"name":{"op":"suffix","value":"s"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"e06","trial":5,"input":{"where":{"name":{"op":"suffix","value":"s"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"f01","trial":1,"input":{"where":{"notes":{"op":"is","value":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"f01","trial":2,"input":{"where":{"notes":{"op":"is","value":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":101}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"f01","trial":4,"input":{"where":{"notes":{"op":"is","value":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":86}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"f01","trial":5,"input":{"where":{"notes":{"op":"is","value":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":94}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"f01","trial":3,"input":{"where":{"notes":{"op":"is","value":"anything"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":91}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"f02","trial":1,"text":"I can’t run a true free-text search across all pet records with the available tool. It only supports structured field queries, so I shouldn’t approximate it.","stopReason":"stop","usage":{"input_tokens":1125,"output_tokens":139}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m01","trial":3,"input":{"where":{"tags":{"op":"some","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m01","trial":1,"input":{"where":{"tags":{"op":"some","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":62}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m01","trial":2,"input":{"where":{"tags":{"op":"some","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"f02","trial":5,"text":"I can’t run a true free-text search across all pet records with the available tool. It only supports structured field-specific queries, not global free-text search.","stopReason":"stop","usage":{"input_tokens":1125,"output_tokens":103}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"f02","trial":3,"text":"I can’t run a free-text search across all pet records with the available tool. It only supports structured field-based queries, not a general full-record text search.","stopReason":"stop","usage":{"input_tokens":1125,"output_tokens":162}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m01","trial":4,"input":{"where":{"tags":{"op":"some","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":62}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m01","trial":5,"input":{"where":{"tags":{"op":"some","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":59}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m02","trial":1,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1140,"output_tokens":125}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"f02","trial":4,"text":"I can’t run a true free-text search across all pet records with the available tool. It only supports structured field-specific queries, not global text search.","stopReason":"stop","usage":{"input_tokens":1125,"output_tokens":230}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m03","trial":1,"input":{"where":{"status":{"op":"is","value":"available"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":27}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m02","trial":3,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1140,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m02","trial":2,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1140,"output_tokens":130}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m03","trial":2,"input":{"where":{"status":{"op":"is","value":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":31}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m02","trial":4,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1140,"output_tokens":126}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a01","trial":1,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":399,"output_tokens":18}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m02","trial":5,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1140,"output_tokens":136}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m03","trial":4,"input":{"where":{"status":{"op":"is","value":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":51}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a01","trial":2,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":399,"output_tokens":18}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m03","trial":3,"input":{"where":{"status":{"op":"is","value":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":52}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a01","trial":3,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":399,"output_tokens":18}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"m03","trial":5,"input":{"where":{"status":{"op":"is","value":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":50}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a01","trial":4,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":399,"output_tokens":18}} +{"provider":"gpt-5.5","arm":"bespoke-json","caseId":"f02","trial":2,"text":"I can’t express a true free-text search across entire pet records with the available search tool. It only supports structured field-specific queries, not a global search for `ada` across all fields.","stopReason":"stop","usage":{"input_tokens":1125,"output_tokens":420}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a01","trial":5,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":399,"output_tokens":18}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a02","trial":1,"input":{"species":"cat","status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":398,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a02","trial":4,"input":{"species":"cat","status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":398,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a02","trial":5,"input":{"species":"cat","status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":398,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a02","trial":2,"input":{"species":"cat","status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":398,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a02","trial":3,"input":{"species":"cat","status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":398,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a04","trial":1,"input":{"minWeightKg":1,"maxWeightKg":5},"text":"","stopReason":"tool_calls","usage":{"input_tokens":419,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a03","trial":1,"input":{"status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":102}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a04","trial":2,"input":{"minWeightKg":1,"maxWeightKg":5},"text":"","stopReason":"tool_calls","usage":{"input_tokens":419,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a04","trial":3,"input":{"minWeightKg":1,"maxWeightKg":5},"text":"","stopReason":"tool_calls","usage":{"input_tokens":419,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a03","trial":3,"input":{"status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":121}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a03","trial":2,"input":{"status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":125}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a04","trial":4,"input":{"minWeightKg":1,"maxWeightKg":5},"text":"","stopReason":"tool_calls","usage":{"input_tokens":419,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a03","trial":5,"input":{"status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":139}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a03","trial":4,"input":{"status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":135}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a04","trial":5,"input":{"minWeightKg":1,"maxWeightKg":5},"text":"","stopReason":"tool_calls","usage":{"input_tokens":419,"output_tokens":26}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a05","trial":1,"input":{"bornFrom":"2020-01-01"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":404,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a05","trial":2,"input":{"bornFrom":"2020-01-01"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":404,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a05","trial":4,"input":{"bornFrom":"2020-01-01"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":404,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a05","trial":3,"input":{"bornFrom":"2020-01-01"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":404,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a05","trial":5,"input":{"bornFrom":"2020-01-01"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":404,"output_tokens":24}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a07","trial":1,"input":{"nameContains":"o","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":405,"output_tokens":23}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a06","trial":1,"text":"I can’t express that search with the available tool: it only supports case-insensitive name substring matching, not case-sensitive “begins with lower-case b” matching.","stopReason":"stop","usage":{"input_tokens":414,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a07","trial":2,"input":{"nameContains":"o"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":405,"output_tokens":34}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a06","trial":3,"text":"I can’t express that search with the available tool. It only supports case-insensitive name substring matching, not case-sensitive “begins with lower-case b” matching.","stopReason":"stop","usage":{"input_tokens":414,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a06","trial":4,"text":"I can’t express that search with the available tool: it only supports case-insensitive name substring matching, not case-sensitive “begins with lower-case b.”","stopReason":"stop","usage":{"input_tokens":414,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a06","trial":5,"text":"I can’t express that search with the available tool. The tool only supports case-insensitive name substring matching, so it can’t distinguish lower-case `b` from capital `B` or require that the name begins with it.","stopReason":"stop","usage":{"input_tokens":414,"output_tokens":89}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a07","trial":3,"input":{"nameContains":"o"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":405,"output_tokens":19}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a07","trial":4,"input":{"nameContains":"o"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":405,"output_tokens":19}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a07","trial":5,"input":{"nameContains":"o","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":405,"output_tokens":23}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a06","trial":2,"text":"I can’t express that search with the available tool: it only supports case-insensitive name substring matching, not case-sensitive “begins with lower-case b” matching.","stopReason":"stop","usage":{"input_tokens":414,"output_tokens":148}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a08","trial":1,"input":{"nameContains":"50%"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":39}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a08","trial":3,"input":{"nameContains":"50%"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":40}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a08","trial":2,"input":{"nameContains":"50%"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":45}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a08","trial":4,"input":{"nameContains":"50%"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":39}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a08","trial":5,"input":{"nameContains":"50%"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":41}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a09","trial":1,"input":{"tag":"indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":127}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a09","trial":4,"input":{"tag":"indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":141}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a09","trial":3,"input":{"tag":"indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":135}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a09","trial":5,"input":{"tag":"indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":156}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a09","trial":2,"input":{"tag":"indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":188}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a10","trial":1,"input":{"tag":"trained","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":402,"output_tokens":188}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a10","trial":2,"input":{"tag":"trained","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":402,"output_tokens":246}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a11","trial":1,"text":"I can’t express “pets with three or more labels” using the available search tool. It only supports filtering by a single specific label, not counting how many labels a pet has.","stopReason":"stop","usage":{"input_tokens":398,"output_tokens":87}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a11","trial":3,"text":"I can’t search for pets by “three or more labels” with the available tool.","stopReason":"stop","usage":{"input_tokens":398,"output_tokens":59}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a11","trial":2,"text":"I can’t express “pets with three or more labels” using the available search tool, because it only supports filtering by a single specific tag—not counting how many labels each pet has.","stopReason":"stop","usage":{"input_tokens":398,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a10","trial":5,"input":{"tag":"trained","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":402,"output_tokens":227}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a11","trial":5,"text":"I can’t express “pets with three or more labels” using the available pet search tool, since it can only filter by a single specific label—not by the number of labels a pet has.","stopReason":"stop","usage":{"input_tokens":398,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a12","trial":1,"text":"I can’t express “pets with no labels at all” with the available search tool. The tool can only filter for pets that have a specific label, not for pets with zero labels.","stopReason":"stop","usage":{"input_tokens":398,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a12","trial":2,"text":"I can’t express “pets with no labels at all” using the available search tool. It can only search for pets that have a specific label, not pets with zero labels.","stopReason":"stop","usage":{"input_tokens":398,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a12","trial":3,"text":"I can’t search for pets with **no labels at all** using the available tool. It only supports filtering for pets that **do have a specific label**, not pets with an empty label list.","stopReason":"stop","usage":{"input_tokens":398,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a13","trial":1,"input":{"shelterCity":"Athens"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":400,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a13","trial":3,"input":{"shelterCity":"Athens"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":400,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a12","trial":4,"text":"I can’t express “pets with no labels at all” with the available search tool, since it only supports filtering for pets that have a specific label, not pets with zero labels.","stopReason":"stop","usage":{"input_tokens":398,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a10","trial":3,"input":{"tag":"trained","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":402,"output_tokens":401}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a13","trial":2,"input":{"shelterCity":"Athens"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":400,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a10","trial":4,"text":"I can’t express “pets that carry both the trained label and the outdoor label” with the available search tool, because it only supports filtering by a single label at a time.","stopReason":"stop","usage":{"input_tokens":402,"output_tokens":438}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a13","trial":5,"input":{"shelterCity":"Athens"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":400,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a13","trial":4,"input":{"shelterCity":"Athens"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":400,"output_tokens":22}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a11","trial":4,"text":"I can’t express “pets with three or more labels” using the available pet search tool, because it only supports filtering by a single specific label, not by label count.","stopReason":"stop","usage":{"input_tokens":398,"output_tokens":345}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a12","trial":5,"text":"I can’t search for pets with no labels using the available tool, because it only supports filtering for pets that have a specific label—not pets with zero labels.","stopReason":"stop","usage":{"input_tokens":398,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a14","trial":1,"text":"I can’t search by the number of kennels at a shelter with the available pet-search tool.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a14","trial":2,"text":"I can’t search by the number of kennels at a shelter with the available pet-search tool.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a14","trial":3,"text":"I can’t search by shelter kennel count with the available pet-store tool. It only supports filters like species, status, birth date, weight, tag, shelter city, neutered status, and name.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a14","trial":4,"text":"I can’t search by the number of kennels at a shelter with the available tool.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":67}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a14","trial":5,"text":"I can’t search by the number of kennels a shelter has with the available pet-search tool.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":53}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a16","trial":3,"input":{"status":"available","shelterCity":"Patras","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":410,"output_tokens":195}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a16","trial":1,"input":{"status":"available","shelterCity":"Patras","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":410,"output_tokens":125}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a15","trial":1,"input":{"species":"cat","status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":406,"output_tokens":262}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a16","trial":2,"input":{"status":"available","shelterCity":"Patras","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":410,"output_tokens":225}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a17","trial":1,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":398,"output_tokens":48}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a17","trial":2,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":398,"output_tokens":45}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a17","trial":4,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":398,"output_tokens":45}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a16","trial":5,"input":{"status":"available","shelterCity":"Patras","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":410,"output_tokens":174}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a17","trial":3,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":398,"output_tokens":53}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a16","trial":4,"input":{"status":"available","shelterCity":"Patras","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":410,"output_tokens":200}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a18","trial":1,"input":{"neutered":true},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":20}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a18","trial":2,"input":{"neutered":true},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":20}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a17","trial":5,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":398,"output_tokens":44}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a18","trial":3,"input":{"neutered":true},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":20}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a15","trial":3,"text":"I can’t express that query exactly with the tool: it supports maximum weight as inclusive, so I can search dogs weighing **20 kg or less**, but not strictly **under 20 kg**.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":504}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a15","trial":2,"text":"I can’t express this query exactly with the available tool, because “dogs weighing under 20 kg” requires an exclusive weight limit, while the tool only supports inclusive maximum weight.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":557}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a18","trial":5,"input":{"neutered":true},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":20}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a18","trial":4,"input":{"neutered":true},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":20}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a15","trial":4,"input":{"species":"cat","status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":406,"output_tokens":585}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a15","trial":5,"text":"I can’t express this query exactly with the tool because “dogs weighing under 20 kg” requires a strict `< 20` filter, but the tool only supports an inclusive maximum weight (`≤ 20`).","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":563}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a19","trial":1,"text":"I can’t answer that with the available pet search tool, because it doesn’t include listing price or cost fields to compare.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a20","trial":1,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":401,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a19","trial":2,"text":"I can’t answer that with the available pet-search tool, because it doesn’t include listing price or cost fields.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a19","trial":5,"text":"I can’t answer that with the available pet search tool because it doesn’t include listing price or cost fields.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":69}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a19","trial":4,"text":"I can’t express that with the available pet search tool, because it doesn’t include filters for listing price, cost, or comparing one field to another.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":94}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a20","trial":2,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":401,"output_tokens":60}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a19","trial":3,"text":"I can’t express that query with the available pet search tool, because it doesn’t provide listing price, cost, or comparison filters.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":93}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a20","trial":3,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":401,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a20","trial":5,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":401,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b01","trial":3,"text":"I can’t express that search with the available pet-store tool. It doesn’t provide fields for vaccinations, rabies shots, or booster due dates, so I can’t reliably filter for pets whose rabies booster was due before 1 January 2025.","stopReason":"stop","usage":{"input_tokens":431,"output_tokens":90}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"a20","trial":4,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":401,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b01","trial":2,"text":"I can’t express that search with the available pet-store tool. It doesn’t support filtering by vaccinations, rabies shots, or booster due dates tied to a specific shot.","stopReason":"stop","usage":{"input_tokens":431,"output_tokens":84}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b01","trial":1,"text":"I can’t express that query with the available pet search tool. It doesn’t have fields for vaccinations, rabies shots, or booster due dates, so I can’t reliably ensure the overdue booster belongs specifically to the rabies shot.","stopReason":"stop","usage":{"input_tokens":431,"output_tokens":111}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b01","trial":5,"text":"I can’t express that search with the available pet-store tool, because it has no parameters for vaccination records, rabies shots, or booster due dates.","stopReason":"stop","usage":{"input_tokens":431,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b01","trial":4,"text":"I can’t express that query with the available pet-search tool, because it doesn’t include vaccination records or booster due dates.","stopReason":"stop","usage":{"input_tokens":431,"output_tokens":91}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b02","trial":2,"text":"I can’t express that query with the available pet search tool. It doesn’t provide fields for vaccination history, rabies shots, or booster due dates.","stopReason":"stop","usage":{"input_tokens":436,"output_tokens":92}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b02","trial":4,"text":"I can’t express that query with the available pet-store search tool, because it doesn’t provide filters for vaccination history, rabies shots, or booster due dates.","stopReason":"stop","usage":{"input_tokens":436,"output_tokens":67}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b03","trial":2,"text":"I can’t search by vaccination date with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":50}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b03","trial":1,"text":"I can’t search for pets by vaccination date with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b02","trial":1,"text":"I’m sorry, but that question can’t be expressed with the available pet search tool. The tool doesn’t support searching vaccination records, shot types, or booster due dates.","stopReason":"stop","usage":{"input_tokens":436,"output_tokens":146}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b03","trial":3,"text":"I can’t express that search with the available pet-store tool, because it has no vaccination-date filter.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b02","trial":3,"text":"I can’t express that query with the available pet search tool. It only supports filters like species, status, birth date, weight, tag, shelter city, neutered status, and name—not vaccination history or booster due dates.","stopReason":"stop","usage":{"input_tokens":436,"output_tokens":131}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b04","trial":1,"text":"I can’t search for “parvo as the very first entry in vaccination history” with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b02","trial":5,"text":"I can’t express that query with the available pet-search tool. It doesn’t support searching vaccination history or booster due dates.","stopReason":"stop","usage":{"input_tokens":436,"output_tokens":145}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b04","trial":2,"text":"I can’t express that search with the available pet-store tool, because it doesn’t support filtering by vaccination history or by the first vaccination entry.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b03","trial":4,"text":"I can’t express that search with the available pet-store tool because it doesn’t have a vaccination date field or filter.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":68}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b03","trial":5,"text":"I can’t search for pets by vaccination date with the available pet-search tool.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b04","trial":4,"text":"I can’t search for that with the available pet-store tool, because it doesn’t support filtering by vaccination history or the order of vaccination entries.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b04","trial":3,"text":"I can’t express that search with the available pet-store tool, because it doesn’t support querying vaccination history or the order of vaccination entries.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":72}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b04","trial":5,"text":"I can’t express that query with the available pet-search tool, because it doesn’t support searching vaccination history or the order of entries.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b05","trial":2,"input":{"tag":"parvo"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":399,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b05","trial":3,"input":{"tag":"parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":399,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b06","trial":2,"text":"I can’t express “pets that do not carry the indoor label” with the available search tool, because it only supports finding pets that have a specified tag, not excluding a tag.","stopReason":"stop","usage":{"input_tokens":399,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b06","trial":1,"text":"I can’t express “pets that do **not** carry the indoor label” with the available search tool, because it only supports finding pets that **do** have a given label.","stopReason":"stop","usage":{"input_tokens":399,"output_tokens":82}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b05","trial":5,"input":{"tag":"parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":399,"output_tokens":130}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b06","trial":5,"text":"I can’t search for pets that **do not** carry a specific label with the available tool. It only supports finding pets that **do** carry a given label.","stopReason":"stop","usage":{"input_tokens":399,"output_tokens":72}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b06","trial":3,"text":"I can’t express “pets that do not carry the indoor label” with the available search tool, because it only supports filtering for pets that do carry a given label, not excluding one.","stopReason":"stop","usage":{"input_tokens":399,"output_tokens":165}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b05","trial":1,"input":{"tag":"parvo vaccination","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":399,"output_tokens":205}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b06","trial":4,"text":"I can’t express “pets that do not carry the indoor label” with the available search tool, since it only supports filtering for pets that do carry a given tag, not excluding a tag.","stopReason":"stop","usage":{"input_tokens":399,"output_tokens":171}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b05","trial":4,"input":{"tag":"parvo","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":399,"output_tokens":292}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b08","trial":1,"text":"I can’t express “indoor as the first label in their list” with the available search tool. It can only search for pets that have the label “indoor” anywhere in their labels, not specifically as the first label.","stopReason":"stop","usage":{"input_tokens":402,"output_tokens":91}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b08","trial":2,"text":"I can’t express “indoor as the first label in their list” with the available pet search tool. It can only search for pets that have the “indoor” tag anywhere, not specifically as the first label.","stopReason":"stop","usage":{"input_tokens":402,"output_tokens":105}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b07","trial":1,"text":"I can’t express “pets with at least one label other than indoor” using the available pet-search tool, because it only supports requiring a single specific tag, not excluding a tag or checking for any non-`indoor` label.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":257}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b07","trial":3,"text":"I can’t express that search with the available tool. It can only filter for pets that have a specific label, not for pets that have any label other than “indoor.”","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":180}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b08","trial":3,"text":"I can’t search for pets where **“indoor” is the first label** because the available search tool can only filter for pets that have a given tag anywhere in their labels, not by label order.","stopReason":"stop","usage":{"input_tokens":402,"output_tokens":83}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b07","trial":2,"text":"I can’t express that search with the available pet-store tool. It can only filter for a single specific label a pet must have, not “any label other than indoor.”","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":286}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b07","trial":4,"text":"I can’t express “has at least one label other than indoor” with the available pet-search tool, because it only supports requiring a specific tag, not excluding or comparing tags.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":258}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b08","trial":4,"text":"I can’t express “indoor as the first label in their list” with the available search tool. It can only search for pets that have a given label anywhere, not by label order.","stopReason":"stop","usage":{"input_tokens":402,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b09","trial":1,"text":"I can’t search for pets by vaccination booster date with the available pet-search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b09","trial":2,"text":"I can’t search for pets by vaccination or booster-date fields with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b08","trial":5,"text":"I can’t determine which pets have “indoor” as the first label with the available search tool. It can only search for pets that have a given tag anywhere in their label list, not by label position.","stopReason":"stop","usage":{"input_tokens":402,"output_tokens":89}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b07","trial":5,"text":"I can’t express “pets with at least one label other than indoor” with the available search tool. The tool can only search for pets that have a specific label, not exclude a label or test for “any non-indoor label.”","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":262}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b09","trial":3,"text":"I can’t search for pets by vaccination or booster-date records with the available pet search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":64}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b09","trial":4,"text":"I can’t search for pets by vaccination or booster-date information with the available pet search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b09","trial":5,"text":"I can’t search for pets by vaccination or booster-date records with the available tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":63}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b10","trial":3,"text":"I can’t answer that with the available pet search tool, because it doesn’t support filtering by vaccination records or missing booster dates.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":60}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b10","trial":2,"text":"I can’t express that search with the available pet-store tool, because it has no vaccination or booster-date filters.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b10","trial":4,"text":"I can’t answer that with the available pet-search tool, because it doesn’t support filtering by vaccination records or missing booster dates.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":59}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b10","trial":5,"text":"I can’t search for pets by vaccination or booster-date information with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b10","trial":1,"text":"I can’t search for “pets with at least one vaccination with no booster date recorded” using the available pet-store search tool. The tool doesn’t include vaccination or booster-date fields.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":82}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b11","trial":1,"text":"I can’t express that search with the available pet-store tool, because it has no vaccination-record filter or field.","stopReason":"stop","usage":{"input_tokens":414,"output_tokens":68}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b11","trial":3,"text":"I can’t express that query with the available pet search tool, because it has no vaccination-related filters or fields.","stopReason":"stop","usage":{"input_tokens":414,"output_tokens":55}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b11","trial":5,"text":"I can’t express that search with the available pet-store tool, because it has no vaccination-record filter or field.","stopReason":"stop","usage":{"input_tokens":414,"output_tokens":58}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b11","trial":2,"text":"I can’t express that query with the available pet search tool, because it has no vaccination-history filter.","stopReason":"stop","usage":{"input_tokens":414,"output_tokens":170}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b11","trial":4,"text":"I can’t express that search with the available pet-store tool, because it has no vaccination-record filter.","stopReason":"stop","usage":{"input_tokens":414,"output_tokens":76}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b12","trial":1,"text":"I can’t express “pets that do not have the indoor label” with the available search tool, because it only supports finding pets that must carry a given label, not excluding a label or checking for empty label lists.","stopReason":"stop","usage":{"input_tokens":413,"output_tokens":103}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b12","trial":4,"text":"I can’t express that search with the available tool. The tool can only find pets that **have** a given label, not pets that **lack** a label like “indoor” or have an empty label list.","stopReason":"stop","usage":{"input_tokens":413,"output_tokens":91}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b12","trial":2,"text":"I can’t express “pets that do not have the indoor label, including empty label lists” with the available search tool, because it only supports requiring a pet to have a given tag, not excluding one.","stopReason":"stop","usage":{"input_tokens":413,"output_tokens":203}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b13","trial":1,"text":"I can’t express that search with the available tool. It can only filter for a single required label, not pets with only “indoor” labels or with no labels at all.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":129}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b13","trial":3,"text":"I can’t express that query with the available pet search tool. It can search for pets with a single required label, but it can’t filter for “only indoor labels” or “no labels at all.”","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":113}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b12","trial":5,"text":"I can’t express “pets that do **not** have the `indoor` label, including empty label lists” with the available search tool. It only supports requiring a pet to have a given tag, not excluding one.","stopReason":"stop","usage":{"input_tokens":413,"output_tokens":206}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b13","trial":2,"text":"I can’t express “nothing but indoor labels, or no labels at all” with the available search tool. It can only filter for pets that have a specific single label, not pets with only that label or with no labels.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":142}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b13","trial":4,"text":"I can’t express that search with the available pet-store tool. It can only filter for a single required label, not for pets with exclusively “indoor” labels or with no labels at all.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":118}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b12","trial":3,"text":"I can’t express that search with the available pet-store tool, because it only supports finding pets that **do** have a given label, not pets that **lack** a label such as `indoor` or have an empty label list.","stopReason":"stop","usage":{"input_tokens":413,"output_tokens":278}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"b13","trial":5,"text":"I can’t express that search with the available tool, because it can only filter for pets that have a single specified tag—not pets with only that tag, nor pets with no labels at all.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":149}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c01","trial":1,"text":"I can’t search for “microchip field is present but blank” with the available pet-store search tool, because it doesn’t provide any microchip-field filter or presence/blank check.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":82}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c01","trial":3,"text":"I can’t search for pets with a “microchip” field that is present but blank using the available pet search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c01","trial":2,"text":"I can’t search for “microchip field is present but blank” with the available pet-store search tool, because it doesn’t support filtering by a microchip field or checking for blank/present values.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":70}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c01","trial":4,"text":"I can’t search for pets with a “microchip” field that is present but blank using the available pet search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":55}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c02","trial":1,"text":"I can’t search for pets with no microchip field recorded because the available pet search tool doesn’t include any microchip-related filter or field.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":52}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c01","trial":5,"text":"I can’t express that search with the available pet-store tool because it has no parameter for the microchip field or for checking whether a field is present but blank.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c02","trial":3,"text":"I can’t search for pets with no microchip field recorded because the available pet-search tool doesn’t include any microchip-related filter or field.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c02","trial":4,"text":"I can’t search for pets with no microchip field recorded because the available pet-search tool doesn’t support filtering by microchip field presence or absence.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c02","trial":2,"text":"I can’t express “pets with no microchip field recorded at all” using the available pet search tool, since it has no microchip-related filter or field-existence query.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":133}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c03","trial":2,"text":"I can’t express that search with the available pet-store tool, because it doesn’t provide a filter for whether a `notes` field is present.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":72}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c02","trial":5,"text":"I can’t express “pets with no microchip field recorded at all” using the available pet search tool, because it has no parameter for microchip presence or missing fields.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c03","trial":1,"text":"I can’t search for “notes field present” with the available pet-store tool, because it has no filter for whether a notes field exists.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c03","trial":3,"text":"I can’t search for pets based on whether a `notes` field is present, because the available search tool doesn’t support filtering by that field.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":63}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c03","trial":4,"text":"I can’t search for pets based on whether a `notes` field is present because the available search tool doesn’t support filtering by `notes` or field existence.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":69}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c03","trial":5,"text":"I can’t express “notes field present” with the available pet search tool, because it doesn’t provide a notes-field filter or a way to query arbitrary fields.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":60}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c04","trial":2,"text":"I can’t search by whether a shelter’s kennel capacity has been recorded, because the available pet search tool doesn’t include that shelter-capacity field.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c04","trial":3,"text":"I can’t search by whether a shelter’s kennel capacity has been recorded, because the available pet-search tool doesn’t include a kennel-capacity field.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c04","trial":1,"text":"I can’t search by whether a shelter’s kennel capacity has been recorded, because the available pet-search tool doesn’t include a kennel-capacity field.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":70}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c04","trial":4,"text":"I can’t search by whether a shelter’s kennel capacity has been recorded with the available pet-search tool.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c04","trial":5,"text":"I can’t search for pets by whether a shelter’s kennel capacity has been recorded—the available search tool doesn’t include kennel capacity or missing shelter-capacity fields.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c07","trial":2,"text":"I can’t express that search with the available pet-store tool, because it has no filter for chip value, “no chip,” or whether the chip field exists.","stopReason":"stop","usage":{"input_tokens":423,"output_tokens":90}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c07","trial":3,"text":"I can’t express that search with the available pet-store tool because it has no filter for chip ID, chip presence, or missing chip fields.","stopReason":"stop","usage":{"input_tokens":423,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c05","trial":3,"text":"I can’t express that query with the available pet-search tool: it has no microchip field/filter and no “not equal to CHIP-001” capability.","stopReason":"stop","usage":{"input_tokens":417,"output_tokens":172}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c07","trial":1,"text":"I can’t express that search with the available pet-store tool because it has no chip/microchip field filter or way to distinguish “no chip recorded” from “chip field missing.”","stopReason":"stop","usage":{"input_tokens":423,"output_tokens":103}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c07","trial":4,"text":"I can’t express that search with the available pet-store tool, because it has no parameter for chip ID, “no chip,” or whether the chip field exists.","stopReason":"stop","usage":{"input_tokens":423,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c07","trial":5,"text":"I can’t express that search with the available pet-store tool, because it has no chip/microchip field or “field exists / missing” filter.","stopReason":"stop","usage":{"input_tokens":423,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c08","trial":1,"input":{"status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":396,"output_tokens":142}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c08","trial":2,"input":{"status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":396,"output_tokens":99}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c08","trial":3,"input":{"status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":396,"output_tokens":113}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c08","trial":4,"input":{"status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":396,"output_tokens":109}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c05","trial":4,"text":"I can’t express that query with the available pet search tool. It has no microchip field/filter, nor a way to search for “has a chip” and exclude `CHIP-001` while leaving out unknowns.","stopReason":"stop","usage":{"input_tokens":417,"output_tokens":411}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c05","trial":5,"text":"I can’t express that search with the available pet-store tool: it supports filtering by a single tag, but not “has a microchip” or “microchip is not CHIP-001,” nor can it exclude unknown chip status.","stopReason":"stop","usage":{"input_tokens":417,"output_tokens":409}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c08","trial":5,"input":{"status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":396,"output_tokens":144}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c05","trial":1,"text":"I can’t express that query with the available pet-search tool. It doesn’t provide a microchip field, a “chip is known” filter, or a “microchip is not CHIP-001” condition.","stopReason":"stop","usage":{"input_tokens":417,"output_tokens":533}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c12","trial":1,"text":"I can’t express that search with the available pet-store tool because it has no microchip/chip ID field or way to filter for blank/missing/different chip values.","stopReason":"stop","usage":{"input_tokens":426,"output_tokens":101}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c05","trial":2,"input":{"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":417,"output_tokens":358},"repair":{"text":"I can’t express that search with the available pet-store tool. It has no microchip/chip-ID field and no way to filter for “chip exists and is not CHIP-001” while excluding unknowns.","stopReason":"stop","usage":{"input_tokens":477,"output_tokens":197},"problemShown":{"type":"malformed-query","detail":"at least one search parameter is required.","pointer":"/"}}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c12","trial":2,"text":"I can’t answer that with the available pet search tool, because it has no field for microchip ID or whether a chip is blank/missing.","stopReason":"stop","usage":{"input_tokens":426,"output_tokens":98}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c12","trial":3,"text":"I can’t express that search with the available pet-store tool, because it has no microchip/chip-ID field or way to filter for blank or missing chip values.","stopReason":"stop","usage":{"input_tokens":426,"output_tokens":105}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c12","trial":5,"text":"I can’t express that search with the available pet-store tool because it has no microchip/chip-ID field or way to query blank or missing chip values.","stopReason":"stop","usage":{"input_tokens":426,"output_tokens":111}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c11","trial":2,"input":{"maxWeightKg":1,"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":503}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c11","trial":3,"text":"I can’t express that exact search with the available tool: it only supports inclusive weight bounds, and “outside the 1-to-5 kg range” requires a split condition (<1 kg OR >5 kg) with strict endpoints.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":474}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c11","trial":4,"input":{"maxWeightKg":1,"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":579}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c11","trial":1,"text":"I can’t express “outside the 1-to-5 kg range” exactly with the search tool, because it only supports inclusive minimum/maximum weight filters and doesn’t support a strict “less than 1 OR greater than 5” query.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":571}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c13","trial":1,"text":"I can’t express that search with the available tool, because it requires an OR condition: pets born before 2020-01-01 **or** pets with no birth date on record. The tool can filter by a date range, but it can’t include missing birth dates or combine those conditions.","stopReason":"stop","usage":{"input_tokens":421,"output_tokens":268}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c13","trial":2,"text":"I can’t express that search with the available tool because it can filter pets born before a date, but it cannot specifically include pets with no birth date on record.","stopReason":"stop","usage":{"input_tokens":421,"output_tokens":174}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c11","trial":5,"text":"I can’t express “outside the 1-to-5 kg range” exactly with the search tool, because it only supports inclusive minimum/maximum weight filters and would include pets weighing exactly 1 kg or 5 kg.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":566}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c12","trial":4,"text":"I can’t express that search with the available pet-store tool, because it has no microchip/chip field or “not CHIP-001 / blank / missing” filter.","stopReason":"stop","usage":{"input_tokens":426,"output_tokens":343}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d01","trial":1,"text":"I can’t express “notes value is a number” with the available pet search tool, because it doesn’t support filtering by a notes field or by value type.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c13","trial":3,"text":"I can’t express that search with the available tool. It can filter for pets born before 2020-01-01, but it cannot also include pets with no birth date on record in the same query.","stopReason":"stop","usage":{"input_tokens":421,"output_tokens":307}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d01","trial":2,"text":"I can’t search by whether a pet’s `notes` value is a number with the available pet-search tool.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":56}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d01","trial":5,"text":"I can’t search by whether a pet’s `notes` value is a number with the available tool.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":47}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d01","trial":3,"text":"I can’t express that search with the available pet-store tool, because it doesn’t provide a way to filter or inspect a `notes` field or its value type.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":81}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d01","trial":4,"text":"I can’t express that query with the available pet search tool, because it has no parameter for filtering by a `notes` field or by value type.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":68}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c13","trial":4,"text":"I can’t express that exact search with the available pet-search tool, because it can filter for pets born before 2020, but it cannot include pets with no birth date on record or perform that “OR” condition.","stopReason":"stop","usage":{"input_tokens":421,"output_tokens":328}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d02","trial":1,"text":"I can’t express that search with the available pet-store tool, because it doesn’t provide any way to filter or inspect a pet’s `notes` value.","stopReason":"stop","usage":{"input_tokens":410,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"c13","trial":5,"text":"I can’t express that search with the available tool, because it requires an OR condition: pets born before 2020-01-01 **or** pets with no birth date on record. The tool can filter by birth-date range, but it can’t include missing birth dates in the same query.","stopReason":"stop","usage":{"input_tokens":421,"output_tokens":262}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d02","trial":4,"text":"I can’t answer that with the available pet search tool, because it doesn’t support filtering or inspecting a `notes` value.","stopReason":"stop","usage":{"input_tokens":410,"output_tokens":59}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d02","trial":5,"text":"I can’t search by a pet’s `notes` value with the available tool, so I can’t answer that query directly.","stopReason":"stop","usage":{"input_tokens":410,"output_tokens":57}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d02","trial":3,"text":"I can’t express that search with the available pet-store tool because it doesn’t support filtering by a `notes` value or checking whether a value is a whole number.","stopReason":"stop","usage":{"input_tokens":410,"output_tokens":68}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d02","trial":2,"text":"I can’t express that search with the available pet-store tool, because it has no filter for a `notes` value or for checking whether a value is a whole number.","stopReason":"stop","usage":{"input_tokens":410,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d03","trial":1,"text":"I can’t express that with the available pet search tool, since it doesn’t provide a way to filter by the `notes` field or by whether a field’s value is a list.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":70}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d03","trial":2,"text":"I can’t search for pets by whether their `notes` value is a list with the available tool.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":55}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d03","trial":4,"text":"I can’t express that query with the available pet-store search tool, because it doesn’t support filtering or inspecting a `notes` field or its data type.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d04","trial":1,"text":"I can’t express that search with the available pet-store tool, because it has no filter for a `notes` field or for distinguishing explicitly blank values from missing values.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":64}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d03","trial":5,"text":"I can’t express that search with the available pet-store tool, because it doesn’t support filtering or inspecting a pet’s `notes` field or checking whether a value is a list.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d04","trial":2,"text":"I can’t express that with the available pet search tool. It doesn’t provide a `notes` filter or a way to distinguish explicitly blank values from missing fields.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d03","trial":3,"text":"I can’t search for pets by whether their `notes` value is a list with the available tool.","stopReason":"stop","usage":{"input_tokens":401,"output_tokens":139}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d04","trial":3,"text":"I can’t express that search with the available pet-store tool, because it has no parameter for filtering on the `notes` field or distinguishing blank values from missing values.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":86}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d04","trial":4,"text":"I can’t express that search with the available pet-store tool, because it doesn’t provide a parameter for filtering by `notes` value or distinguishing blank from missing fields.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":69}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d05","trial":1,"text":"I can’t search by a `notes` value with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d05","trial":2,"text":"I can’t search by a `notes` value with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":43}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d04","trial":5,"text":"I can’t express that search with the available pet-store tool, because it does not provide a `notes` field filter or a way to distinguish explicitly blank values from missing values.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":76}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d05","trial":3,"text":"I can’t search by a `notes` value with the available pet-store tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":42}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d06","trial":1,"text":"I can’t search by a `notes` value with the available pet-search tool, so I can’t determine which pets have a numeric notes value above 2.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":65}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d05","trial":5,"text":"I can’t search by a `notes` value with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":47}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d05","trial":4,"text":"I can’t search by a pet’s `notes` value with the available tool, since it doesn’t support a `notes` parameter.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d06","trial":5,"text":"I can’t search by a pet’s `notes` value with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":55}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d06","trial":3,"text":"I can’t search by a `notes` value with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":50}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d06","trial":4,"text":"I can’t search by a pet’s `notes` value with the available tool, so I can’t answer that query.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":50}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d06","trial":2,"text":"I can’t search by a `notes` value with the available pet-store search tool, since it doesn’t include a `notes` parameter or numeric comparison for notes.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":64}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d07","trial":1,"text":"I can’t express that search with the available pet-store tool, because it doesn’t support filtering or sorting by a `notes` value.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d07","trial":4,"text":"I can’t search by a pet’s `notes` value or compare text sorting with this tool.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":50}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d07","trial":3,"text":"I can’t search by a pet’s `notes` value with the available tool, so I can’t answer that query.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d07","trial":2,"text":"I can’t express that search with the available pet-store tool: it doesn’t support filtering by a `notes` value or comparing text sort order.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d07","trial":5,"text":"I can’t express that search with the available pet-store tool, because it has no parameter for filtering or sorting by a “notes” value.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":58}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d08","trial":1,"text":"I can’t express that search with the available pet-store tool, because it doesn’t provide a filter for a “notes” value or “not equal to 3.”","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":74}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d08","trial":2,"text":"I can’t express “notes value is not 3” with the available pet search tool, since it has no `notes` field or “not equal” filter.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":125}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d09","trial":2,"text":"I can’t express that search with the available pet-search tool, because it has no parameter for a “notes” value.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d08","trial":5,"text":"I can’t express that search with the available pet-store tool: it doesn’t have a `notes` field or a “not equal to 3” filter.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":73}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d09","trial":3,"text":"I can’t search or confirm pets by a “notes” value with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":76}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d08","trial":3,"text":"I can’t express “notes value is not 3” with the available pet-search tool, because it has no searchable `notes` parameter.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":161}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d08","trial":4,"text":"I can’t express “notes value is not 3” with the available pet search tool, because it has no `notes` filter or field parameter.","stopReason":"stop","usage":{"input_tokens":405,"output_tokens":187}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d09","trial":4,"text":"I can’t express that search with the available pet-store tool, because it has no parameter for a “notes” value.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":67}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e01","trial":1,"text":"I can’t express that search with the available pet-store tool, because it has no parameter for a field named `$rate` or any rate filter.","stopReason":"stop","usage":{"input_tokens":410,"output_tokens":54}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d09","trial":5,"text":"I can’t express that search with the available pet-store tool, because it doesn’t include a filter for a “notes” value.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"d09","trial":1,"text":"I can’t express that search with the available pet-store tool, because it has no filter for a “notes” value.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":72}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e01","trial":2,"text":"I can’t express that search with the available pet-store tool. It doesn’t include a `$rate` or rate filter parameter.","stopReason":"stop","usage":{"input_tokens":410,"output_tokens":75}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e02","trial":1,"text":"I can’t search by a field named `size.raw` with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":421,"output_tokens":46}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e01","trial":3,"text":"I can’t search for pets by a “rate” field—especially one literally named with a leading dollar sign—because the available pet search tool does not support that parameter.","stopReason":"stop","usage":{"input_tokens":410,"output_tokens":70}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e01","trial":4,"text":"I can’t search for pets by a “rate” field or a field named with a leading dollar sign using the available pet-store search tool. The tool only supports filters like species, status, birth date, weight, tag, shelter city, neutered, and name.","stopReason":"stop","usage":{"input_tokens":410,"output_tokens":94}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e01","trial":5,"text":"I can’t express that search with the available pet-store tool. It doesn’t support filtering by a field named `$rate` or by “rate above 5.”","stopReason":"stop","usage":{"input_tokens":410,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e02","trial":3,"text":"I can’t search by a field named `size.raw` with the available pet-store search tool. It only supports filters like species, status, birth date, weight, tag, shelter city, neutered status, and name substring.","stopReason":"stop","usage":{"input_tokens":421,"output_tokens":79}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e02","trial":2,"text":"I can’t search by the literal field `size.raw` with the available pet-store search tool. It only supports filters like species, status, birth date, weight, tag, shelter city, neutered status, and name substring.","stopReason":"stop","usage":{"input_tokens":421,"output_tokens":80}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e02","trial":5,"text":"I can’t search for pets by the literal field `size.raw` with the available tool.","stopReason":"stop","usage":{"input_tokens":421,"output_tokens":52}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e02","trial":4,"text":"I can’t search by a field named `size.raw` with the available pet-search tool. It only supports filters like species, status, birth date, weight, tag, shelter city, neutered, and name substring.","stopReason":"stop","usage":{"input_tokens":421,"output_tokens":71}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e03","trial":2,"text":"I can’t express “name starts with C and ends with o” with the available pet search tool. It only supports a case-insensitive substring search for names, not start/end matching.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":87}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e03","trial":3,"text":"I can’t express “name starts with C and ends with o” using the available pet search tool.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":77}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e03","trial":4,"text":"I can’t express “name starts with C and ends with o” with the available pet-search tool. It only supports case-insensitive name substring searches, not start/end pattern matching.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":94}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e03","trial":5,"text":"I can’t express “name starts with C and ends with o” using the available pet search tool, which only supports a case-insensitive name substring search.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":125}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e03","trial":1,"text":"I can’t express “name starts with C and ends with o” using the available pet search tool. It only supports a case-insensitive substring search for names, not starts/ends pattern matching.","stopReason":"stop","usage":{"input_tokens":404,"output_tokens":196}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e04","trial":2,"text":"I can’t express “name starting with A, B, or C” or “capitals only” with the available search tool. It only supports a case-insensitive name substring search, not starts-with or case-sensitive matching.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":114}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e04","trial":1,"text":"I can’t express “name starting with A, B, or C” with “capitals only” using the available pet search tool. The tool only supports case-insensitive name substring searches, not starts-with or case-sensitive matching.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":119}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e04","trial":3,"text":"I can’t express “name starting with capital A, B, or C” with the available pet search tool. It only supports a case-insensitive name substring search, not starts-with or capitalization matching.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":132}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e04","trial":5,"text":"I can’t express “name starting with A, B, or C” with capitals only using the available search tool. It only supports case-insensitive name substring matching, not case-sensitive “starts with” or OR conditions.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":177}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e05","trial":1,"text":"I can’t express “name starting with A, B, or C” with the available pet search tool. It only supports case-insensitive name substring matching, not prefix matching or multiple starting letters.","stopReason":"stop","usage":{"input_tokens":407,"output_tokens":106}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e04","trial":4,"text":"I can’t express “name starting with A, B, or C — capitals only” with the available pet search tool. It only supports case-insensitive name substring matching, not starts-with or capitalization-sensitive matching.","stopReason":"stop","usage":{"input_tokens":406,"output_tokens":239}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e05","trial":5,"text":"I can’t express “name starting with A, B, or C” using the available pet-search tool. It only supports a case-insensitive substring match for names, not prefix matching or multiple starting letters.","stopReason":"stop","usage":{"input_tokens":407,"output_tokens":126}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e05","trial":4,"text":"I can’t express “name starting with a, b, or c” with the available pet search tool. It only supports a case-insensitive substring match on names, not prefix matching or multiple starting letters.","stopReason":"stop","usage":{"input_tokens":407,"output_tokens":147}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e05","trial":3,"text":"I can’t express “name starts with A, B, or C” using the available pet search tool. It only supports case-insensitive name substring matching, not “starts with” or multiple initial-letter matching.","stopReason":"stop","usage":{"input_tokens":407,"output_tokens":220}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e06","trial":3,"text":"I can’t express “name ending in s” with the available pet search tool. It only supports case-insensitive name substring matching, not suffix/ending matching.","stopReason":"stop","usage":{"input_tokens":399,"output_tokens":113}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e06","trial":1,"text":"I’m sorry, but the pet search tool can’t search for names ending in a specific letter. It only supports name substring matching, not suffix matching.","stopReason":"stop","usage":{"input_tokens":399,"output_tokens":185}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"f01","trial":1,"text":"I can’t search by internal staff notes with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":45}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e06","trial":2,"text":"I can’t express “name ending in s” with the available pet search tool. It only supports case-insensitive name substring matching, not suffix/ending matching.","stopReason":"stop","usage":{"input_tokens":399,"output_tokens":165}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e06","trial":4,"text":"I can’t express “name ending in s” with the available pet search tool. It only supports a case-insensitive name substring search, not suffix matching.","stopReason":"stop","usage":{"input_tokens":399,"output_tokens":116}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"f01","trial":2,"text":"I can’t search by internal staff notes with the available pet-search tool, so I can’t answer that query.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":66}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"f01","trial":4,"text":"I can’t search by internal staff notes with the available pet-store search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":58}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e06","trial":5,"text":"I can’t express “name ending in s” with the available pet search tool. It only supports a case-insensitive name substring search, not suffix/ending matching.","stopReason":"stop","usage":{"input_tokens":399,"output_tokens":165}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"e05","trial":2,"text":"I can’t express that search with the available tool: it supports case-insensitive “name contains” searches, but not “name starts with” or grouped initial-letter matching.","stopReason":"stop","usage":{"input_tokens":407,"output_tokens":313}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"f01","trial":3,"text":"I can’t search by internal staff notes with the available pet search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":52}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"f01","trial":5,"text":"I can’t search by internal staff notes with the available pet-search tool.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":61}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"f02","trial":2,"text":"I can’t run a free-text search across all pet record fields with the available tool. It only supports specific filters like species, status, city, tag, and name substring.","stopReason":"stop","usage":{"input_tokens":409,"output_tokens":87}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m01","trial":1,"input":{"tag":"indoor"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":19}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m01","trial":2,"input":{"tag":"indoor"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":19}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"f02","trial":1,"text":"I can’t run a free-text search across all pet records with the available tool. It only supports structured filters like species, status, shelter city, tag, or name substring.","stopReason":"stop","usage":{"input_tokens":409,"output_tokens":108}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m01","trial":4,"input":{"tag":"indoor"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":19}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m01","trial":3,"input":{"tag":"indoor"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":19}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m01","trial":5,"input":{"tag":"indoor"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":19}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"f02","trial":3,"text":"I can’t run a free-text search across all pet records with the available tool. It only supports specific filters such as name substring, species, status, tag, shelter city, etc.","stopReason":"stop","usage":{"input_tokens":409,"output_tokens":137}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"f02","trial":4,"text":"I can’t run a free-text search across all pet record fields with the available tool. It only supports specific filters like species, status, city, tag, and name substring.","stopReason":"stop","usage":{"input_tokens":409,"output_tokens":128}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"f02","trial":5,"text":"I can’t run a true free-text search across all pet record fields with the available tool. It only supports structured filters like species, status, city, tag, or a name substring search.","stopReason":"stop","usage":{"input_tokens":409,"output_tokens":146}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m02","trial":1,"text":"I’m sorry, but that search can’t be expressed with the available pet search tool. It doesn’t support filtering by microchip ID, checking blank/missing microchips, or excluding a specific chip value like `CHIP-001`.","stopReason":"stop","usage":{"input_tokens":424,"output_tokens":100}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m03","trial":2,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":36}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m03","trial":1,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":38}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m02","trial":5,"text":"I can’t express that search with the available pet-store tool: it doesn’t have a microchip field or a way to search for “not CHIP-001” / blank or missing chips.","stopReason":"stop","usage":{"input_tokens":424,"output_tokens":76}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m02","trial":3,"text":"I can’t express that search with the available pet-store tool because it has no microchip/chip-ID filter, nor a way to query blank or missing chip values.","stopReason":"stop","usage":{"input_tokens":424,"output_tokens":78}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m03","trial":3,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":36}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m03","trial":4,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":34}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m03","trial":5,"input":{"status":"available"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":34}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m02","trial":4,"text":"I can’t express that search with the available pet-store tool: it has no microchip field filter and no “not equal to / missing” filter.","stopReason":"stop","usage":{"input_tokens":424,"output_tokens":148}} +{"provider":"gpt-5.5","arm":"fixed-params","caseId":"m02","trial":2,"text":"I can’t express that search with the available pet-search tool: it has no microchip field and no “not equal / blank or missing” filter.","stopReason":"stop","usage":{"input_tokens":424,"output_tokens":246}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a01","trial":1,"input":{"filter":{"status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1099,"output_tokens":122}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a01","trial":2,"input":{"filter":{"status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1099,"output_tokens":116}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a01","trial":3,"input":{"filter":{"status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1099,"output_tokens":128}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a01","trial":4,"input":{"filter":{"status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1099,"output_tokens":129}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a01","trial":5,"input":{"filter":{"status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1099,"output_tokens":133}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a02","trial":1,"input":{"filter":{"species":"cat","status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1098,"output_tokens":83}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a02","trial":2,"input":{"filter":{"species":"cat","status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1098,"output_tokens":89}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a02","trial":3,"input":{"filter":{"species":"cat","status":"available"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1098,"output_tokens":101}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a02","trial":5,"input":{"filter":{"species":"cat","status":"available"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1098,"output_tokens":89}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a03","trial":1,"input":{"filter":{"$or":[{"status":"available"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":137}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a02","trial":4,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7901, Requested 1214. Please try again in 8.3625s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a03","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/$or/1/status`: value must be \\\"available\\\", `/filter/$or/1/status`: value must be \\\"pending\\\", `/filter/$or/1/status`: value must be \\\"sold\\\", `/filter/$or/1/status`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a03","trial":3,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/$or/1/status`: value must be \\\"available\\\", `/filter/$or/1/status`: value must be \\\"pending\\\", `/filter/$or/1/status`: value must be \\\"sold\\\", `/filter/$or/1/status`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a03","trial":5,"input":{"filter":{"$or":[{"status":"available"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":173}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a04","trial":1,"input":{"filter":{"$and":[{"weightKg":{"$gte":1}},{"weightKg":{"$lte":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":186}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a03","trial":4,"input":{"filter":{"$or":[{"status":"available"},{"status":"pending"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":142}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a04","trial":3,"input":{"filter":{"$and":[{"weightKg":{"$gte":1}},{"weightKg":{"$lte":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":397}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a04","trial":2,"input":{"filter":{"$and":[{"weightKg":{"$gte":1}},{"weightKg":{"$lte":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":162}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a04","trial":4,"input":{"filter":{"weightKg":{"$gte":1,"$lte":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":216}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a05","trial":1,"input":{"filter":{"born":{"$gte":"2020-01-01"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":181}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a05","trial":2,"input":{"filter":{"born":{"$gte":"2020-01-01"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":109}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a04","trial":5,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7680, Requested 1300. Please try again in 7.35s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a05","trial":3,"input":{"filter":{"born":{"$gte":"2020-01-01"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":119}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a05","trial":5,"input":{"filter":{"born":{"$gte":"2020-01-01"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":120}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a06","trial":1,"text":"I’m sorry, but the available pet‑store search tool doesn’t support a “starts with” or case‑sensitive string‑matching operation, so I can’t express “name beginning with a lower‑case b” as a query.","stopReason":"stop","usage":{"input_tokens":1114,"output_tokens":160}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a06","trial":2,"text":"I’m sorry, but the available search tool can’t express a “name starts with a lower‑case b” condition. It only supports exact matches and the logical operators shown in its schema.","stopReason":"stop","usage":{"input_tokens":1114,"output_tokens":159}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a06","trial":3,"text":"I’m sorry, but the available pet‑store search tool doesn’t provide a way to filter on a name’s initial character or to perform case‑sensitive “starts‑with” string matching. Therefore I can’t express that query with the tool as described.","stopReason":"stop","usage":{"input_tokens":1114,"output_tokens":169}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a05","trial":4,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7815, Requested 1285. Please try again in 8.25s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a06","trial":4,"text":"I’m sorry, but the pet‑store search tool doesn’t provide a way to filter names by a case‑sensitive prefix (e.g., “starts with a lower‑case b”). Therefore I can’t answer that query using the available tool.","stopReason":"stop","usage":{"input_tokens":1114,"output_tokens":196}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a07","trial":1,"text":"I’m sorry, but the available pet‑search tool doesn’t support a case‑insensitive “contains” or substring query on the `name` field, so I can’t express “name includes the letter o” with it.","stopReason":"stop","usage":{"input_tokens":1105,"output_tokens":164}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a06","trial":5,"input":{"filter":{"name":{"$regex":"^b"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":208}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a07","trial":3,"text":"I’m sorry, but the available pet‑store search tool can only filter on exact values or simple comparisons—it doesn’t support case‑insensitive substring matching on the name field. So I can’t express “names that contain the letter o” with this tool.","stopReason":"stop","usage":{"input_tokens":1105,"output_tokens":161}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a07","trial":4,"text":"I’m sorry, but the available pet‑search tool doesn’t provide a way to query for a substring (such as the letter “o”) within a pet’s name, so I can’t express that request with the tool.","stopReason":"stop","usage":{"input_tokens":1105,"output_tokens":125}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a07","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7677, Requested 1502. Please try again in 8.8425s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a07","trial":5,"text":"I’m sorry, but the pet‑search tool doesn’t support a “contains” or case‑insensitive substring query on the `name` field, so I can’t express “names that contain the letter o (ignoring capitalization)” with it.","stopReason":"stop","usage":{"input_tokens":1105,"output_tokens":198}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a08","trial":1,"text":"I’m sorry, but the available pet‑store search tool doesn’t support searching for a substring (like “50%”) within a pet’s name, so I can’t answer that query using the tool.","stopReason":"stop","usage":{"input_tokens":1103,"output_tokens":129}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a08","trial":2,"text":"I’m sorry, but the pet‑store search tool can only test for exact matches or the listed comparison operators; it doesn’t support searching for a substring like “50%” within a pet’s name. Therefore, I can’t express that query with the available tool.","stopReason":"stop","usage":{"input_tokens":1103,"output_tokens":194}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a08","trial":4,"text":"I’m sorry, but the search tool only supports exact matches and basic comparisons on fields—it can’t query for a substring like “50%” inside a pet’s name.","stopReason":"stop","usage":{"input_tokens":1103,"output_tokens":115}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a08","trial":5,"text":"I’m sorry, but the pet‑store search tool can only filter by exact values or numeric comparisons—it doesn’t support searching for a substring such as “50%” within a pet’s name.","stopReason":"stop","usage":{"input_tokens":1103,"output_tokens":142}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a09","trial":1,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":160}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a09","trial":2,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":141}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a08","trial":3,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 6643, Requested 1500. Please try again in 1.0725s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a09","trial":4,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":165}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a09","trial":5,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":131}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a10","trial":1,"input":{"filter":{"$and":[{"tags":{"$some":{"$in":["trained"]}}},{"tags":{"$some":{"$in":["outdoor"]}}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1102,"output_tokens":226}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a09","trial":3,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 6556, Requested 1500. Please try again in 420ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a10","trial":3,"input":{"filter":{"$and":[{"tags":{"$some":{"$in":["trained"]}}},{"tags":{"$some":{"$in":["outdoor"]}}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1102,"output_tokens":246}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a10","trial":2,"input":{"filter":{"$and":[{"tags":{"$some":{"$in":["trained"]}}},{"tags":{"$some":{"$in":["outdoor"]}}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1102,"output_tokens":231}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a10","trial":4,"input":{"filter":{"$and":[{"tags":{"$some":{"$eq":"trained"}}},{"tags":{"$some":{"$eq":"outdoor"}}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1102,"output_tokens":240}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a11","trial":1,"text":"I’m sorry, but the available pet‑store search tool doesn’t provide a way to filter records based on the number of items in the `tags` array, so I can’t express “pets that have three or more labels” with it.","stopReason":"stop","usage":{"input_tokens":1098,"output_tokens":206}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a11","trial":2,"text":"I’m sorry, but the pet‑search tool doesn’t provide a way to filter pets by the number of tags (labels) they have, so I can’t answer that query using the available tool.","stopReason":"stop","usage":{"input_tokens":1098,"output_tokens":270}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a11","trial":3,"text":"I’m sorry, but the available pet‑search tool doesn’t provide a way to filter records by the number of items in an array field (such as the `tags` “labels”). Consequently, I can’t express “pets that have three or more labels” with the given search function.","stopReason":"stop","usage":{"input_tokens":1098,"output_tokens":292}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a11","trial":4,"text":"I’m sorry, but the available search tool doesn’t provide a way to filter pets by the number of tags (labels) they have, so I can’t answer that query directly.","stopReason":"stop","usage":{"input_tokens":1098,"output_tokens":287}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a10","trial":5,"input":{"filter":{"$and":[{"tags":{"$some":{"$in":["trained"]}}},{"tags":{"$some":{"$in":["outdoor"]}}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1102,"output_tokens":252}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a12","trial":1,"text":"I’m sorry, but the available pet‑search tool doesn’t provide a way to query for records where the `tags` field (the pet’s labels) is completely absent or an empty array. Consequently, I can’t retrieve “pets with no labels at all” using this tool.","stopReason":"stop","usage":{"input_tokens":1098,"output_tokens":513}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a12","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/$or/1/tags/$eq`: expected array, but got null]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"$or\\\": [\\n {\\n \\\"tags\\\": {\\n \\\"$eq"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a11","trial":5,"text":"I’m sorry, but the available pet‑search tool doesn’t provide a way to filter records by the number of items in the `tags` (labels) array, so I can’t answer that query with it.","stopReason":"stop","usage":{"input_tokens":1098,"output_tokens":248}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a12","trial":4,"input":{"filter":{"tags":{"$eq":[]}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1098,"output_tokens":506}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a12","trial":5,"input":{"filter":{"tags":{"$eq":[]}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1098,"output_tokens":205}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a13","trial":1,"input":{"filter":{"shelter.city":"Athens"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1100,"output_tokens":82}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a13","trial":2,"input":{"filter":{"shelter.city":"Athens"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1100,"output_tokens":92}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a13","trial":3,"input":{"filter":{"shelter.city":"Athens"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1100,"output_tokens":84}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a13","trial":4,"input":{"filter":{"shelter.city":"Athens"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1100,"output_tokens":69}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a13","trial":5,"input":{"filter":{"shelter.city":"Athens"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1100,"output_tokens":69}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a14","trial":1,"input":{"filter":{"shelter.capacity":{"$lte":12}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":198}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a12","trial":3,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/tags/$eq`: expected array, but got null]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"tags\\\": {\\n \\\"$eq\\\": null,\\n \\\"$unknownAs\\\": true\\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a14","trial":2,"input":{"filter":{"shelter.capacity":{"$lte":12}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":140}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a14","trial":4,"input":{"filter":{"shelter.capacity":{"$lte":12}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":203}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a14","trial":5,"input":{"filter":{"shelter.capacity":{"$lte":12}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":159}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a15","trial":1,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1106,"output_tokens":207}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a15","trial":2,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1106,"output_tokens":228}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a14","trial":3,"input":{"filter":{"shelter.capacity":{"$lte":12}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":152}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a15","trial":4,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1106,"output_tokens":226}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a15","trial":3,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1106,"output_tokens":230}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a16","trial":1,"input":{"filter":{"$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$eq":"trained"}}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1110,"output_tokens":181}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a16","trial":2,"input":{"filter":{"$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$eq":"trained"}}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1110,"output_tokens":344}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a15","trial":5,"input":{"filter":{"$or":[{"species":"cat","status":"available"},{"species":"dog","weightKg":{"$lt":20}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1106,"output_tokens":232}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a16","trial":3,"input":{"filter":{"$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$eq":"trained"}}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1110,"output_tokens":340}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a16","trial":5,"input":{"filter":{"$or":[{"shelter.city":"Patras"},{"tags":{"$some":{"$eq":"trained"}}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1110,"output_tokens":382}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a16","trial":4,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 6849, Requested 1504. Please try again in 2.6475s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a17","trial":1,"input":{"filter":{"$and":[{"status":{"$ne":"sold"}},{"status":{"$ne":"pending"}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1098,"output_tokens":198}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a17","trial":2,"input":{"filter":{"$nor":[{"status":"sold"},{"status":"pending"}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1098,"output_tokens":273}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a17","trial":4,"input":{"filter":{"$and":[{"status":{"$ne":"sold"}},{"status":{"$ne":"pending"}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1098,"output_tokens":208}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a17","trial":3,"input":{"filter":{"$and":[{"status":{"$ne":"sold"}},{"status":{"$ne":"pending"}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1098,"output_tokens":186}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a18","trial":1,"input":{"filter":{"neutered":true}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1097,"output_tokens":102}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a18","trial":2,"input":{"filter":{"neutered":true}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1097,"output_tokens":82}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a18","trial":3,"input":{"filter":{"neutered":true}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1097,"output_tokens":91}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a18","trial":4,"input":{"filter":{"neutered":true}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1097,"output_tokens":85}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a18","trial":5,"input":{"filter":{"neutered":true}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1097,"output_tokens":74}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a19","trial":1,"text":"I’m sorry, but the pet‑store search tool can’t express a query that compares one field to another (e.g., “price > cost”). It can only filter on fixed values or simple operators, so this request can’t be handled directly.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":230}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a17","trial":5,"input":{"filter":{"$and":[{"status":{"$ne":"sold"}},{"status":{"$ne":"pending"}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1098,"output_tokens":226}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a19","trial":2,"text":"I’m sorry, but the pet‑store search tool can only filter on a field’s value against a constant (e.g., “priceCents > 5000”). It can’t compare one field to another (like “priceCents > costCents”), so this question can’t be expressed with the available tool.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":246}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a19","trial":4,"text":"I’m not able to express a query that compares one field to another (e.g., “price > cost”) with the available pet‑store search tool, so I can’t retrieve that information directly.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":349}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a19","trial":5,"text":"I’m sorry, but the available pet‑store search tool can only filter on fixed values (e.g., “priceCents > 5000”) and doesn’t support comparing one field to another (such as “priceCents > costCents”). Therefore, I can’t express “pets listed for more than they cost us” with this tool.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":319}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a20","trial":1,"text":"I’m sorry, but the available pet‑store search tool doesn’t support a “starts with” or similar prefix filter for the `status` field, so I can’t express that query with it.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":204}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a20","trial":2,"text":"I’m sorry, but the pet‑store search tool doesn’t support “starts with” or similar partial‑match queries, so I can’t express “status beginning with the letter a” using it.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":184}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a19","trial":3,"text":"I’m sorry, but the pet‑store search tool can only filter on fixed values (e.g., “priceCents > 5000”), not on comparisons between two fields such as “priceCents > costCents.” Therefore I can’t express “pets listed for more than they cost us” with the available tool.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":286}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a20","trial":4,"text":"I’m sorry, but the pet‑store search tool doesn’t support a “starts‑with” (or similar pattern‑matching) query on the `status` field, so I can’t express that request with it.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":231}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"a20","trial":3,"text":"I’m sorry, but the available search tool can only filter by exact matches or the listed comparison operators—it doesn’t support pattern matching such as “starts with ‘a’”. So I can’t express that query with the tool.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":117}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b01","trial":1,"text":"I’m sorry, but I can’t express that request with the available pet‑search tool because the schema for vaccination details (such as the type of shot and its booster‑due date) isn’t defined for the query language.","stopReason":"stop","usage":{"input_tokens":1131,"output_tokens":241}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b01","trial":2,"text":"I’m sorry, but I can’t express that request with the pet‑store search tool as it’s currently described. The tool’s schema doesn’t provide enough detail about the structure of the vaccinations field to query for a specific rabies shot and its booster‑due date.","stopReason":"stop","usage":{"input_tokens":1131,"output_tokens":447}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b02","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/$and/0/vaccinations/$some`: minimum 1 properties allowed, but found 0 properties, `/filter/$and/1/vaccinations/$some`: additionalProperties 'boosterDueDate' not allowed]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b02","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/$and/0/vaccinations/$some`: minimum 1 properties allowed, but found 0 properties]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"$and\\\": [\\n {\\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b03","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/vaccinations/$some`: minimum 1 properties allowed, but found 0 properties]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"vaccinations\\\": {\\n \\\""} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b04","trial":1,"text":"I’m sorry, but the available search tool can’t express a query that checks the order of items in a pet’s vaccination list, so I can’t answer that question using it.","stopReason":"stop","usage":{"input_tokens":1105,"output_tokens":282}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b03","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7599, Requested 1580. Please try again in 8.8425s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b04","trial":2,"text":"I’m sorry, but the available search tool can’t express a query that checks whether “parvo” is specifically the first element of a pet’s vaccination list.","stopReason":"stop","usage":{"input_tokens":1105,"output_tokens":231}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b05","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/vaccinations/$some`: minimum 1 properties allowed, but found 0 properties]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"vaccinations\\\": {\\n \\\""} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b06","trial":1,"input":{"filter":{"$nor":[{"tags":{"$some":{"$in":["indoor"]}}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1099,"output_tokens":313}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b06","trial":2,"input":{"filter":{"$not":{"tags":{"$some":{"$eq":"indoor"}}}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1099,"output_tokens":242}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b07","trial":1,"input":{"filter":{"tags":{"$some":{"$ne":"indoor"}}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":202}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b07","trial":2,"input":{"filter":{"tags":{"$some":{"$ne":"indoor"}}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":235}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b05","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 6588, Requested 1575. Please try again in 1.2225s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b08","trial":1,"text":"I’m sorry, but the available search tool can’t express a condition that checks the first element of a pet’s `tags` list. It can only test whether a value appears somewhere in the array, not its position.","stopReason":"stop","usage":{"input_tokens":1102,"output_tokens":189}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b09","trial":1,"text":"I’m sorry, but I can’t formulate a query for that request with the information available about the data schema.","stopReason":"stop","usage":{"input_tokens":1103,"output_tokens":295}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b08","trial":2,"text":"I’m sorry, but the available search tool doesn’t provide a way to query for the first element of a list (e.g., “indoor” being the first label in the `tags` array). Therefore, I can’t express that request with the tool.","stopReason":"stop","usage":{"input_tokens":1102,"output_tokens":203}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b10","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/vaccinations/$some`: minimum 1 properties allowed, but found 0 properties]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"vaccinations\\\": {\\n \\\""} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b09","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/vaccinations/$some`: minimum 1 properties allowed, but found 0 properties]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"vaccinations\\\": {\\n \\\""} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b11","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/$or/0/vaccinations/$eq`: expected array, but got null, `/filter/$or/1/vaccinations/$not/$some`: additionalProperties '$ne' not allowed]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"argum"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b10","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/vaccinations/$some`: minimum 1 properties allowed, but found 0 properties]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"vaccinations\\\": {\\n \\\""} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b11","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/$not/vaccinations/$some`: additionalProperties '$ne', '$unknownAs' not allowed]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"$not\\\": {\\n \\\"vac"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b12","trial":2,"input":{"filter":{"$not":{"tags":{"$some":{"$eq":"indoor"}}}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":305}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b12","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 6989, Requested 1450. Please try again in 3.2925s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b13","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/$or/0/tags/$eq`: expected array, but got null, `/filter/$or/1/$and/1/tags`: additionalProperties '$nor' not allowed]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filt"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"b13","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/$or/0/tags/$eq`: expected array, but got null]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"$or\\\": [\\n {\\n \\\"tags\\\": {\\n \\\"$eq"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c01","trial":2,"input":{"filter":{"microchip":{"$eq":""}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":284}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c02","trial":1,"text":"I’m sorry, but the available search tool can’t express a query that selects records solely because a field is missing. It can filter on values that are present, but it has no way to test for the absence of the `microchip` field.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":509}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c01","trial":1,"input":{"filter":{"microchip":""},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":146}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c03","trial":1,"input":{"filter":{"$not":{"notes":{"$eq":null,"$unknownAs":true}}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":984}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c03","trial":2,"input":{"filter":{"notes":{"$ne":null}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":288}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c04","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/shelter.capacity`: expected integer, but got object, `/filter/shelter.capacity/$eq`: expected integer, but got null]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filt"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c02","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 6941, Requested 1753. Please try again in 5.205s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c05","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":170}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c05","trial":2,"input":{"filter":{"microchip":{"$ne":"CHIP-001"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1117,"output_tokens":182}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c07","trial":1,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1123,"output_tokens":450}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c07","trial":2,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1123,"output_tokens":324}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c08","trial":1,"input":{"filter":{"status":{"$ne":"sold"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1096,"output_tokens":186}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c08","trial":2,"input":{"filter":{"status":{"$ne":"sold"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1096,"output_tokens":160}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c11","trial":1,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":239}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c04","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 6916, Requested 1580. Please try again in 3.719999999s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c11","trial":2,"input":{"filter":{"$or":[{"weightKg":{"$lt":1}},{"weightKg":{"$gt":5}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":166}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c12","trial":2,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1126,"output_tokens":509}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c13","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/$or/1/born`: expected string, but got object, `/filter/$or/1/born/$eq`: expected string, but got null]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c13","trial":2,"input":{"filter":{"$not":{"born":{"$gte":"2020-01-01","$unknownAs":true}}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":300}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d01","trial":1,"text":"I’m sorry, but the available pet‑search tool doesn’t provide a way to filter records based on the data type of a field (such as “notes is a number”). Therefore I can’t express that query with the tool.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":225}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"c12","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7007, Requested 1778. Please try again in 5.8875s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d01","trial":2,"text":"I’m sorry, but the available search tool doesn’t provide a way to filter records based on the type of a field (such as “notes is a number”). Therefore I can’t express that query with the tool.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":174}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d02","trial":1,"text":"I’m sorry, but the available pet‑search tool doesn’t provide a way to filter records based on whether a field’s value is a whole number (or to treat 3.0 as whole). Without such an operator, this query can’t be expressed using the tool.","stopReason":"stop","usage":{"input_tokens":1110,"output_tokens":245}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d03","trial":1,"input":{"filter":{"notes":{"$ne":null}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1101,"output_tokens":436}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d03","trial":2,"text":"I’m sorry, but the available search tool doesn’t provide a way to query for the type of a field (e.g., to ask whether `notes` is a list). I can’t express that request with the given filter language.","stopReason":"stop","usage":{"input_tokens":1101,"output_tokens":268}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d04","trial":1,"input":{"filter":{"notes":{"$eq":""}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1106,"output_tokens":107}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d04","trial":2,"input":{"filter":{"notes":{"$eq":""}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1106,"output_tokens":135}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d02","trial":2,"text":"I’m sorry, but the available search tool can’t express a query that checks whether a `notes` value is a whole number (treating 3.0 as whole). It doesn’t provide the necessary numeric‑type or modulo operators to formulate that condition.","stopReason":"stop","usage":{"input_tokens":1110,"output_tokens":273}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d05","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/notes`: expected object, but got number]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"notes\\\": 3\\n }\\n}}\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d06","trial":1,"input":{"filter":{"notes":{"$gt":2}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":115}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d06","trial":2,"input":{"filter":{"notes":{"$gt":2}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1104,"output_tokens":192}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d07","trial":1,"input":{"filter":{"notes":{"$gt":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1105,"output_tokens":183}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d07","trial":2,"input":{"filter":{"notes":{"$gt":"a"}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1105,"output_tokens":220}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d08","trial":1,"input":{"filter":{"notes":{"$ne":3}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1105,"output_tokens":126}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d08","trial":2,"input":{"filter":{"notes":{"$ne":3}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1105,"output_tokens":217}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d09","trial":1,"input":{"filter":{"notes":{"$lte":2}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1106,"output_tokens":305}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d09","trial":2,"text":"I’m sorry, but the request can’t be expressed with the available pet‑store search tool.","stopReason":"stop","usage":{"input_tokens":1106,"output_tokens":317}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"d05","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 6624, Requested 1785. Please try again in 3.0675s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"e01","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter`: minimum 1 properties allowed, but found 0 properties]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"$$rate?\\\": {\\n \\\"$gt\\\": 5\\n }\\n },\\n "} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"e02","trial":1,"input":{"filter":{"size\\.raw":"XS"},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":97}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"e02","trial":2,"input":{"filter":{"size\\.raw":"XS"}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1121,"output_tokens":92}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"e03","trial":1,"text":"I’m sorry, but the pet‑store search tool doesn’t support pattern matching (e.g., “starts with … and ends with …”) for the `name` field, so I can’t express that query with the available tool.","stopReason":"stop","usage":{"input_tokens":1104,"output_tokens":139}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"e03","trial":2,"text":"I’m sorry, but the available pet‑store search tool can only test for exact values or simple comparisons—it doesn’t support pattern matching such as “starts with C and ends with o.” Therefore I can’t express that query with the tool.","stopReason":"stop","usage":{"input_tokens":1104,"output_tokens":134}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"e04","trial":1,"text":"I’m sorry, but the pet‑store search tool doesn’t provide a way to filter names by a prefix or by “starts‑with” criteria, so this request can’t be expressed with the available query language.","stopReason":"stop","usage":{"input_tokens":1106,"output_tokens":155}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"e04","trial":2,"text":"I’m sorry, but the available pet‑search tool can’t express a “name starts with A, B, or C” condition, so I can’t retrieve that list for you.","stopReason":"stop","usage":{"input_tokens":1106,"output_tokens":166}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"e05","trial":1,"text":"I’m sorry, but the available pet‑search tool can only filter on exact values or simple comparisons—it doesn’t support pattern matching (e.g., “starts with” or case‑insensitive string queries). So this request can’t be expressed with the tool as described.","stopReason":"stop","usage":{"input_tokens":1107,"output_tokens":234}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"e05","trial":2,"text":"I’m sorry, but the available pet‑search tool doesn’t support a “starts with” (prefix) query on the name field, so I can’t express that request with it.","stopReason":"stop","usage":{"input_tokens":1107,"output_tokens":241}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"e01","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter`: minimum 1 properties allowed, but found 0 properties]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filter\\\": {\\n \\\"$rate\\\": {\\n \\\"$gt\\\": 5\\n }\\n }\\n}}\"}"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"e06","trial":1,"text":"I’m sorry, but the available pet‑store search tool doesn’t provide a way to filter names by a pattern such as “ending in s.” It can only test for exact matches and simple logical combinations, not string‑pattern queries.","stopReason":"stop","usage":{"input_tokens":1099,"output_tokens":196}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"f01","trial":1,"input":{"filter":{"notes":{"$eq":"anything"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":112}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"f01","trial":2,"input":{"filter":{"notes":{"$eq":"anything"}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1103,"output_tokens":95}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"f02","trial":1,"text":"I’m sorry, but the available search tool only lets me query specific fields with structured filters—it can’t perform a free‑text search across all pet records. If you’d like, I can look for “ada” in particular fields (like name, notes, tags, etc.) using those filters. Let me know how you’d like to proceed.","stopReason":"stop","usage":{"input_tokens":1109,"output_tokens":136}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"e06","trial":2,"text":"I’m sorry, but the pet‑store search tool doesn’t provide a way to filter names by a pattern such as “ending with ‘s’.” This query can’t be expressed with the available search capabilities.","stopReason":"stop","usage":{"input_tokens":1099,"output_tokens":210}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"f02","trial":2,"text":"I’m not able to run a free‑text search across all pet records with the available tool, since it only supports structured field‑based filters.","stopReason":"stop","usage":{"input_tokens":1109,"output_tokens":166}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"m01","trial":2,"input":{"filter":{"tags":{"$some":{"$eq":"indoor"}}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1097,"output_tokens":145}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"m02","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1124,"output_tokens":268}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"m02","trial":2,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1124,"output_tokens":146}} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"m03","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/status`: value must be \\\"available\\\", `/filter/status`: value must be \\\"pending\\\", `/filter/status`: value must be \\\"sold\\\", `/filter/status`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generati"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"m03","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/status`: value must be \\\"available\\\", `/filter/status`: value must be \\\"pending\\\", `/filter/status`: value must be \\\"sold\\\", `/filter/status`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generati"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a01","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24755, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a01","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24755, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a02","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25050, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a02","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24844, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a03","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24759, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a03","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24759, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a04","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25071, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a04","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24865, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a05","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24760, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a05","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24760, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a06","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25066, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a06","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24860, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a07","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24761, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a07","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24761, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a08","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25055, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a08","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24849, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a09","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24759, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a09","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24759, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a10","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25054, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a10","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24848, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a11","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24754, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a11","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24754, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a12","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25050, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a12","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24844, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a13","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24756, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a13","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24756, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a14","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25056, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a14","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24850, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a15","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24762, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-raw","caseId":"m01","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 6621, Requested 1483. Please try again in 779.999999ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a16","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25062, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a15","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24852, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a16","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24766, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a17","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24754, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a17","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25050, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a18","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24843, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a18","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24753, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a19","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24757, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a19","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25053, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a20","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24847, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"a20","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24757, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b01","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24787, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b01","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25083, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b02","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24882, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b02","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24792, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b03","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24760, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b03","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24850, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b04","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25057, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b04","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24761, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b05","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24755, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b05","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24845, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b06","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25051, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b06","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25051, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b07","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24760, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b07","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24850, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b08","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25054, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b09","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24759, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b08","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24758, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b10","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24849, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b09","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25055, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b10","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24759, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b11","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24770, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b11","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24860, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b12","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25065, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b12","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24769, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b13","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24760, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"b13","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24850, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c01","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25055, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c01","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25055, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c02","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24757, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c02","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24847, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c03","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25056, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c03","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24760, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c04","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24760, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c04","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24850, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c05","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25069, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c05","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24773, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c07","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24779, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c07","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24869, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c08","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25048, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c08","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24752, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c11","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24849, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c11","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24759, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c12","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25078, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c12","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24782, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c13","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24867, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"c13","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24777, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d01","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25053, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d01","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24757, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d02","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24856, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d02","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24766, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d03","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25053, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d03","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24757, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d04","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24852, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d04","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24762, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d05","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25055, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d05","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24759, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d06","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24760, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d06","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24850, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d07","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24851, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d07","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24761, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d08","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24851, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d08","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24761, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d09","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24762, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"d09","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24852, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"e01","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24766, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"e01","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24766, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"e02","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24867, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"e02","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24777, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"e03","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24760, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"e03","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24850, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"e04","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24762, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"e04","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24762, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"e05","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24853, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"e05","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24763, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"e06","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24755, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"e06","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24845, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"f01","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24759, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"f01","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24759, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"f02","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24855, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"f02","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24765, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"m01","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24753, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"m01","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25049, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"m02","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24870, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"m02","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24780, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"m03","trial":1,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 24753, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-generated","caseId":"m03","trial":2,"error":"413: {\"error\":{\"message\":\"Request too large for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 25049, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a01","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/adoption_status`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/adoption_status`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arg"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a01","trial":1,"text":"I’m sorry, but I don’t have enough information about the data fields in the pet store’s database to express “available for adoption right now” as a query using the provided search tool. Without knowing the exact field names or values that indicate adoption availability, I can’t construct a valid filter.","stopReason":"stop","usage":{"input_tokens":807,"output_tokens":375}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a02","trial":1,"input":{"filter":{"$and":[{"species":{"$eq":"cat"}},{"available":{"$eq":true}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":806,"output_tokens":201},"repair":{"input":{"filter":{"$and":[{"type":{"$eq":"cat"}},{"status":{"$eq":"available"}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1037,"output_tokens":102},"problemShown":{"type":"unknown-field","detail":"\"available\" is not a queryable path on this resource.","pointer":"/filter/$and/1/available"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a03","trial":1,"input":{"filter":{"$or":[{"status":{"$eq":"available"}},{"status":{"$eq":"application pending"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":811,"output_tokens":240}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a02","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/type`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/type`: expected object, but got string, `/filter/available`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/available`: expected object, but got string"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a04","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/weight`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/weight`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"f"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a03","trial":2,"input":{"filter":{"$or":[{"status":{"$eq":"available"}},{"status":{"$eq":"pending"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":811,"output_tokens":150}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a04","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/weight`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/weight`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"f"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a05","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/birthDate`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/birthDate`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a06","trial":1,"text":"I’m sorry, but the available search tool doesn’t support a filter that checks whether a string field starts with a particular lower‑case letter. It can only handle exact matches and numeric comparisons, so I can’t express “name begins with a lower‑case b” using the provided query language.","stopReason":"stop","usage":{"input_tokens":822,"output_tokens":247}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a05","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/birthDate`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/birthDate`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a06","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/name`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/name`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filte"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a07","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/name`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/name`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filte"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a08","trial":1,"text":"I’m sorry, but the available search tool doesn’t support a query that looks for a substring like “50%” within a pet’s name, so I can’t express that request using it.","stopReason":"stop","usage":{"input_tokens":811,"output_tokens":273}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a07","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7393, Requested 1289. Please try again in 5.115s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a09","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/labels`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/labels`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"f"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a09","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/labels`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/labels`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"f"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a08","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7332, Requested 1197. Please try again in 3.9675s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a10","trial":2,"input":{"filter":{"$and":[{"labels":{"$in":["trained"]}},{"labels":{"$in":["outdoor"]}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":810,"output_tokens":465},"repair":{"input":{"filter":{"$and":[{"trained":true},{"outdoor":true}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1294,"output_tokens":169},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$and/0/labels"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a11","trial":1,"text":"I’m sorry, but the available search tool doesn’t provide a way to filter pets based on the number of items in an array (such as requiring three or more labels), so I can’t express that query with the tool.","stopReason":"stop","usage":{"input_tokens":806,"output_tokens":203}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a10","trial":1,"input":{"filter":{"$and":[{"labels":{"$in":["trained"]}},{"labels":{"$in":["outdoor"]}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":810,"output_tokens":342},"repair":{"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7444, Requested 1576. Please try again in 7.65s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n","problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$and/0/labels"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a12","trial":1,"input":{"filter":{"$or":[{"labels":{"$eq":[]}},{"labels":{"$eq":null}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":806,"output_tokens":303},"repair":{"text":"I’m sorry, but I can’t answer that question with the available pet‑store search tool, because the data source doesn’t expose a “labels” field that I can query.","stopReason":"stop","usage":{"input_tokens":1128,"output_tokens":138},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$or/0/labels"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a12","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/labels`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/labels`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"f"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a11","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7276, Requested 1282. Please try again in 4.185s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a13","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/city`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/city`: expected object, but got string, `/filter/city`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"nam"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a14","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/kennelCount`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/kennelCount`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\""} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a14","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/shelterKennels`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/shelterKennels`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"argum"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a13","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7481, Requested 1205. Please try again in 5.145s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a15","trial":2,"input":{"filter":{"$or":[{"$and":[{"type":{"$eq":"cat"}},{"status":{"$eq":"available"}}]},{"$and":[{"type":{"$eq":"dog"}},{"weightKg":{"$lt":20}}]}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":814,"output_tokens":280},"repair":{"input":{"filter":{"$or":[{"$and":[{"species":{"$eq":"cat"}},{"status":{"$eq":"available"}}]},{"$and":[{"species":{"$eq":"dog"}},{"weightKg":{"$lt":20}}]}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1099,"output_tokens":102},"problemShown":{"type":"unknown-field","detail":"\"type\" is not a queryable path on this resource.","pointer":"/filter/$or/0/$and/0/type"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a16","trial":1,"text":"I’m not able to express that request with the available search tool, because I don’t know the exact field names used for a pet’s shelter location or its labels in the dataset. Without that information I can’t construct a valid filter.","stopReason":"stop","usage":{"input_tokens":818,"output_tokens":433}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a15","trial":1,"input":{"filter":{"$or":[{"$and":[{"type":{"$eq":"cat"}},{"status":{"$eq":"available"}}]},{"$and":[{"type":{"$eq":"dog"}},{"weight":{"$lt":20}}]}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":814,"output_tokens":264},"repair":{"input":{"filter":{"$or":[{"$and":[{"species":{"$eq":"cat"}},{"status":{"$eq":"available"}}]},{"$and":[{"species":{"$eq":"dog"}},{"weight":{"$lt":20}}]}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1083,"output_tokens":169},"problemShown":{"type":"unknown-field","detail":"\"type\" is not a queryable path on this resource.","pointer":"/filter/$or/0/$and/0/type"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a17","trial":1,"input":{"filter":{"$nor":[{"status":"sold"},{"status":"pending"}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":806,"output_tokens":328}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a17","trial":2,"input":{"filter":{"$nor":[{"status":{"$eq":"sold"}},{"status":{"$eq":"pending"}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":806,"output_tokens":226}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a16","trial":2,"input":{"filter":{"$or":[{"shelter_city":{"$eq":"Patras"}},{"labels":{"$eq":"trained"}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":818,"output_tokens":498},"repair":{"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7123, Requested 1848. Please try again in 7.2825s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n","problemShown":{"type":"unknown-field","detail":"\"shelter_city\" is not a queryable path on this resource.","pointer":"/filter/$or/0/shelter_city"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a18","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/neutered`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/neutered`: expected object, but got string, `/filter/neutered`: expected object, but got boolean]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_genera"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a18","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/neutered`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/neutered`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n "} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a19","trial":2,"text":"I’m sorry, but the query “list the pets where the listed price is greater than the cost price” can’t be expressed with the search tool as it’s described. The filter language only lets you compare a field to a constant value, not compare one field to another field.","stopReason":"stop","usage":{"input_tokens":809,"output_tokens":300}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a20","trial":1,"text":"I’m sorry, but the available search tool doesn’t provide a way to query for a string that starts with a particular letter, so I can’t express that request with it.","stopReason":"stop","usage":{"input_tokens":809,"output_tokens":274}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a20","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/status`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/status`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"f"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"a19","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 6704, Requested 1311. Please try again in 112.499999ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b01","trial":1,"text":"I’m sorry, but I can’t express that query with the pet‑search tool because the structure of the pet records (e.g., how vaccinations and booster dates are stored) isn’t defined, so I don’t know how to formulate the appropriate filter.","stopReason":"stop","usage":{"input_tokens":839,"output_tokens":219}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b02","trial":1,"input":{"filter":{"$and":[{"shots.type":"rabies"},{"shots.boosterDue":{"$lt":"2025-01-01"}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":844,"output_tokens":362},"repair":{"input":{"filter":{"$and":[{"shots":{"$elemMatch":{"type":"rabies"}}},{"shots":{"$elemMatch":{"boosterDue":{"$lt":"2025-01-01"}}}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1234,"output_tokens":206},"problemShown":{"type":"unknown-field","detail":"\"shots.type\" is not a queryable path on this resource.","pointer":"/filter/$and/0/shots.type"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b02","trial":2,"text":"I’m sorry, but the request can’t be expressed with the available search tool.","stopReason":"stop","usage":{"input_tokens":844,"output_tokens":237}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b03","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/vaccinationDate`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/vaccinationDate`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arg"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b01","trial":2,"input":{"filter":{"$and":[{"shots.type":"rabies"},{"shots.boosterDueDate":{"$lt":"2025-01-01"}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":839,"output_tokens":382},"repair":{"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7447, Requested 1668. Please try again in 8.3625s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n","problemShown":{"type":"unknown-field","detail":"\"shots.type\" is not a queryable path on this resource.","pointer":"/filter/$and/0/shots.type"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b03","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/vaccinationDate`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/vaccinationDate`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arg"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b04","trial":2,"text":"I’m sorry, but the available search tool doesn’t let me express a query that checks whether “parvo” is the very first item in a pet’s vaccination‑history list.","stopReason":"stop","usage":{"input_tokens":813,"output_tokens":256}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b05","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/vaccinations.parvo`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/vaccinations.parvo`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\","} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b04","trial":1,"text":"I’m sorry, but the search tool’s query language doesn’t provide a way to specify “the very first entry in an array” (e.g., the first element of a vaccination‑history list). Therefore I can’t express that request with the tool.","stopReason":"stop","usage":{"input_tokens":813,"output_tokens":226}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b06","trial":1,"input":{"filter":{"$not":{"indoor":{"$eq":true}}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":807,"output_tokens":211},"repair":{"input":{"filter":{"$nor":[{"labels":{"$eq":"indoor"}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1051,"output_tokens":122},"problemShown":{"type":"unknown-field","detail":"\"indoor\" is not a queryable path on this resource.","pointer":"/filter/$not/indoor"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b06","trial":2,"input":{"filter":{"$nor":[{"indoor":true}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":807,"output_tokens":207},"repair":{"text":"I’m sorry, but I can’t express that query with the available search tool because the “indoor” label isn’t a searchable field in the pet data.","stopReason":"stop","usage":{"input_tokens":1048,"output_tokens":133},"problemShown":{"type":"unknown-field","detail":"\"indoor\" is not a queryable path on this resource.","pointer":"/filter/$nor/0/indoor"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b07","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/labels`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/labels`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"f"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b05","trial":2,"text":"I’m sorry—I don’t have enough information about the structure of the pet records (e.g., how vaccination data is stored) to formulate a valid filter for the search tool. Without knowing the exact field names and types, I can’t express “pets that have had a parvo vaccination” using the available query language.","stopReason":"stop","usage":{"input_tokens":807,"output_tokens":287}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b08","trial":1,"text":"I’m sorry, but the available search tool doesn’t provide a way to filter on the position of an element within an array (e.g., “the first label is ‘indoor’”). Therefore, I can’t express that query with the tool as described.","stopReason":"stop","usage":{"input_tokens":810,"output_tokens":200}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b08","trial":2,"text":"I’m sorry, but the search tool’s filter language doesn’t provide a way to specify “the first element of a list” in a query, so I can’t express that request with the available tool.","stopReason":"stop","usage":{"input_tokens":810,"output_tokens":238}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b09","trial":1,"text":"I’m not able to formulate that request using the available pet‑search tool, because the necessary field names and array‑query operators (e.g., checking for a vaccination entry that has a booster date) aren’t defined in the tool’s schema.","stopReason":"stop","usage":{"input_tokens":811,"output_tokens":334}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b09","trial":2,"text":"I’m sorry, but the available search tool doesn’t provide a way to express “has at least one vaccination with a booster date recorded,” so I can’t formulate a query for that.","stopReason":"stop","usage":{"input_tokens":811,"output_tokens":264}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b07","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7831, Requested 1146. Please try again in 7.3275s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b10","trial":1,"text":"I’m sorry, but I can’t express that request using the available pet‑search tool.","stopReason":"stop","usage":{"input_tokens":811,"output_tokens":252}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b10","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/vaccinations.boosterDate`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/vaccinations.boosterDate`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"se"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b11","trial":1,"input":{"filter":{"$or":[{"vaccinations":{"$eq":[]}},{"vaccinations":{"$eq":["rabies"]}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":822,"output_tokens":345}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b11","trial":2,"input":{"filter":{"$or":[{"vaccinations":{"$eq":[]}},{"vaccinations":{"$eq":["rabies"]}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":822,"output_tokens":493}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b12","trial":2,"text":"I’m sorry, but the available search tool doesn’t provide a way to express “the pet’s list of labels does not contain the value ‘indoor’ (including the case where the list is empty).” Therefore I can’t formulate a query for that request.","stopReason":"stop","usage":{"input_tokens":821,"output_tokens":361}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b13","trial":1,"input":{"filter":{"$or":[{"labels":{"$exists":false}},{"labels":{"$size":0}},{"$not":{"labels":{"$elemMatch":{"$ne":"indoor"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":812,"output_tokens":843},"repair":{"text":"I’m sorry, but the query “pets that have only indoor labels, or have no labels at all” can’t be expressed with the search tool’s available filter operators.","stopReason":"stop","usage":{"input_tokens":1682,"output_tokens":124},"problemShown":{"type":"unsupported-operator","detail":"$elemMatch is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$or/2/$not/labels/$elemMatch"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b13","trial":2,"text":"I’m sorry, but the query “pets that have nothing but indoor labels, or no labels at all” can’t be expressed with the search tool as it’s currently defined. The tool’s filter language doesn’t provide the necessary array‑level operators to require that a label list contain only “indoor” (and nothing else) while also handling the case of an empty label list.","stopReason":"stop","usage":{"input_tokens":812,"output_tokens":584}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c01","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/microchip`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/microchip`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"b12","trial":1,"input":{"filter":{"$nor":[{"labels":"indoor"}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":821,"output_tokens":281},"repair":{"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 7472, Requested 1610. Please try again in 8.115s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n","problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$nor/0/labels"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c02","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/microchip`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/microchip`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c02","trial":2,"text":"I’m sorry, but the available pet‑search tool doesn’t provide a way to query for records where a field is completely absent or null, so I can’t retrieve “pets with no microchip field recorded at all.”","stopReason":"stop","usage":{"input_tokens":809,"output_tokens":776}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c03","trial":1,"input":{"filter":{"$not":{"notes":{"$eq":null}}},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":812,"output_tokens":568}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c01","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/microchip`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/microchip`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c03","trial":2,"input":{"filter":{"$nor":[{"notes":{"$eq":null}}]}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1068,"output_tokens":324}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c04","trial":1,"text":"I’m sorry, but I can’t express that request using the available pet‑search tool.","stopReason":"stop","usage":{"input_tokens":812,"output_tokens":284}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c04","trial":2,"text":"I’m sorry, but I can’t express that query using the available pet‑search tool. It only lets me filter directly on fields of the pet records themselves, and I don’t have a way to reference or test a related shelter’s “kennel capacity” field within those filters.","stopReason":"stop","usage":{"input_tokens":812,"output_tokens":230}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c05","trial":2,"input":{"filter":{"$and":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$ne":null}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":825,"output_tokens":316}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c07","trial":1,"input":{"filter":{"$or":[{"chip":{"$ne":"CHIP-001"}},{"chip":{"$eq":null}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":831,"output_tokens":640},"repair":{"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter`: minimum 1 properties allowed, but found 0 properties]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\\"filter\\\":{},\\\"limit\\\":5}}\"}}\n","problemShown":{"type":"unknown-field","detail":"\"chip\" is not a queryable path on this resource.","pointer":"/filter/$or/0/chip"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c05","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 6826, Requested 1287. Please try again in 847.5ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c08","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/sold`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/sold`: expected object, but got string, `/filter/sold`: expected object, but got boolean]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"na"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c08","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/sold`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/sold`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"filte"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c11","trial":1,"input":{"filter":{"$or":[{"weight":{"$lt":1}},{"weight":{"$gt":5}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":811,"output_tokens":165},"repair":{"text":"I’m unable to formulate a query for that request because the pet records don’t have a searchable field named “weight,” so I can’t ask the tool to find pets whose weight falls outside the 1‑to‑5 kg range.","stopReason":"stop","usage":{"input_tokens":1003,"output_tokens":166},"problemShown":{"type":"unknown-field","detail":"\"weight\" is not a queryable path on this resource.","pointer":"/filter/$or/0/weight"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c07","trial":2,"input":{"filter":{"$or":[{"chip":{"$ne":"CHIP-001"}},{"chip":{"$eq":null}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":831,"output_tokens":272},"repair":{"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Used 6489, Requested 1583. Please try again in 539.999999ms. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n","problemShown":{"type":"unknown-field","detail":"\"chip\" is not a queryable path on this resource.","pointer":"/filter/$or/0/chip"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c12","trial":1,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$eq":null}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":834,"output_tokens":852}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c11","trial":2,"input":{"filter":{"$or":[{"weight":{"$lt":1}},{"weight":{"$gt":5}}]},"limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":811,"output_tokens":303},"repair":{"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198902, Requested 1603. Please try again in 3m38.16s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n","problemShown":{"type":"unknown-field","detail":"\"weight\" is not a queryable path on this resource.","pointer":"/filter/$or/0/weight"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c12","trial":2,"input":{"filter":{"$nor":[{"chip":{"$eq":"CHIP-001"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":834,"output_tokens":359},"repair":{"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198872, Requested 2063. Please try again in 6m43.92s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n","problemShown":{"type":"unknown-field","detail":"\"chip\" is not a queryable path on this resource.","pointer":"/filter/$nor/0/chip"}}} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c13","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198831, Requested 1639. Please try again in 3m23.04s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"c13","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198799, Requested 1429. Please try again in 1m38.496s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d01","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198758, Requested 1428. Please try again in 1m20.352s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d01","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198727, Requested 1428. Please try again in 1m6.959999999s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d02","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198687, Requested 1437. Please try again in 53.568s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d02","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198656, Requested 1437. Please try again in 40.176s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d03","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198615, Requested 1428. Please try again in 18.576s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d03","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198585, Requested 1428. Please try again in 5.616s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d04","trial":2,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/notes`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/notes`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"fil"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d04","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199978, Requested 1953. Please try again in 13m54.192s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d05","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199915, Requested 1621. Please try again in 11m3.552s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d05","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199909, Requested 1654. Please try again in 11m15.216s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d06","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199844, Requested 1622. Please try again in 10m33.312s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d06","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199838, Requested 1622. Please try again in 10m30.72s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d07","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199772, Requested 1656. Please try again in 10m16.896s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d07","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199768, Requested 1623. Please try again in 10m0.912s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d08","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199701, Requested 1623. Please try again in 9m31.968s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d08","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199697, Requested 1623. Please try again in 9m30.24s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d09","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199631, Requested 1077. Please try again in 5m5.856s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"d09","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199626, Requested 1077. Please try again in 5m3.696s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"e01","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199560, Requested 1081. Please try again in 4m36.912s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"e01","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199555, Requested 1081. Please try again in 4m34.752s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"e02","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199489, Requested 1050. Please try again in 3m52.848s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"e02","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199485, Requested 1050. Please try again in 3m51.119999999s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"e03","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199418, Requested 1016. Please try again in 3m7.488s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"e03","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199414, Requested 1050. Please try again in 3m20.448s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"e04","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199346, Requested 1018. Please try again in 2m37.248s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"e04","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199343, Requested 1052. Please try again in 2m50.64s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"e05","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199275, Requested 1053. Please try again in 2m21.696s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"e05","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199272, Requested 1053. Please try again in 2m20.4s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"e06","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199204, Requested 1080. Please try again in 2m2.688s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"e06","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199201, Requested 1018. Please try again in 1m34.608s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"f01","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199133, Requested 1084. Please try again in 1m33.744s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"f01","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199131, Requested 1046. Please try again in 1m16.464s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"f02","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199062, Requested 1128. Please try again in 1m22.08s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"f02","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199060, Requested 1128. Please try again in 1m21.216s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"m01","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198991, Requested 1057. Please try again in 20.736s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"m01","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198989, Requested 1113. Please try again in 44.064s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"m02","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198920, Requested 1115. Please try again in 15.12s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"m02","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 198918, Requested 1148. Please try again in 28.512s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"m03","trial":1,"error":"400: {\"error\":{\"message\":\"Tool call validation failed: tool call validation failed: parameters for tool search_pets did not match schema: errors: [`/filter/status`: value must be one of \\\"$and\\\", \\\"$or\\\", \\\"$nor\\\", \\\"$not\\\", `/filter/status`: expected object, but got string]\",\"type\":\"invalid_request_error\",\"code\":\"tool_use_failed\",\"failed_generation\":\"{\\\"name\\\": \\\"search_pets\\\", \\\"arguments\\\": {\\n \\\"f"} +{"provider":"gpt-oss-120b","arm":"jql-published","caseId":"m03","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199936, Requested 1086. Please try again in 7m21.504s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a01","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199922, Requested 1236. Please try again in 8m20.256s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a01","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199865, Requested 1215. Please try again in 7m46.56s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a02","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199851, Requested 1251. Please try again in 7m56.064s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a02","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199794, Requested 1251. Please try again in 7m31.439999999s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a03","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199780, Requested 1256. Please try again in 7m27.551999999s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a03","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199723, Requested 1228. Please try again in 6m50.832s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a04","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199709, Requested 1250. Please try again in 6m54.287999999s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a04","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199652, Requested 1250. Please try again in 6m29.664s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a05","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199638, Requested 1275. Please try again in 6m34.415999999s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a05","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199581, Requested 1335. Please try again in 6m35.712s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a06","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199567, Requested 1345. Please try again in 6m33.984s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a06","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199510, Requested 1345. Please try again in 6m9.36s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a07","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199496, Requested 1336. Please try again in 5m59.424s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a07","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199439, Requested 1336. Please try again in 5m34.8s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a08","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199426, Requested 1334. Please try again in 5m28.32s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a08","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199368, Requested 1334. Please try again in 5m3.264s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a09","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199355, Requested 1334. Please try again in 4m57.648s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a09","trial":2,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199297, Requested 1275. Please try again in 4m7.104s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gpt-oss-120b","arm":"search-string","caseId":"a10","trial":1,"error":"429: {\"error\":{\"message\":\"Rate limit reached for model `openai/gpt-oss-120b` in organization `org_01je0c27nhepbb5ajwa00re7jn` service tier `on_demand` on tokens per day (TPD): Limit 200000, Used 199284, Requested 1288. Please try again in 4m7.104s. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing\",\"type\":\"tokens\",\"code\":\"rate_limit_exceeded\"}}\n"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a01","trial":2,"input":{"q":"status:available"},"text":"","stopReason":"STOP","usage":{"input_tokens":872,"output_tokens":18}} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a01","trial":3,"input":{"q":"status:available"},"text":"","stopReason":"STOP","usage":{"input_tokens":872,"output_tokens":18}} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a01","trial":4,"input":{"q":"status:available"},"text":"","stopReason":"STOP","usage":{"input_tokens":872,"output_tokens":18}} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a01","trial":1,"input":{"q":"status:available"},"text":"","stopReason":"STOP","usage":{"input_tokens":872,"output_tokens":18}} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a02","trial":1,"input":{"q":"species:cat AND status:available"},"text":"","stopReason":"STOP","usage":{"input_tokens":871,"output_tokens":22}} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a02","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 5, "} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a01","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 5, "} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a02","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 5, "} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a02","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 5, "} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a03","trial":3,"input":{"q":"status:(available OR pending)"},"text":"","stopReason":"STOP","usage":{"input_tokens":876,"output_tokens":21}} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a03","trial":2,"input":{"q":"status:(available OR pending)"},"text":"","stopReason":"STOP","usage":{"input_tokens":876,"output_tokens":21}} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a03","trial":1,"input":{"q":"status:(available OR pending)"},"text":"","stopReason":"STOP","usage":{"input_tokens":876,"output_tokens":21}} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a03","trial":4,"input":{"q":"status:(available OR pending)"},"text":"","stopReason":"STOP","usage":{"input_tokens":876,"output_tokens":21}} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a04","trial":1,"input":{"q":"weightKg:[1 TO 5]"},"text":"","stopReason":"STOP","usage":{"input_tokens":892,"output_tokens":23}} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a02","trial":5,"error":"503: {\n \"error\": {\n \"code\": 503,\n \"message\": \"This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\",\n \"status\": \"UNAVAILABLE\"\n }\n}\n"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a03","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 5, "} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a04","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 5, "} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a04","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 5, "} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a04","trial":3,"error":"503: {\n \"error\": {\n \"code\": 503,\n \"message\": \"This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\",\n \"status\": \"UNAVAILABLE\"\n }\n}\n"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a05","trial":3,"input":{"q":"born:>=2020-01-01"},"text":"","stopReason":"STOP","usage":{"input_tokens":879,"output_tokens":28}} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a04","trial":5,"error":"503: {\n \"error\": {\n \"code\": 503,\n \"message\": \"This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\",\n \"status\": \"UNAVAILABLE\"\n }\n}\n"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a05","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a05","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a05","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a05","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a06","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a06","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a06","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a06","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a06","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a07","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a07","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a07","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a07","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a07","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a08","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a08","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a08","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a08","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a08","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a09","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a09","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a09","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a09","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a09","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a10","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a10","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a10","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a10","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a10","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a11","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a11","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a11","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a11","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a11","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a12","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a12","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a12","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a12","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a12","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a13","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a13","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a13","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a13","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a13","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a14","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a14","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a14","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a14","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a14","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a15","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a15","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a15","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a15","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a15","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a16","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a16","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a16","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a16","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a16","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a17","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a17","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a17","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a17","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a17","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a18","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a18","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a18","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a01","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a01","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a01","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a01","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a01","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a02","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a02","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a02","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a02","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a02","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a03","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a03","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a03","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a03","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a03","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a04","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a04","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a04","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a04","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a04","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a05","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a05","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a05","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a05","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a05","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a06","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a06","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a06","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a06","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a06","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a07","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a07","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a07","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a07","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a07","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a08","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a08","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a08","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a08","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a08","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a09","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a09","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a09","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a09","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a09","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a10","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a10","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a10","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a10","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a10","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a11","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a11","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a11","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a11","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a11","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a12","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a12","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a12","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a12","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a12","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a13","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a13","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a13","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a13","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a13","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a14","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a14","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a14","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a14","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a14","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a15","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a15","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a15","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a15","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a15","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a16","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a16","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a16","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a16","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a16","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a17","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a17","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a17","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a17","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a17","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a18","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a18","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a18","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a18","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a18","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a19","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a19","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a19","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a19","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a19","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a20","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a20","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a20","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a20","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"a20","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b01","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b01","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b01","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b01","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b01","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b02","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b02","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b02","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b02","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b02","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b03","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b03","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b03","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b03","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b03","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b04","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b04","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b04","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b04","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b04","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b05","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b05","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b05","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b05","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b05","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b06","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b06","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b06","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b06","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b06","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b07","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b07","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b07","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b07","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b07","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b08","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b08","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b08","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b08","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b08","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b09","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b09","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b09","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b09","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b09","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b10","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b10","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b10","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b10","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b10","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b11","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b11","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b11","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b11","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b11","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b12","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b12","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b12","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b12","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b12","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b13","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b13","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b13","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b13","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"b13","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c01","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c01","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c01","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c01","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c01","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c02","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c02","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c02","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c02","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c02","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c03","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c03","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c03","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c03","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c03","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c04","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c04","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c04","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c04","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c04","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c05","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c05","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c05","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c05","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c05","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c07","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c07","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c07","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c07","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c07","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c08","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c08","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c08","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c08","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c08","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c11","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c11","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c11","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c11","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c11","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c12","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c12","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c12","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c12","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c12","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c13","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c13","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c13","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c13","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"c13","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d01","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d01","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d01","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d01","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d01","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d02","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d02","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d02","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d02","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d02","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d03","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d03","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d03","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d03","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d03","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d04","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d04","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d04","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d04","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d04","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d05","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d05","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d05","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d05","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d05","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d06","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d06","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d06","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d06","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d06","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d07","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d07","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d07","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d07","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d07","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d08","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d08","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d08","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d08","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d08","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d09","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d09","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d09","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d09","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"d09","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e01","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e01","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e01","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e01","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e01","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e02","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e02","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e02","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e02","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e02","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e03","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e03","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e03","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e03","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e03","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e04","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e04","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e04","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e04","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e04","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e05","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e05","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e05","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e05","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e05","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e06","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e06","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e06","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e06","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"e06","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"f01","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"f01","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"f01","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"f01","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"f01","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"f02","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"f02","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"f02","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"f02","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"f02","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m01","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m01","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m01","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m01","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m01","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m02","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m02","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m02","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m02","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m02","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m03","trial":1,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m03","trial":2,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m03","trial":3,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m03","trial":4,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-raw","caseId":"m03","trial":5,"incompatible":"Gemini function declarations accept no $ref, and the grammar is recursive, so the schema cannot be shipped as the README describes"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a01","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a01","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a01","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a01","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a01","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a02","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a02","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a02","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a02","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a02","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a03","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a03","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a03","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a03","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a03","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a04","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a04","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a04","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a04","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a04","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a05","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a05","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a05","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a05","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a05","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a06","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a06","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a06","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a06","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a06","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a07","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a07","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a07","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a07","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a07","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a08","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a08","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a08","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a08","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a08","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a09","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a09","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a09","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a09","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a09","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a10","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a10","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a10","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a10","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a10","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a11","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a11","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a11","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a11","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a11","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a12","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a12","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a12","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a12","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a12","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a13","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a13","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a13","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a13","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a13","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a14","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a14","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a14","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a14","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a14","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a15","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a15","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a15","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a15","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a15","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a16","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a16","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a16","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a16","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a16","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a17","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a17","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a17","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a17","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a17","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a18","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a18","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a18","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a18","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a18","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a19","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a19","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a19","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a19","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a19","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a20","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a20","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a20","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a20","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"a20","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b01","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b01","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b01","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b01","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b01","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b02","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b02","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b02","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b02","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b02","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b03","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b03","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b03","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b03","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b03","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b04","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b04","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b04","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b04","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b04","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b05","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b05","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b05","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b05","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b05","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b06","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b06","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b06","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b06","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b06","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b07","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b07","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b07","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b07","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b07","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b08","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b08","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b08","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b08","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b08","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b09","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b09","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b09","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b09","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b09","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b10","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b10","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b10","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b10","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b10","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b11","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b11","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b11","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b11","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b11","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b12","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b12","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b12","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b12","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b12","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b13","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b13","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b13","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b13","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"b13","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c01","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c01","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c01","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c01","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c01","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c02","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c02","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c02","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c02","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c02","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c03","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c03","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c03","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c03","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c03","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c04","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c04","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c04","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c04","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c04","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c05","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c05","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c05","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c05","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c05","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c07","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c07","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c07","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c07","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c07","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c08","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c08","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c08","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c08","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c08","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c11","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c11","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c11","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c11","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c11","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c12","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c12","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c12","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c12","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c12","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c13","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c13","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c13","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c13","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"c13","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d01","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d01","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d01","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d01","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d01","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d02","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d02","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d02","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d02","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d02","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d03","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d03","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d03","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d03","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d03","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d04","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d04","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d04","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d04","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d04","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d05","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d05","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d05","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d05","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d05","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d06","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d06","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d06","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d06","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d06","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d07","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d07","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d07","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d07","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d07","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d08","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d08","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d08","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d08","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d08","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d09","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d09","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d09","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d09","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"d09","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e01","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e01","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e01","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e01","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e01","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e02","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e02","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e02","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e02","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e02","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e03","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e03","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e03","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e03","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e03","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e04","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e04","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e04","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e04","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e04","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e05","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e05","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e05","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e05","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e05","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e06","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e06","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e06","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e06","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"e06","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"f01","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"f01","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"f01","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"f01","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"f01","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"f02","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"f02","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"f02","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"f02","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"f02","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m01","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m01","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m01","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m01","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m01","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m02","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m02","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m02","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m02","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m02","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m03","trial":1,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m03","trial":2,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m03","trial":3,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m03","trial":4,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-generated","caseId":"m03","trial":5,"incompatible":"Gemini function declarations have no \"dependentRequired\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a01","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a01","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a01","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a01","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a01","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a02","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a02","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a02","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a02","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a02","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a03","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a03","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a03","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a03","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a03","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a04","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a04","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a04","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a04","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a04","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a05","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a05","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a05","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a05","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a05","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a06","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a06","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a06","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a06","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a06","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a07","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a07","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a07","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a07","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a07","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a08","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a08","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a08","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a08","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a08","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a09","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a09","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a09","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a09","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a09","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a10","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a10","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a10","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a10","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a10","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a11","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a11","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a11","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a11","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a11","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a12","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a12","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a12","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a12","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a12","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a13","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a13","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a13","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a13","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a13","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a14","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a14","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a14","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a14","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a14","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a15","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a15","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a15","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a15","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a15","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a16","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a16","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a16","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a16","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a16","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a17","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a17","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a17","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a17","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a17","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a18","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a18","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a18","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a18","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a18","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a19","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a19","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a19","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a19","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a19","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a20","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a20","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a20","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a20","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"a20","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b01","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b01","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b01","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b01","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b01","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b02","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b02","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b02","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b02","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b02","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b03","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b03","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b03","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b03","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b03","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b04","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b04","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b04","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b04","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b04","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b05","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b05","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b05","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b05","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b05","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b06","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b06","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b06","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b06","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b06","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b07","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b07","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b07","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b07","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b07","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b08","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b08","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b08","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b08","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b08","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b09","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b09","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b09","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b09","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b09","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b10","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b10","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b10","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b10","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b10","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b11","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b11","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b11","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b11","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b11","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b12","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b12","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b12","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b12","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b12","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b13","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b13","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b13","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b13","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"b13","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c01","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c01","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c01","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c01","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c01","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c02","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c02","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c02","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c02","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c02","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c03","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c03","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c03","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c03","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c03","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c04","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c04","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c04","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c04","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c04","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c05","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c05","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c05","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c05","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c05","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c07","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c07","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c07","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c07","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c07","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c08","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c08","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c08","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c08","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c08","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c11","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c11","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c11","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c11","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c11","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c12","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c12","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c12","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c12","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c12","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c13","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c13","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c13","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c13","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"c13","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d01","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d01","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d01","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d01","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d01","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d02","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d02","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d02","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d02","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d02","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d03","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d03","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d03","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d03","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d03","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d04","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d04","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d04","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d04","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d04","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d05","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d05","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d05","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d05","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d05","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d06","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d06","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d06","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d06","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d06","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d07","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d07","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d07","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d07","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d07","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d08","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d08","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d08","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d08","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d08","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d09","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d09","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d09","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d09","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"d09","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e01","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e01","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e01","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e01","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e01","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e02","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e02","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e02","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e02","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e02","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e03","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e03","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e03","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e03","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e03","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e04","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e04","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e04","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e04","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e04","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e05","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e05","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e05","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e05","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e05","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e06","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e06","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e06","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e06","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"e06","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"f01","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"f01","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"f01","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"f01","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"f01","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"f02","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"f02","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"f02","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"f02","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"f02","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m01","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m01","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m01","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m01","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m01","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m02","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m02","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m02","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m02","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m02","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m03","trial":1,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m03","trial":2,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m03","trial":3,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m03","trial":4,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"jql-published","caseId":"m03","trial":5,"incompatible":"Gemini function declarations have no \"patternProperties\" field; the keyword carries semantics the grammar needs, so omitting it would describe a different language"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a18","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a18","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a19","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a19","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a19","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a19","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a19","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a20","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a20","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a20","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a20","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"a20","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b01","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b01","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b01","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b01","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b01","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b02","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b02","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b02","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b02","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b02","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b03","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b03","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b03","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b03","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b03","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b04","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b04","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b04","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b04","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b04","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b05","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b05","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b05","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b05","trial":4,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b05","trial":5,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b06","trial":1,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b06","trial":2,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} +{"provider":"gemini-3.8-flash","arm":"search-string","caseId":"b06","trial":3,"error":"429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20,"} diff --git a/experiments/intent-to-filter/results/pilot.jsonl b/experiments/intent-to-filter/results/pilot.jsonl new file mode 100644 index 0000000..b3dbea0 --- /dev/null +++ b/experiments/intent-to-filter/results/pilot.jsonl @@ -0,0 +1,90 @@ +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"a04","trial":1,"input":{"filter":{"$and":[{"weightKg":{"$gte":1}},{"weightKg":{"$lte":5}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":714,"output_tokens":43}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"a09","trial":1,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":35}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"c01","trial":1,"input":{"filter":{"microchip":""},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":25}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"b06","trial":1,"input":{"filter":{"tags":{"$not":{"$some":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":36}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"a01","trial":1,"input":{"filter":{"status":"open"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":694,"output_tokens":25},"repair":{"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":771,"output_tokens":25},"problemShown":{"type":"invalid-operand","detail":"/status must be one of: \"available\", \"pending\", \"sold\".","pointer":"/filter/status"}}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"e03","trial":1,"input":{"filter":{"name":{"$regex":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":699,"output_tokens":32}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"c05","trial":1,"input":{"filter":{"microchip":{"$unknownAs":false}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":712,"output_tokens":42}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"a12","trial":1,"input":{"filter":{"$or":[{"tags":{"$eq":null}},{"tags":{"$not":{"$some":{"$ne":null}},"$unknownAs":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":693,"output_tokens":54},"repair":{"input":{"filter":{"$or":[{"tags":{"$eq":[]}},{"tags":{"$unknownAs":true,"$not":{"$some":{"$eq":""}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":810,"output_tokens":53},"problemShown":{"type":"invalid-operand","detail":"/$or/1/tags/$not/$some/$ne must be string.","pointer":"/filter/$or/1/tags/$not/$some/$ne"}}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"c12","trial":1,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},{"microchip":null},{"microchip":{"$not":{"$eq":"CHIP-001"},"$unknownAs":true}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":721,"output_tokens":70}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"b01","trial":1,"input":{"filter":{"vaccinations":{"$some":{"name":"rabies","boosterDue":{"$lt":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":726,"output_tokens":47},"repair":{"text":"I can’t express that query with the available pet search tool.\n\nThe tool does support querying `vaccinations`, but it doesn’t expose a queryable `name` path for an individual vaccination record, so I can’t reliably restrict the overdue booster to the rabies shot itself rather than another shot.","stopReason":"stop","usage":{"input_tokens":825,"output_tokens":63},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/name"}}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"b11","trial":1,"input":{"filter":{"$or":[{"vaccinations":{"$eq":null}},{"vaccinations":{"$not":{"$some":{"$ne":"rabies"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":51},"repair":{"input":{"filter":{"$or":[{"vaccinations":{"$eq":null}},{"vaccinations":{"$size":0}},{"$and":[{"vaccinations":{"$some":{"$in":["rabies"]}}},{"vaccinations":{"$not":{"$some":{"$not":{"$in":["rabies"]}}}}}]}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":827,"output_tokens":90},"problemShown":{"type":"unsupported-operator","detail":"$ne is not available here. The operators this field accepts are listed under it in the schema.","pointer":"/filter/$or/1/vaccinations/$not/$some/$ne"}}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"m01","trial":1,"input":{"filter":{"tags":{"$some":{"$eq":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":32}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"a01","trial":1,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":25}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"m02","trial":1,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},{"microchip":null},{"$not":{"microchip":{"$eq":"CHIP-001"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":719,"output_tokens":64}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"d05","trial":1,"input":{"filter":{"notes":3},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":698,"output_tokens":24},"repair":{"text":"I can’t express that query with this tool as written: `notes` must be filtered with an object constraint, but the tool rejected a direct numeric equality on `notes`. If you want, I can try a different notes-based filter if you can tell me the intended structure of `notes`.","stopReason":"stop","usage":{"input_tokens":764,"output_tokens":62},"problemShown":{"type":"invalid-operand","detail":"/notes must be object.","pointer":"/filter/notes"}}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"a04","trial":1,"input":{"filter":{"weightKg":{"$between":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14099,"output_tokens":32}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"a09","trial":1,"input":{"filter":{"tags":{"$some":{"$in":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":35}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"a12","trial":1,"input":{"filter":{"tags":{"$size":{"$eq":0}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14078,"output_tokens":31}} +{"provider":"gpt-5.4-mini","arm":"jql-raw","caseId":"m03","trial":1,"input":{"filter":{"status":"Available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":25},"repair":{"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":769,"output_tokens":25},"problemShown":{"type":"invalid-operand","detail":"/status must be one of: \"available\", \"pending\", \"sold\".","pointer":"/filter/status"}}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"c05","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true,"$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14097,"output_tokens":43}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"c01","trial":1,"input":{"filter":{"microchip":{"$eq":"","$exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":34}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"c12","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14106,"output_tokens":38}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"d05","trial":1,"input":{"filter":{"notes":{"$eq":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14083,"output_tokens":28}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"e03","trial":1,"input":{"filter":{"name":{"$startsWith":"C","$endsWith":"o"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14084,"output_tokens":35}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"a01","trial":1,"input":{"filter":{"status":{"$eq":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":28}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"m02","trial":1,"input":{"filter":{"microchip":{"$ne":"CHIP-001","$unknownAs":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14104,"output_tokens":38}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"b01","trial":1,"input":{"filter":{"vaccinations":{"$some":{"$and":[{"name":"rabies"},{"boosterDue":{"$lt":"2025-01-01"}}]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14111,"output_tokens":52},"repair":{"text":"I can’t express that exactly with the available pet search tool.\n\nThe tool does support filtering within `vaccinations`, but it does not expose a queryable `name` field for vaccination records, so I can’t specifically identify the rabies shot versus other shots.","stopReason":"stop","usage":{"input_tokens":14219,"output_tokens":56},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/vaccinations/$some/$and/0/name"}}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"m03","trial":1,"input":{"filter":{"status":"available"},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":25}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"b11","trial":1,"input":{"filter":{"$or":[{"vaccinations":{"$exists":false}},{"vaccinations":{"$size":{"$eq":0}}},{"vaccinations":{"$every":{"name":{"$eq":"rabies"}}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14094,"output_tokens":62},"repair":{"text":"I can’t express that query with the tool as written.\n\nThe tool lets me filter on whether `vaccinations` exists, its size, and quantify over vaccination items, but it does not expose a queryable field for the vaccination name itself. So I can’t reliably ask for “only rabies” records.","stopReason":"stop","usage":{"input_tokens":14212,"output_tokens":72},"problemShown":{"type":"unknown-field","detail":"\"name\" is not a queryable path on this resource.","pointer":"/filter/$or/2/vaccinations/$every/name"}}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"b06","trial":1,"input":{"filter":{"tags":{"$not":{"$some":{"$eq":"indoor"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14079,"output_tokens":36}} +{"provider":"gpt-5.4-mini","arm":"jql-generated","caseId":"m01","trial":1,"input":{"filter":{"tags":{"$some":{"$eq":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":14077,"output_tokens":32}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"b11","trial":1,"input":{"filter":{"$nor":[{"vaccinations":{"$not":{"$eq":"rabies"}}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":692,"output_tokens":39}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"c01","trial":1,"input":{"filter":{"microchip":{"$eq":""}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":29}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"a09","trial":1,"input":{"filter":{"$or":[{"labels":{"$eq":"indoor"}},{"labels":{"$eq":"small"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":42},"repair":{"text":"I can’t express that query with the available pet search tool because `labels` is not a queryable field on the pet records.","stopReason":"stop","usage":{"input_tokens":775,"output_tokens":30},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$or/0/labels"}}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"b06","trial":1,"input":{"filter":{"$nor":[{"labels":{"$eq":"indoor"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":677,"output_tokens":34},"repair":{"input":{"filter":{"$nor":[{"label":{"$eq":"indoor"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":763,"output_tokens":34},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$nor/0/labels"}}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"a12","trial":1,"input":{"filter":{"$nor":[{"labels":{"$eq":"*"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":676,"output_tokens":33},"repair":{"text":"I can’t express “pets with no labels at all” with the available pet search tool, because `labels` is not a queryable field in this collection.","stopReason":"stop","usage":{"input_tokens":761,"output_tokens":36},"problemShown":{"type":"unknown-field","detail":"\"labels\" is not a queryable path on this resource.","pointer":"/filter/$nor/0/labels"}}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"d05","trial":1,"input":{"filter":{"notes":{"$eq":3}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":681,"output_tokens":28}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"c12","trial":1,"input":{"filter":{"$or":[{"microchip":{"$ne":"CHIP-001"}},{"microchip":{"$exists":false}},{"microchip":null}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":704,"output_tokens":52}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"a04","trial":1,"input":{"filter":{"weight":{"$gte":1,"$lte":5}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":697,"output_tokens":33},"repair":{"text":"I can’t express that query with the available pet-store search tool because `weight` is not a queryable field on the pet records.","stopReason":"stop","usage":{"input_tokens":778,"output_tokens":31},"problemShown":{"type":"unknown-field","detail":"\"weight\" is not a queryable path on this resource.","pointer":"/filter/weight"}}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"e03","trial":1,"input":{"filter":{"name":{"$regex":"^C.*o$"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":682,"output_tokens":32}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"b01","trial":1,"input":{"filter":{"rabies_shot":{"booster_due":{"$lt":"2025-01-01"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":709,"output_tokens":40},"repair":{"text":"I can’t express that query with the available pet-search tool because the tool doesn’t expose a queryable field for a rabies shot’s booster due date, so I can’t reliably filter for “rabies booster overdue before 1 Jan 2025.”\nI can’t express that query with the available pet-search tool because it doesn’t expose a queryable field for a rabies shot’s booster due date. So I can’t reliably filter for “rabies booster overdue before 1 Jan 2025” using this tool.","stopReason":"stop","usage":{"input_tokens":805,"output_tokens":117},"problemShown":{"type":"unknown-field","detail":"\"booster_due\" is not a queryable path on this resource.","pointer":"/filter/rabies_shot/booster_due"}}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"c05","trial":1,"input":{"filter":{"chip_id":{"$ne":"","$not":{"$eq":null}}}},"text":"","stopReason":"tool_calls","usage":{"input_tokens":695,"output_tokens":39},"repair":{"text":"I can’t answer that with the available pet search tool, because the tool doesn’t expose a queryable field for microchip IDs. So I can’t confirm which pets have a chip other than `CHIP-001`, and I can’t reliably exclude the uncertain cases.","stopReason":"stop","usage":{"input_tokens":776,"output_tokens":58},"problemShown":{"type":"unknown-field","detail":"\"chip_id\" is not a queryable path on this resource.","pointer":"/filter/chip_id"}}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"m02","trial":1,"input":{"filter":{"$nor":[{"microchip":{"$eq":"CHIP-001"}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":702,"output_tokens":37}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"m03","trial":1,"input":{"filter":{"status":{"$eq":"Available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":28}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"a01","trial":1,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":24}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"a04","trial":1,"input":{"q":"weightKg:[1 TO 5]","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":953,"output_tokens":29}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"a12","trial":1,"input":{"q":"- _exists_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":932,"output_tokens":26}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"a09","trial":1,"input":{"q":"tags:(indoor OR small)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":28}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"b01","trial":1,"input":{"q":"vaccinations.vaccine:rabies AND vaccinations.boosterDue:<2025-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":965,"output_tokens":40}} +{"provider":"gpt-5.4-mini","arm":"jql-published","caseId":"m01","trial":1,"input":{"filter":{"indoor":{"$eq":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":675,"output_tokens":29},"repair":{"text":"I can’t answer that with the pet search tool as given: `indoor` is not a queryable field on the pet records. If you want, I can help search using any other known field names.","stopReason":"stop","usage":{"input_tokens":754,"output_tokens":46},"problemShown":{"type":"unknown-field","detail":"\"indoor\" is not a queryable path on this resource.","pointer":"/filter/indoor"}}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"c01","trial":1,"input":{"q":"microchip:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":24}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"b06","trial":1,"input":{"q":"-tags:indoor OR -_exists_:tags","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":933,"output_tokens":32}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"b11","trial":1,"input":{"q":"NOT vaccinations.vaccine:(distemper OR parvo)","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":948,"output_tokens":32}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"c05","trial":1,"input":{"q":"_exists_:microchip AND -microchip:CHIP-001 AND -microchip:null","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":951,"output_tokens":40}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"c12","trial":1,"input":{"q":"-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":960,"output_tokens":40}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"d05","trial":1,"input":{"q":"notes:3","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":937,"output_tokens":24}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"e03","trial":1,"input":{"q":"name:C*o","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":938,"output_tokens":25}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"m01","trial":1,"input":{"q":"tags:indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":25}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"m03","trial":1,"input":{"q":"status:available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":931,"output_tokens":24}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"a01","trial":1,"input":{"where":{"status":{"op":"is","value":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":31}} +{"provider":"gpt-5.4-mini","arm":"search-string","caseId":"m02","trial":1,"input":{"q":"-microchip:CHIP-001 OR microchip:null OR -_exists_:microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":958,"output_tokens":40}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"a04","trial":1,"input":{"where":{"weightKg":{"op":"between","value":[1,5]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1135,"output_tokens":35}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"a12","trial":1,"input":{"where":{"tags":{"op":"count","value":{"op":"is","value":0}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1114,"output_tokens":37}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"a09","trial":1,"input":{"where":{"tags":{"op":"some","value":{"op":"anyof","value":["indoor","small"]}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":42}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"b06","trial":1,"input":{"where":{"tags":{"op":"noneof","value":["indoor"]}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1115,"output_tokens":34}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"b01","trial":1,"input":{"where":{"vaccinations":{"op":"some","value":{"vaccine":{"op":"is","value":"rabies"},"boosterDue":{"op":"lt","value":"2025-01-01"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1147,"output_tokens":61}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"b11","trial":1,"input":{"where":{"vaccinations":{"op":"every","value":{"vaccine":{"op":"is","value":"rabies"}}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1130,"output_tokens":43}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"c01","trial":1,"input":{"where":{"microchip":{"op":"isblank","value":true,"exists":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":37}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"c05","trial":1,"input":{"where":{"all":[{"microchip":{"op":"exists","value":true}},{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":false}}]},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1133,"output_tokens":57}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"c12","trial":1,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1142,"output_tokens":41}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"d05","trial":1,"input":{"where":{"notes":{"op":"is","value":3,"type":"number"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1119,"output_tokens":35}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"m01","trial":1,"input":{"where":{"tags":{"op":"some","value":{"op":"is","value":"indoor"}}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":38}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"e03","trial":1,"input":{"where":{"name":{"op":"like","value":"C%o"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1120,"output_tokens":33}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"m02","trial":1,"input":{"where":{"microchip":{"op":"isnot","value":"CHIP-001","orUnknown":true}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1140,"output_tokens":41}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"a01","trial":1,"input":{"status":"available","limit":20},"text":"","stopReason":"tool_calls","usage":{"input_tokens":399,"output_tokens":22}} +{"provider":"gpt-5.4-mini","arm":"bespoke-json","caseId":"m03","trial":1,"input":{"where":{"status":{"op":"is","value":"available"}},"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":1113,"output_tokens":31}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"a04","trial":1,"input":{"minWeightKg":1,"maxWeightKg":5,"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":419,"output_tokens":30}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"b01","trial":1,"input":{"tag":"rabies shot","bornTo":"2025-01-01","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":431,"output_tokens":34}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"a09","trial":1,"input":{"tag":"indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":403,"output_tokens":23}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"c05","trial":1,"input":{"limit":50,"tag":"microchip"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":417,"output_tokens":23}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"b11","trial":1,"input":{"limit":50,"tag":"rabies"},"text":"","stopReason":"tool_calls","usage":{"input_tokens":414,"output_tokens":23}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"c01","trial":1,"text":"I can’t answer that with the available pet-store search tool.\n\nThe tool only supports filtering by fields like status, species, dates, weight, tag, city, neutered, and name substring. It does not expose the raw pet records or a way to inspect whether a `microchip` field is present but blank.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":68}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"e03","trial":1,"input":{"nameContains":"C","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":404,"output_tokens":23}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"m01","trial":1,"input":{"tag":"indoor","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":23}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"a12","trial":1,"input":{"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":398,"output_tokens":18},"repair":{"text":"I can’t express “pets with no labels at all” with the available pet-search tool.\n\nThe tool supports filtering by a single required label, but it does not support searching for pets that have no labels.","stopReason":"stop","usage":{"input_tokens":458,"output_tokens":45},"problemShown":{"type":"malformed-query","detail":"at least one search parameter is required.","pointer":"/"}}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"c12","trial":1,"input":{"tag":"microchip","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":426,"output_tokens":23}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"d05","trial":1,"text":"I can’t express that query with the available pet-search tool. The tool supports filtering by status, species, birth date, weight, tag, shelter city, neutered, and name substring, but not by a `notes` field or numeric note values.\nI can’t query that with the available pet-search tool. It doesn’t support filtering by a `notes` value, only by fields like status, species, birth date, weight, tag, shelter city, neutered, and name substring.","stopReason":"stop","usage":{"input_tokens":403,"output_tokens":111}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"b06","trial":1,"input":{"limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":399,"output_tokens":18},"repair":{"text":"I can’t express “do not carry the indoor label” with the available pet search tool.\n\nThe tool only supports filtering by a single required `tag` value, not excluding pets by label or searching for pets that lack a tag.","stopReason":"stop","usage":{"input_tokens":459,"output_tokens":50},"problemShown":{"type":"malformed-query","detail":"at least one search parameter is required.","pointer":"/"}}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"m03","trial":1,"input":{"status":"available","limit":50},"text":"","stopReason":"tool_calls","usage":{"input_tokens":397,"output_tokens":22}} +{"provider":"gpt-5.4-mini","arm":"fixed-params","caseId":"m02","trial":1,"text":"I can’t express “microchip CHIP-001” or “chip is blank or missing” with the available pet-search tool, because it only supports filtering by status, species, birth date, weight, tag, city, neutered, and name substring. It has no field for microchip/chip number or blank/missing chip.\nI can’t answer that with the available tool.\n\nThe pet search tool does **not** support filtering by microchip/chip number, or by blank/missing chip values. It only lets me search by things like status, species, birth date, weight, tag, city, neutered, and name substring.\n\nIf you want, I can still help search pets by one of those supported fields.","stopReason":"stop","usage":{"input_tokens":424,"output_tokens":158}} diff --git a/experiments/intent-to-filter/runner.mjs b/experiments/intent-to-filter/runner.mjs new file mode 100644 index 0000000..a908f82 --- /dev/null +++ b/experiments/intent-to-filter/runner.mjs @@ -0,0 +1,209 @@ +/** + * runner.mjs — put the questions to the models, and write down what came back. + * + * node experiments/intent-to-filter/runner.mjs --dry-run + * node experiments/intent-to-filter/runner.mjs --pilot + * node experiments/intent-to-filter/runner.mjs --run + * + * Two decisions here are deliberate, and each would corrupt the measurement if + * taken the other way. + * + * **No structured-output or strict-schema mode.** Constrained decoding would + * drive the two schema arms to structural validity by construction, and it is + * available to them and to no other arm at any price. That is a real advantage + * the design declines to bank, because it is not what MCP servers do today. + * + * **Tool choice left on auto.** Two questions cannot be answered by the tool at + * all, and saying so is the correct response. Forcing a call would turn the + * abstention probe into a fabrication probe. + * + * Results are appended to JSONL as they arrive, so an interrupted run loses + * nothing and can be resumed against the same file. + */ + +import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { dirname, join } from "node:path"; +import { parseArgs } from "node:util"; + +import { CORPUS } from "./corpus.mjs"; +import { ARMS, BY_NAME, approxTokens } from "./arms/index.mjs"; +import { PROVIDERS, ask, incompatibility, pooled } from "./providers.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const RESULTS = join(here, "results"); + +/** Concurrency, per provider. Groq rate-limits on input tokens per minute, and + * the generated schema is 15k tokens a call, so it gets the smallest pool. */ +const CONCURRENCY = { "gpt-5.5": 8, "gpt-5.4-mini": 8, "gpt-oss-120b": 2, "gemini-3.8-flash": 2 }; + +const key = (provider, arm, caseId, trial) => `${provider}|${arm}|${caseId}|t${trial}`; + +// --------------------------------------------------------------------------- +// Repair: hand the rejection back the way an MCP server would +// --------------------------------------------------------------------------- + +function repairTurns(providerKey, arm, attempt) { + const problem = JSON.stringify(attempt.problem, null, 2); + if (PROVIDERS[providerKey].family === "gemini") { + return [ + { role: "model", parts: [{ functionCall: { name: arm.tool.name, args: attempt.input ?? {} } }] }, + { role: "user", parts: [{ functionResponse: { name: arm.tool.name, response: { error: attempt.problem } } }] }, + ]; + } + return [ + attempt.assistantMessage, + { role: "tool", tool_call_id: attempt.toolCallId, content: problem }, + ]; +} + +// --------------------------------------------------------------------------- +// Planning +// --------------------------------------------------------------------------- + +function cells({ providers, arms, cases, trials }) { + const out = []; + for (const provider of providers) { + // Trials can be capped per provider. Groq rate-limits on tokens per minute + // and jql-raw alone is 61k characters a call, so five trials there would + // take hours of mostly backoff. Fewer trials is an honest widening of that + // provider's intervals, reported as such; more concurrency would not help, + // because the ceiling is tokens, not connections. + const n = typeof trials === "number" ? trials : (trials[provider] ?? 5); + for (const armName of arms) { + const arm = BY_NAME[armName]; + const blocked = incompatibility(provider, arm); + for (const testCase of cases) { + for (let trial = 1; trial <= n; trial += 1) { + out.push({ provider, arm: armName, caseId: testCase.id, trial, blocked, testCase, armModule: arm }); + } + } + } + } + return out; +} + +function dryRun(plan) { + const all = cells(plan); + const live = all.filter((c) => !c.blocked); + console.log("\nCompatibility\n"); + console.log(` ${"provider".padEnd(18)} ${ARMS.map((a) => a.name.slice(0, 13).padEnd(14)).join("")}`); + for (const provider of plan.providers) { + const row = ARMS.map((a) => (incompatibility(provider, BY_NAME[a.name]) ? "—" : "yes").padEnd(14)).join(""); + console.log(` ${provider.padEnd(18)} ${row}`); + } + + console.log("\nRequests\n"); + for (const provider of plan.providers) { + const mine = live.filter((c) => c.provider === provider); + const tokens = mine.reduce((sum, c) => sum + approxTokens(JSON.stringify(c.armModule.tool)) + 200, 0); + console.log(` ${provider.padEnd(18)} ${String(mine.length).padStart(5)} calls ~${String(Math.round(tokens / 1000)).padStart(5)}k input tokens`); + } + console.log(`\n ${live.length} calls, ${all.length - live.length} cells skipped as provider-incompatible.`); + console.log(" Nothing was sent.\n"); +} + +// --------------------------------------------------------------------------- +// Running +// --------------------------------------------------------------------------- + +async function run(plan, { label, repair }) { + if (!existsSync(RESULTS)) mkdirSync(RESULTS, { recursive: true }); + const path = join(RESULTS, `${label}.jsonl`); + + const done = new Set(); + if (existsSync(path)) { + for (const line of readFileSync(path, "utf8").split("\n").filter(Boolean)) { + const row = JSON.parse(line); + done.add(key(row.provider, row.arm, row.caseId, row.trial)); + } + console.log(`resuming: ${done.size} responses already in ${path}`); + } + + const all = cells(plan); + const write = (row) => appendFileSync(path, JSON.stringify(row) + "\n"); + + // Incompatible cells are recorded once, not called. + for (const cell of all.filter((c) => c.blocked)) { + const id = key(cell.provider, cell.arm, cell.caseId, cell.trial); + if (done.has(id)) continue; + write({ provider: cell.provider, arm: cell.arm, caseId: cell.caseId, trial: cell.trial, incompatible: cell.blocked }); + done.add(id); + } + + for (const provider of plan.providers) { + const todo = all.filter((c) => c.provider === provider && !c.blocked && !done.has(key(c.provider, c.arm, c.caseId, c.trial))); + if (todo.length === 0) continue; + console.log(`\n${PROVIDERS[provider].label}: ${todo.length} calls`); + + const tasks = todo.map((cell) => async () => { + const response = await ask(cell.provider, cell.armModule, cell.testCase.ask); + const row = { + provider: cell.provider, arm: cell.arm, caseId: cell.caseId, trial: cell.trial, + input: response.input, text: response.text, stopReason: response.stopReason, + usage: response.usage, error: response.error, + }; + + // One repair round trip, if the arm rejected what came back. + if (repair && !response.error && response.input !== undefined) { + const rejected = cell.armModule.validate(response.input); + if (rejected) { + const second = await ask(cell.provider, cell.armModule, cell.testCase.ask, + repairTurns(cell.provider, cell.armModule, { ...response, problem: rejected })); + row.repair = { + input: second.input, text: second.text, stopReason: second.stopReason, + usage: second.usage, error: second.error, problemShown: rejected, + }; + } + } + write(row); + return row; + }); + + await pooled(tasks, CONCURRENCY[provider] ?? 4, (n, total) => { + if (n % 10 === 0 || n === total) process.stdout.write(`\r ${n}/${total}`); + }); + console.log(""); + } + + console.log(`\nwrote ${path}`); + console.log("report with: node experiments/intent-to-filter/report.mjs"); +} + +// --------------------------------------------------------------------------- + +const FULL = { + providers: ["gpt-5.5", "gemini-3.8-flash", "gpt-oss-120b"], + arms: ARMS.map((a) => a.name), + cases: CORPUS, + trials: { "gpt-5.5": 5, "gemini-3.8-flash": 5, "gpt-oss-120b": 2 }, +}; + +const PILOT = { + providers: ["gpt-5.4-mini"], + arms: ARMS.map((a) => a.name), + cases: CORPUS.filter((c) => ["a01", "a04", "a09", "a12", "b01", "b06", "b11", "c01", "c05", "c12", "d05", "e03", "m01", "m02", "m03"].includes(c.id)), + trials: 1, +}; + +export { FULL, PILOT }; + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + const { values } = parseArgs({ + options: { + "dry-run": { type: "boolean", default: false }, + pilot: { type: "boolean", default: false }, + run: { type: "boolean", default: false }, + "no-repair": { type: "boolean", default: false }, + label: { type: "string" }, + }, + }); + const plan = values.pilot ? PILOT : FULL; + if (values["dry-run"]) dryRun(plan); + else if (values.pilot || values.run) { + await run(plan, { label: values.label ?? (values.pilot ? "pilot" : "full"), repair: !values["no-repair"] }); + } else { + console.log("choose one of --dry-run, --pilot, --run"); + process.exit(1); + } +} diff --git a/package-lock.json b/package-lock.json index 7bc7f37..4e0a316 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,19 +1,16 @@ { "name": "json-query-language", - "version": "0.2.0", + "version": "0.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "json-query-language", - "version": "0.2.0", + "version": "0.4.0", "license": "MIT", "devDependencies": { "ajv": "^8.17.1", "ajv-formats": "^3.0.1" - }, - "engines": { - "node": ">=20" } }, "node_modules/ajv": { diff --git a/package.json b/package.json index dca71a8..039bb7d 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,10 @@ "example:mcp": "node examples/mcp-server/server.mjs", "example:mcp:demo": "node examples/mcp-server/demo.mjs", "test:experiment": "node --test experiments/filter-to-sql/compile.test.mjs", - "experiment": "node experiments/filter-to-sql/run.mjs" + "experiment": "node experiments/filter-to-sql/run.mjs", + "test:intent": "node --test experiments/intent-to-filter/gold.test.mjs", + "intent": "node experiments/intent-to-filter/report.mjs", + "intent:run": "node experiments/intent-to-filter/runner.mjs" }, "devDependencies": { "ajv": "^8.17.1",