From 4e90c9fa32f8b00cb8187ceee1eaec61cb5dece3 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Mon, 19 Oct 2026 07:07:32 -0700 Subject: [PATCH 1/7] - --- .github/workflows/flutter_packages_test.yml | 32 +++ conformance/README.md | 8 + conformance/agent/direct_json/scaling.yaml | 201 ++++++++++++++++++ conformance/agent/express/scaling.yaml | 38 ++++ conformance/conformance_schema.json | 104 +++++++++ .../core/message_processor_scaling.yaml | 67 ++++++ dart/a2ui_agent/CHANGELOG.md | 4 +- dart/a2ui_agent/README.md | 5 + dart/a2ui_agent/dart_test.yaml | 27 +++ .../direct_json/json_reader.dart | 63 ++++-- .../inference_formats/direct_json/parser.dart | 148 ++++++++++--- .../lib/src/utils/payload_checker.dart | 16 ++ .../conformance/scaling_conformance_test.dart | 175 +++++++++++++++ dart/a2ui_agent/test/direct_json_test.dart | 41 ++++ dart/a2ui_core/CHANGELOG.md | 6 + dart/a2ui_core/README.md | 2 + dart/a2ui_core/dart_test.yaml | 27 +++ dart/a2ui_core/lib/a2ui_core.dart | 3 +- .../lib/src/validation/component_graph.dart | 70 +++++- dart/a2ui_core/test/component_graph_test.dart | 77 +++++++ ...ge_processor_scaling_conformance_test.dart | 133 ++++++++++++ python/a2ui_core/tests/test_conformance.py | 9 + .../tests/conformance/conformance_test.mjs | 5 + 23 files changed, 1210 insertions(+), 51 deletions(-) create mode 100644 conformance/agent/direct_json/scaling.yaml create mode 100644 conformance/agent/express/scaling.yaml create mode 100644 conformance/core/message_processor_scaling.yaml create mode 100644 dart/a2ui_agent/dart_test.yaml create mode 100644 dart/a2ui_agent/test/conformance/scaling_conformance_test.dart create mode 100644 dart/a2ui_core/dart_test.yaml create mode 100644 dart/a2ui_core/test/conformance/message_processor_scaling_conformance_test.dart diff --git a/.github/workflows/flutter_packages_test.yml b/.github/workflows/flutter_packages_test.yml index 40c582659b..de36135e1d 100644 --- a/.github/workflows/flutter_packages_test.yml +++ b/.github/workflows/flutter_packages_test.yml @@ -171,3 +171,35 @@ jobs: - name: Analyze code working-directory: ${{ matrix.package.path }} run: flutter analyze --fatal-infos + + # The tests tagged `performance` time streaming and checking payloads of two + # sizes, as conformance/agent/direct_json/scaling.yaml describes. A slow or + # loaded runner can fail them, so `dart_test.yaml` skips them in the jobs + # above and they run here, in a job of their own. package:test cannot stop a + # test that never yields, and a regression can make these run for minutes, + # so the job has a time limit of its own. + performance: + name: ${{ matrix.package.name }} (stable, performance) + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + package: + - name: dart_a2ui_agent + path: dart/a2ui_agent + - name: dart_a2ui_core + path: dart/a2ui_core + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/setup-dart + with: + channel: stable + - name: Install dependencies + working-directory: ${{ matrix.package.path }} + run: dart pub get + - name: Run performance tests + working-directory: ${{ matrix.package.path }} + run: dart test -P performance diff --git a/conformance/README.md b/conformance/README.md index 1176fe11a7..b65bdf87e4 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -18,6 +18,7 @@ Test suites are organized by functional domain: - `core/node_resolution.yaml`: Contains node-tree, scoped-binding, notification, identity, action, and destruction scenarios. See [the case format](core/node_resolution.md). - `core/expressions.yaml`: Contains test cases for the client-side expression parser behind `formatString`, covering literals, data bindings, function calls, nested interpolation, escaped markers and parse errors. - `core/functions.yaml`: Contains test cases for the basic catalog functions, and for the catalog a function call runs in on a surface. See [Writing cases for `evaluate_function`](#writing-cases-for-evaluate_function). +- `core/message_processor_scaling.yaml`: Timing cases for `processMessages`, with a validation config and without one, which check that a surface arriving in many small messages takes time in proportion to its size. See [Timing cases](#timing-cases). ### Agent (`agent/`) @@ -43,6 +44,7 @@ Everything a format decides for itself is covered once per format, in a folder n - `compiler.yaml`: Compiling one raw block into A2UI messages. Both folders. - `decompiler.yaml`: Writing messages back into the format's own notation, which is how a prompt example is authored as messages and shown to a model as notation. Both folders. - `response_streaming.yaml`: Parsing a response one chunk at a time, including the point at which a payload can first be emitted. `direct_json/` only, since the Express parser does not implement `parse_chunk` and an Express agent parses a buffered response whole. +- `scaling.yaml`: Timing cases, which check that the time to parse a payload grows in proportion to the payload: streaming and compiling in `direct_json/`, compiling in `express/`. See [Timing cases](#timing-cases). The blueprint declares `wrap`, `unwrap`, `compile`, `decompile` and `parse_response` on a single `Parser` interface, so `response_parser.yaml`, `compiler.yaml` and `decompiler.yaml` are three files for one interface. They are split by call because each has rules of its own: where a payload begins and ends, what it means, and how it is written back. The Express compiler suite is by far the largest of the three, since Express has a grammar to get right and direct JSON does not. @@ -110,6 +112,12 @@ Harnesses join adjacent literal parts before comparing, and drop empty ones. A c Errors are expressed with the suite's language-agnostic categories rather than an SDK's class names: `ParseError` maps to `A2uiExpressionError` in both the Dart and TypeScript clients, and `message` is matched as a regular expression against the error's text. +### Timing cases + +The `scaling.yaml` suites and `core/message_processor_scaling.yaml` time an operation rather than check what it returns, so that a parser or processor that returns the right result with a cost growing with the square of the payload fails a test. A `measure_growth` case (`MeasureGrowthTest` in `conformance_schema.json`) times `operation` on a payload written at the two `sizes`, n and 4n, and expects the larger to take at most `max_ratio` times as long as the smaller: a linear cost takes about 4 times as long and a quadratic one about 16 times, and the limit of 8 tells them apart. A `measure_overhead` case (`MeasureOverheadTest`) times streaming a payload against compiling it, and expects streaming to take at most `max_ratio` times as long. A payload is a `PayloadTemplate`: text segments and segments repeated n times, so every SDK times the same bytes. [`agent/direct_json/scaling.yaml`](agent/direct_json/scaling.yaml) describes how a harness times a case. + +Times depend on the machine and its load, so a harness runs these cases apart from its other tests and one at a time. The Dart packages tag them `performance`, which their `dart_test.yaml` skips unless `dart test -P performance` asks for them, and CI runs them in a job of their own. A harness that has no such runner skips the two actions with a reason, as the Python core and TypeScript web core harnesses do. + ### Writing cases for `evaluate_function` An `evaluate_function` case (`EvaluateFunctionTest` in `conformance_schema.json`) names a `function` and its `args`, and states the result in `expect` or the failure in `expectError`. Without a `surface` block, the harness invokes the function's implementation directly in the case's default catalog, which is how `core/functions.yaml` checks what each basic catalog function computes. diff --git a/conformance/agent/direct_json/scaling.yaml b/conformance/agent/direct_json/scaling.yaml new file mode 100644 index 0000000000..3bca57af6f --- /dev/null +++ b/conformance/agent/direct_json/scaling.yaml @@ -0,0 +1,201 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# How the time to parse a direct JSON payload grows with the payload. +# +# The other suites fix what a parser returns, and none of them notices a parser +# that returns the right parts slowly. A parser that reads the whole block again +# on every chunk, or checks every message it has read again, still passes them, +# while the time it takes to stream a payload grows with the square of the +# payload's size. +# +# A `measure_growth` case times one operation on a payload written at two +# sizes, n and 4n, and fails when the larger takes more than `max_ratio` times +# as long as the smaller. A cost that grows in proportion to the payload takes +# about 4 times as long at 4n, and one that grows with its square about 16 +# times, so a limit of 8 tells the two apart with room for noise. CI machines +# differ in speed and load too much for a limit in milliseconds, but two times +# taken on the same machine moments apart can be compared. +# +# `measure_overhead` times streaming a payload against compiling it whole, and +# fails when streaming takes more than `max_ratio` times as long. Streaming +# reads what each chunk adds and checks what it changes, so it does more work +# than compiling, but that work should stay a fixed multiple of it. The limit +# is loose and catches a parser that does far more on each chunk than the chunk +# needs. +# +# `payload` is a template, written out at each size as `PayloadTemplate` in +# `conformance_schema.json` describes. `parse_chunk` streams the response +# ``, payload, `` in chunks of `chunk_size` characters, +# about the size of a few tokens, and `compile` reads the payload whole. The +# payloads are ASCII, so a chunk of 16 characters is 16 bytes in any encoding. +# +# A harness times a case as follows: +# +# 1. It builds the format before the timing starts. Each timed run creates a +# parser from it and runs the operation to the end of the payload. +# 2. It runs each size once before timing either, so that a runtime that +# compiles code on first use has done so. +# 3. It then runs the sizes in turn, at least five times each, and compares the +# medians, so that one slow run cannot fail the case. +# +# The cases are slow and depend on the machine, so a harness runs them apart +# from its other tests and one at a time. The Dart SDKs run them in a CI job of +# their own. +# +# The sizes are those of payloads a model writes, from a few kilobytes to a few +# tens of kilobytes. A cost that grows with the square of the payload but is +# small for each byte, such as copying the text received so far on every chunk, +# stays below the limit at these sizes. +# +# A component healed as it arrives has to satisfy its catalog each time a chunk +# changes it, so it is checked again whole on that chunk. A component that +# itself grows with n, such as a root naming n children, therefore costs time +# that grows with the square of n. The v0.9 case adds its components to a +# surface an earlier response created, so that none of them names the others. +# In v1.0 a component is held until it closes, and is checked once. + +# --- Components --- + +- name: test_scaling_stream_components_v10 + description: Streaming a surface of n components in small chunks takes about four times as long at 4n, since each chunk reads and checks only what it adds. + action: measure_growth + args: + format: direct_json + catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + operation: parse_chunk + chunk_size: 16 + payload: + - '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}, {"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Column", "children": [' + - repeat: '"item_{i}"' + separator: ", " + - "]}, " + - repeat: '{"id": "item_{i}", "component": "Text", "text": "Item number {i}"}' + separator: ", " + - "]}}]" + sizes: [100, 400] + max_ratio: 8 + +- name: test_scaling_stream_components_healed_v09 + description: Streaming n components healed as their text arrives takes about four times as long at 4n, since each chunk checks again only the component it changes. + action: measure_growth + args: + format: direct_json + catalog: "test_data/catalogs/streaming_catalog_v0_9.json" + progressive_keys: ["text"] + operation: parse_chunk + chunk_size: 16 + payload: + - '[{"version": "v0.9", "updateComponents": {"surfaceId": "s1", "components": [' + - repeat: '{"id": "item_{i}", "component": "Text", "text": "Item number {i}"}' + separator: ", " + - "]}}]" + sizes: [100, 400] + max_ratio: 8 + +- name: test_scaling_stream_single_component_messages_v10 + description: Streaming n messages of one component each takes about four times as long at 4n, since a message that has closed is not read or checked again. + action: measure_growth + args: + format: direct_json + catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + operation: parse_chunk + chunk_size: 16 + payload: + - '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}, {"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Column", "children": [' + - repeat: '"item_{i}"' + separator: ", " + - "]}]}}, " + - repeat: '{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "item_{i}", "component": "Text", "text": "Item number {i}"}]}}' + separator: ", " + - "]" + sizes: [100, 400] + max_ratio: 8 + +- name: test_scaling_compile_components_v10 + description: Compiling a surface of n components takes about four times as long at 4n. + action: measure_growth + args: + format: direct_json + catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + operation: compile + payload: + - '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}, {"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Column", "children": [' + - repeat: '"item_{i}"' + separator: ", " + - "]}, " + - repeat: '{"id": "item_{i}", "component": "Text", "text": "Item number {i}"}' + separator: ", " + - "]}}]" + sizes: [200, 800] + max_ratio: 8 + +- name: test_scaling_compile_single_component_messages_v10 + description: Compiling n messages of one component each takes about four times as long at 4n, since applying a message checks what it changes rather than the whole surface. + action: measure_growth + args: + format: direct_json + catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + operation: compile + payload: + - '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}, {"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Column", "children": [' + - repeat: '"item_{i}"' + separator: ", " + - "]}]}}, " + - repeat: '{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "item_{i}", "component": "Text", "text": "Item number {i}"}]}}' + separator: ", " + - "]" + sizes: [200, 800] + max_ratio: 8 + +# --- Data model --- + +- name: test_scaling_stream_data_model_v10 + description: Streaming a data model of n items, its strings healed as they arrive, takes about four times as long at 4n, since each chunk checks only what the value gains. + action: measure_growth + args: + format: direct_json + catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + progressive_keys: ["name", "description"] + operation: parse_chunk + chunk_size: 16 + payload: + - '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}, {"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Text", "text": {"@path": "/items/0/name"}}]}}, {"version": "v1.0", "updateDataModel": {"surfaceId": "s1", "path": "/items", "value": [' + - repeat: '{"name": "Item number {i}", "description": "The item listed at position {i}", "price": {i}}' + separator: ", " + - "]}}]" + sizes: [100, 400] + max_ratio: 8 + +# --- Streaming against compiling --- + +- name: test_scaling_stream_over_compile_v10 + description: Streaming a surface in small chunks costs at most a fixed multiple of compiling it whole. + action: measure_overhead + args: + format: direct_json + catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + operation: parse_chunk + baseline: compile + chunk_size: 16 + payload: + - '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}, {"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Column", "children": [' + - repeat: '"item_{i}"' + separator: ", " + - "]}, " + - repeat: '{"id": "item_{i}", "component": "Text", "text": "Item number {i}"}' + separator: ", " + - "]}}]" + size: 400 + max_ratio: 10 diff --git a/conformance/agent/express/scaling.yaml b/conformance/agent/express/scaling.yaml new file mode 100644 index 0000000000..6dc4838648 --- /dev/null +++ b/conformance/agent/express/scaling.yaml @@ -0,0 +1,38 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# How the time to compile an Express block grows with the block. +# +# A `measure_growth` case times `compile` on a block written at two sizes, n +# and 4n, and fails when the larger takes more than `max_ratio` times as long +# as the smaller. `agent/direct_json/scaling.yaml` explains the measure, how a +# harness times a case, and why the cases run apart from the other tests. The +# Express parser does not stream, so there is nothing else to time. + +- name: test_scaling_compile_express_v10 + description: Compiling a block of n components takes about four times as long at 4n. + action: measure_growth + args: + format: express + catalog: "test_data/catalogs/simplified_catalog_v1_0.json" + operation: compile + payload: + - "root = Column([" + - repeat: "item_{i}" + separator: ", " + - "])\n" + - repeat: 'item_{i} = Text("Item number {i}")' + separator: "\n" + sizes: [200, 800] + max_ratio: 8 diff --git a/conformance/conformance_schema.json b/conformance/conformance_schema.json index c618672c0a..fb2b962c63 100644 --- a/conformance/conformance_schema.json +++ b/conformance/conformance_schema.json @@ -21,6 +21,8 @@ {"$ref": "#/$defs/DecompileTest"}, {"$ref": "#/$defs/ParseResponseTest"}, {"$ref": "#/$defs/ParseChunkTest"}, + {"$ref": "#/$defs/MeasureGrowthTest"}, + {"$ref": "#/$defs/MeasureOverheadTest"}, {"$ref": "#/$defs/CreateFormatTest"}, {"$ref": "#/$defs/CreateProcessorTest"}, {"$ref": "#/$defs/EvaluateFunctionTest"}, @@ -1298,6 +1300,108 @@ }, "required": ["action", "args", "steps"] }, + "PayloadTemplate": { + "type": "array", + "minItems": 1, + "description": "The text of a payload of size n, written as segments that are joined in order. A string is copied as it is. A `repeat` is its text written n times, with each `{i}` in it replaced by the number of the copy, counting from 0, and `separator` between the copies.", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "repeat": {"type": "string"}, + "separator": {"type": "string", "default": ""} + }, + "required": ["repeat"], + "additionalProperties": false + } + ] + } + }, + "MeasuredArgs": { + "type": "object", + "description": "What the timed operation is built from: a format from `format`, `catalog` and `progressive_keys`, or a processor from the case's `catalogPaths` and `validation_config`. The format and the catalogs are built before the timing starts, and each timed run creates a new parser from the format, or a new processor.", + "properties": { + "format": {"$ref": "#/$defs/InferenceFormatName"}, + "catalog": {"$ref": "#/$defs/AgentCatalogRef"}, + "progressive_keys": { + "type": "array", + "items": {"type": "string"}, + "description": "The string keys whose value a direct JSON parser may close early while it is still arriving." + }, + "validation_config": { + "enum": ["none", "relaxed"], + "description": "The checks the processor makes. `none` is a processor without a validation config, as a renderer has by default, which checks each batch of components for duplicate ids and against its catalog. `relaxed` also checks the graph each batch leaves behind for cycles and depth, while allowing orphans, references to components that have not arrived and a missing root, as a surface arriving over several messages needs." + } + }, + "additionalProperties": false + }, + "MeasuredOperation": { + "enum": ["parse_chunk", "compile", "process_messages"], + "description": "The call a timing case times. `parse_chunk` hands a new parser the response ``, payload, ``, in chunks of `chunk_size` characters. `compile` hands a new parser the payload whole. `process_messages` hands a new processor the messages of the payload, a JSON list decoded before the timing starts, in one call." + }, + "MeasureGrowthTest": { + "type": "object", + "description": "Timing one operation on the payload written at two sizes, to show that its cost grows in proportion to the payload rather than with its square.", + "properties": { + "name": {"type": "string"}, + "description": {"type": "string"}, + "action": {"const": "measure_growth"}, + "protocolVersion": {"$ref": "#/$defs/ProtocolVersion"}, + "catalogPaths": { + "type": "array", + "items": {"type": "string"}, + "description": "For `process_messages`, the catalogs the processor holds, as paths relative to the repository root." + }, + "args": {"$ref": "#/$defs/MeasuredArgs"}, + "operation": {"$ref": "#/$defs/MeasuredOperation"}, + "chunk_size": {"type": "integer", "minimum": 1}, + "payload": {"$ref": "#/$defs/PayloadTemplate"}, + "sizes": { + "type": "array", + "items": {"type": "integer", "minimum": 1}, + "minItems": 2, + "maxItems": 2, + "description": "The two sizes n the payload is written at, the smaller first." + }, + "max_ratio": { + "type": "number", + "exclusiveMinimum": 1, + "description": "The largest the median time at the second size may be, as a multiple of the median time at the first." + } + }, + "required": ["name", "action", "args", "operation", "payload", "sizes", "max_ratio"], + "allOf": [ + { + "if": {"properties": {"operation": {"const": "parse_chunk"}}}, + "then": {"required": ["chunk_size"]} + } + ], + "additionalProperties": false + }, + "MeasureOverheadTest": { + "type": "object", + "description": "Timing streaming a payload against compiling it whole, to show that streaming costs at most a fixed multiple of compiling.", + "properties": { + "name": {"type": "string"}, + "description": {"type": "string"}, + "action": {"const": "measure_overhead"}, + "args": {"$ref": "#/$defs/MeasuredArgs"}, + "operation": {"const": "parse_chunk"}, + "baseline": {"const": "compile"}, + "chunk_size": {"type": "integer", "minimum": 1}, + "payload": {"$ref": "#/$defs/PayloadTemplate"}, + "size": {"type": "integer", "minimum": 1, "description": "The size n the payload is written at."}, + "max_ratio": { + "type": "number", + "exclusiveMinimum": 1, + "description": "The largest the median time of `operation` may be, as a multiple of the median time of `baseline`." + } + }, + "required": ["name", "action", "args", "operation", "baseline", "chunk_size", "payload", "size", "max_ratio"], + "additionalProperties": false + }, "CreateFormatTest": { "type": "object", "description": "Building an inference format from its factory and checking what the format hands back.", diff --git a/conformance/core/message_processor_scaling.yaml b/conformance/core/message_processor_scaling.yaml new file mode 100644 index 0000000000..2a4ea662e3 --- /dev/null +++ b/conformance/core/message_processor_scaling.yaml @@ -0,0 +1,67 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# How the time to process messages grows with the surface they build. +# +# A processor checks each `updateComponents` against the surface it updates. +# One that walks the whole surface for every message takes time that grows +# with the square of the surface's size when the surface arrives in many small +# messages, as it does from an agent that streams. +# +# A `measure_growth` case processes a payload written at two sizes, n and 4n, +# and fails when the larger takes more than `max_ratio` times as long as the +# smaller. `agent/direct_json/scaling.yaml` explains the measure, how a harness +# times a case, and why the cases run apart from the other tests. Each timed run +# creates a processor holding the catalogs of `catalogPaths` and hands it the +# payload's messages in one `processMessages` call. + +- name: test_scaling_process_single_component_messages_relaxed_v10 + description: Processing n messages of one component each, under a validation config, takes about four times as long at 4n, since each message is checked against what it changes rather than the whole surface. + action: measure_growth + protocolVersion: "v1.0" + catalogPaths: + - "catalogs/basic/v1/catalog.json" + args: + validation_config: relaxed + operation: process_messages + payload: + - '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json"}}, {"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Column", "children": [' + - repeat: '"item_{i}"' + separator: ", " + - "]}]}}, " + - repeat: '{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "item_{i}", "component": "Text", "text": "Item number {i}"}]}}' + separator: ", " + - "]" + sizes: [200, 800] + max_ratio: 8 + +- name: test_scaling_process_single_component_messages_v10 + description: Processing n messages of one component each, without a validation config, takes about four times as long at 4n. + action: measure_growth + protocolVersion: "v1.0" + catalogPaths: + - "catalogs/basic/v1/catalog.json" + args: + validation_config: none + operation: process_messages + payload: + - '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json"}}, {"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Column", "children": [' + - repeat: '"item_{i}"' + separator: ", " + - "]}]}}, " + - repeat: '{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "item_{i}", "component": "Text", "text": "Item number {i}"}]}}' + separator: ", " + - "]" + sizes: [200, 800] + max_ratio: 8 diff --git a/dart/a2ui_agent/CHANGELOG.md b/dart/a2ui_agent/CHANGELOG.md index 464bc3e971..724cb6bb43 100644 --- a/dart/a2ui_agent/CHANGELOG.md +++ b/dart/a2ui_agent/CHANGELOG.md @@ -59,7 +59,9 @@ `updateComponents` and `updateDataModel` once, after it closes. - The direct JSON parser's `parseChunk` reads and checks only what each chunk adds, rather than the whole block so far: a message once it closes, - and a growing `updateComponents` for its new components. The time to + after which it is not read again, a growing `updateComponents` for its new + components, and a growing `updateDataModel` for the lists and maps its + value gains, with the value applied once it closes. The time to stream a payload grows with its size rather than with its square: streaming 6.4 KB in 16-byte chunks takes about 0.1 s rather than 1.4 s, and 105 KB about 0.3 s. A message that grows is emitted again with the diff --git a/dart/a2ui_agent/README.md b/dart/a2ui_agent/README.md index 26a15c3d8c..28f8590f7d 100644 --- a/dart/a2ui_agent/README.md +++ b/dart/a2ui_agent/README.md @@ -127,6 +127,11 @@ something v0.9 or this API does not have are skipped with the reason. [conformance/agent/multi_catalog_formats.yaml](../../conformance/agent/multi_catalog_formats.yaml) runs as written in v1.0 only. +The `scaling.yaml` suites time parsing rather than check what it returns, and +a slow or loaded machine can fail them, so their tests are tagged +`performance` and [dart_test.yaml](dart_test.yaml) skips them. Run them with +`dart test -P performance`. CI runs them in a job of their own. + The other tests cover what the suites do not: the published basic catalogs, decisions of this package, errors, and a round trip of every basic catalog example through both formats. diff --git a/dart/a2ui_agent/dart_test.yaml b/dart/a2ui_agent/dart_test.yaml new file mode 100644 index 0000000000..227d216895 --- /dev/null +++ b/dart/a2ui_agent/dart_test.yaml @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tags: + # Tests that time an operation, such as the shared scaling suites. A slow + # or loaded machine can fail them, so they are skipped unless the + # `performance` preset is selected, which runs only them, one test file at + # a time: `dart test -P performance`. + performance: + skip: "Timing test, run with `dart test -P performance`." + presets: {performance: {skip: false}} + +presets: + performance: + include_tags: performance + concurrency: 1 diff --git a/dart/a2ui_agent/lib/src/inference_formats/direct_json/json_reader.dart b/dart/a2ui_agent/lib/src/inference_formats/direct_json/json_reader.dart index 587cf318fa..7a6ec72cf3 100644 --- a/dart/a2ui_agent/lib/src/inference_formats/direct_json/json_reader.dart +++ b/dart/a2ui_agent/lib/src/inference_formats/direct_json/json_reader.dart @@ -59,8 +59,9 @@ typedef PartialMessage = ({Object? message, bool closed}); /// /// Given a [cache] that earlier reads of the same payload filled, a list or /// map that closed in one of them is taken from it rather than read again, -/// as the same object, and a list still open is read on from its last item -/// that closed. A caller must not change the lists and maps returned. +/// as the same object, and a list still open, the list of messages +/// included, is read on from its last item that closed. A caller must not +/// change the lists and maps returned. /// /// Returns no messages if [text] cannot be the start of a JSON document. List readPartialMessages( @@ -116,6 +117,10 @@ class _ReadState { /// Each list still open, by the offset of its opening bracket, with the /// items that closed and the offset after the last of them. final Map, int)> open = {}; + + /// The messages of the list of messages that closed, with the offset after + /// the last of them, or null before the list holds one. + (List, int)? messages; } String _straightenQuotes(String text) => text @@ -188,27 +193,61 @@ class _Reader { } } _i++; - final messages = []; + var messages = []; + // How many of the messages closed, and where the text after the last of + // them starts, for the next read to go on from. + var closedMessages = 0; + int resume = _i; + var afterItem = false; + if (_state?.messages case ( + final List closed, + final int at, + )) { + messages = [...closed]; + closedMessages = closed.length; + resume = _i = at; + afterItem = closed.isNotEmpty; + } + + List soFar() { + _state?.messages = ( + closedMessages == messages.length + ? messages + : messages.sublist(0, closedMessages), + resume, + ); + return messages; + } + while (true) { + if (afterItem) { + _skipSpace(); + if (_atEnd) return soFar(); + if (text[_i] == ',') { + _i++; + } else if (text[_i] != ']') { + _fail("expected ',' or ']'"); + } + } + afterItem = true; _skipSpace(); - if (_atEnd) return messages; + if (_atEnd) return soFar(); if (text[_i] == ']') { _i++; _skipSpace(); if (!_atEnd) _fail('unexpected text after the JSON value'); return messages; } + final PartialMessage message; try { - messages.add(_message()); + message = _message(); } on _Unfinished { - return messages; + return soFar(); } - _skipSpace(); - if (_atEnd) return messages; - if (text[_i] == ',') { - _i++; - } else if (text[_i] != ']') { - _fail("expected ',' or ']'"); + messages.add(message); + if (message.closed) { + closedMessages = messages.length; + resume = _i; } } } diff --git a/dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart b/dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart index eb42d1a28f..0840c74c58 100644 --- a/dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart +++ b/dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart @@ -225,6 +225,10 @@ class DirectJsonParser extends Parser { /// The messages of the current block emitted so far. final List _emitted = []; + /// How many of [_emitted] had closed when they were emitted, so that no + /// later chunk can change them. + int _settled = 0; + /// Finds the close tag of the current block as its text arrives. _BlockScanner _scanner = _BlockScanner(0); @@ -297,6 +301,7 @@ class DirectJsonParser extends Parser { _inBlock = false; _block = ''; _emitted.clear(); + _settled = 0; _scanner = _BlockScanner(0); _readCache = PartialReadCache(); _readStart = null; @@ -361,11 +366,20 @@ class DirectJsonParser extends Parser { // Built from the reader, so that catalogs it cannot read throw rather // than pass for a message that has not arrived in full. final _StreamedChecks checks = _checks ??= _StreamedChecks(_reader); - _emitChanged(parts, checks.update(envelopes), streamed: true); + // The messages emitted closed cannot have changed, so only the ones + // after them are compared. + _emitChanged( + parts, + checks.update(envelopes, from: _settled), + from: _settled, + streamed: true, + ); + _settled = checks.closedCount; } /// Emits the messages of [messages] that differ from the ones emitted at - /// the same place before. + /// the same place before. [messages] are the messages of the block from + /// the one at [from] on. /// /// The checks of a streamed block return a message that has not changed /// as the same object, so [streamed] messages are compared by identity. @@ -374,10 +388,12 @@ class DirectJsonParser extends Parser { void _emitChanged( List parts, List messages, { + int from = 0, bool streamed = false, }) { final changed = []; - for (final (int i, AgentToRendererMessage message) in messages.indexed) { + for (var i = from; i < from + messages.length; i++) { + final AgentToRendererMessage message = messages[i - from]; if (i < _emitted.length) { final AgentToRendererMessage emitted = _emitted[i]; if (identical(emitted, message) || @@ -403,7 +419,8 @@ class DirectJsonParser extends Parser { /// components, and for the last one it checked when that one has grown, /// since applying them over the ones checked before leaves the surface as /// applying the whole message would. An `updateDataModel` that grows is -/// checked again whole when it changes. +/// checked again only for the lists and maps its value gains, and applied +/// once it closes. /// /// A message that has not changed since the last chunk is returned as the /// same object. @@ -429,8 +446,8 @@ class _StreamedChecks { /// the checks, or null. AgentToRendererMessage? _growing; - /// The JSON of the message [_growing] was read from, less its components - /// for an `updateComponents`. + /// The JSON of the message [_growing] was read from, less what it grows + /// by; see [_restOf]. String? _growingJson; /// The JSON of the last component of the `updateComponents` [_growing] was @@ -446,19 +463,26 @@ class _StreamedChecks { /// compile. bool _stalled = false; + /// How many messages have closed and passed the checks. + int get closedCount => _closed.length; + /// Checks what [envelopes], the messages of the block read so far, add to - /// the ones checked before, and returns the messages that may be emitted. + /// the ones checked before, and returns the messages that may be emitted, + /// from the one at [from] on. /// /// Throws what [DirectJsonParser.compile] throws for a v1.0 /// `updateComponents` whose components have closed. - List update(List envelopes) { + List update( + List envelopes, { + int from = 0, + }) { for (int i = _closed.length; !_stalled && i < envelopes.length; i++) { final (:Object? message, :bool closed) = envelopes[i]; // Held until it closes, and nothing after it is emitted ahead of it. if (!closed && !_growsInPlace(message)) break; final AgentToRendererMessage read; try { - read = _check(message); + read = _check(message, closed: closed); } on A2uiError { if (_reader.version.isAtLeast(A2uiProtocolVersion.v1_0) && _hasClosedComponents(message)) { @@ -486,23 +510,30 @@ class _StreamedChecks { _growingLast = null; _growingIds = {}; } - return [..._closed, ?_growing]; + return [..._closed.skip(from), ?_growing]; } /// Reads and checks [envelope], the message after [_closed], over the - /// messages checked before, and returns it read. + /// messages checked before, and returns it read. [closed] says whether + /// its JSON value has closed. /// /// A rejected message leaves the checks as they were, so the next chunk /// can check it again. - AgentToRendererMessage _check(Object? envelope) { + AgentToRendererMessage _check(Object? envelope, {required bool closed}) { final AgentToRendererMessage? growing = _growing; - if (growing is UpdateComponentsMessage) { - final String? rest = _restOf(envelope); - if (rest != null && rest == _growingJson) { + final String? rest = growing == null ? null : _restOf(envelope); + switch (growing) { + case UpdateComponentsMessage() when rest != null && rest == _growingJson: return _checkAdded(growing, envelope! as Map); - } - } else if (growing != null && jsonEncode(envelope) == _growingJson) { - return growing; + case UpdateDataModelMessage() when rest != null && rest == _growingJson: + return _checkGrown( + growing, + envelope! as Map, + closed: closed, + ); + case final AgentToRendererMessage growing + when rest == null && jsonEncode(envelope) == _growingJson: + return growing; } final AgentToRendererMessage read = _reader.read(envelope, _surfaces); _checker.add(read); @@ -549,6 +580,36 @@ class _StreamedChecks { return message; } + /// Checks the value of [envelope], the `updateDataModel` [growing] was + /// read from on an earlier chunk, and returns the message it makes now. + /// + /// Only the value can have changed, and of what a renderer checks only the + /// depth of the value depends on it, so only the lists and maps the value + /// has gained are checked. The value is applied once the message has + /// [closed], since no message after it can be checked before then. + AgentToRendererMessage _checkGrown( + UpdateDataModelMessage growing, + Map envelope, { + required bool closed, + }) { + final Object? value = + (envelope['updateDataModel']! as Map)['value']; + final message = _sameJson(value, growing.value) + ? growing + : UpdateDataModelMessage( + version: growing.version, + surfaceId: growing.surfaceId, + path: growing.path, + value: value, + ); + if (closed) { + _checker.add(message); + } else if (!identical(message, growing)) { + _checker.checkGrown(message, grownFrom: growing); + } + return message; + } + /// Releases the surfaces the messages checked so far create. void dispose() => _checker.dispose(); } @@ -568,15 +629,50 @@ Map? _bodyOf(Object? envelope) => switch (envelope) { _ => null, }; -/// The JSON of [envelope], an `updateComponents` with a list of components, -/// less its components, or null for any other message. +/// The JSON of [envelope] less what it grows by: the components of an +/// `updateComponents` with a list of components, or the value of an +/// `updateDataModel` that has one. Null for any other message. String? _restOf(Object? envelope) { - final Map? body = _bodyOf(envelope); - if (body == null) return null; - return jsonEncode({ - ...envelope! as Map, - 'updateComponents': {...body}..remove('components'), - }); + String without(Map message, String type, String field) => + jsonEncode({ + ...message, + type: {...message[type]! as Map}..remove(field), + }); + + return switch (envelope) { + final Map message when _bodyOf(message) != null => without( + message, + 'updateComponents', + 'components', + ), + final Map message && {'updateDataModel': {'value': _}} => + without(message, 'updateDataModel', 'value'), + _ => null, + }; +} + +/// Whether [a] and [b] are the same JSON value. A list or map that is the +/// same object in both, as a read through a cache returns one that closed, +/// is not read. +bool _sameJson(Object? a, Object? b) { + if (identical(a, b)) return true; + if (a is List && b is List) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (!_sameJson(a[i], b[i])) return false; + } + return true; + } + if (a is Map && b is Map) { + if (a.length != b.length) return false; + for (final MapEntry entry in a.entries) { + if (!b.containsKey(entry.key) || !_sameJson(entry.value, b[entry.key])) { + return false; + } + } + return true; + } + return a == b; } /// Whether [envelope] is a message a renderer applies again as it grows, so diff --git a/dart/a2ui_agent/lib/src/utils/payload_checker.dart b/dart/a2ui_agent/lib/src/utils/payload_checker.dart index aa02ad562a..713fa96e31 100644 --- a/dart/a2ui_agent/lib/src/utils/payload_checker.dart +++ b/dart/a2ui_agent/lib/src/utils/payload_checker.dart @@ -120,6 +120,22 @@ class PayloadChecker { _processor.processMessages([message]); } + /// Checks [message], an `updateDataModel` that has grown from [grownFrom], + /// without applying it. [grownFrom] passed [add] or this check before. + /// + /// Only the depth of the value depends on the value, so only the lists and + /// maps it has gained are checked, and only for a surface [add] applies + /// messages to. + /// + /// Throws [A2uiRecursionError] if the value nests too deep. + void checkGrown( + UpdateDataModelMessage message, { + required UpdateDataModelMessage grownFrom, + }) { + if (_processor.model.getSurface(message.surfaceId) == null) return; + checkPathsAndRecursion(message, grownFrom: grownFrom); + } + /// Releases the surfaces. void dispose() => _processor.model.dispose(); } diff --git a/dart/a2ui_agent/test/conformance/scaling_conformance_test.dart b/dart/a2ui_agent/test/conformance/scaling_conformance_test.dart new file mode 100644 index 0000000000..67a64e22da --- /dev/null +++ b/dart/a2ui_agent/test/conformance/scaling_conformance_test.dart @@ -0,0 +1,175 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Runs the shared `scaling.yaml` suites of `conformance/agent/`, which time +/// parsing a payload written at two sizes, or streaming a payload against +/// compiling it, and compare the median times. +/// +/// The tests are tagged `performance`, which `dart_test.yaml` skips unless +/// the `performance` preset is selected, as in `dart test -P performance`. +/// A case is written in the version its catalog declares, so nothing is +/// lowered. +@Tags(['performance']) +library; + +import 'dart:math'; + +import 'package:a2ui_agent/a2ui_agent.dart'; +import 'package:a2ui_core/a2ui_core.dart'; +import 'package:test/test.dart'; + +import 'suites.dart'; + +void main() { + for (final path in const [ + 'agent/direct_json/scaling.yaml', + 'agent/express/scaling.yaml', + ]) { + group('conformance $path', () { + final List> cases = loadSuite(path); + test('suite is not empty', () => expect(cases, isNotEmpty)); + for (final testCase in cases) { + test(testCase['name']! as String, () => _runCase(testCase)); + } + }); + } +} + +void _runCase(Map testCase) { + final InferenceFormat format = _format( + testCase['args']! as Map, + ); + final operation = testCase['operation']! as String; + final chunkSize = testCase['chunk_size'] as int?; + final maxRatio = testCase['max_ratio']! as num; + switch (testCase['action']) { + case 'measure_growth': + final [int smaller, int larger] = (testCase['sizes']! as List) + .cast(); + expectTimeRatio( + _call(format, operation, writePayload(testCase, larger), chunkSize), + _call(format, operation, writePayload(testCase, smaller), chunkSize), + maxRatio, + what: '$operation at $larger against $smaller', + ); + case 'measure_overhead': + final String payload = writePayload(testCase, testCase['size']! as int); + expectTimeRatio( + _call(format, operation, payload, chunkSize), + _call(format, testCase['baseline']! as String, payload, chunkSize), + maxRatio, + what: '$operation against ${testCase['baseline']}', + ); + default: + throw UnsupportedError('Unknown action ${testCase['action']}'); + } +} + +InferenceFormat _format(Map args) { + final List catalogs = caseCatalogs(args, lower: false); + return switch (args['format']) { + 'direct_json' => DirectJsonFormatFactory( + progressiveKeys: { + ...?(args['progressive_keys'] as List?)?.cast(), + }, + ).createFormat(catalogs), + 'express' => const ExpressFormatFactory().createFormat(catalogs), + final Object? name => throw UnsupportedError('Unknown format $name'), + }; +} + +/// The call [operation] names, made on [payload] with a new parser of +/// [format]. `parse_chunk` hands the parser the response holding [payload] +/// in chunks of [chunkSize] characters. +void Function() _call( + InferenceFormat format, + String operation, + String payload, + int? chunkSize, +) { + switch (operation) { + case 'parse_chunk': + final response = '$payload'; + final int size = chunkSize!; + final List chunks = [ + for (var i = 0; i < response.length; i += size) + response.substring(i, min(i + size, response.length)), + ]; + return () { + final Parser parser = format.createParser(); + for (final chunk in chunks) { + parser.parseChunk(chunk); + } + }; + case 'compile': + return () => format.createParser().compile(payload); + default: + throw UnsupportedError('Unknown operation $operation'); + } +} + +/// The payload of a timing case written at [size]; see `PayloadTemplate` in +/// `conformance_schema.json`. +String writePayload(Map testCase, int size) => [ + for (final Object? segment in testCase['payload']! as List) + switch (segment) { + final String text => text, + {'repeat': final String text} => [ + for (var i = 0; i < size; i++) text.replaceAll('{i}', '$i'), + ].join((segment as Map)['separator'] as String? ?? ''), + _ => throw ArgumentError.value(segment, 'segment'), + }, +].join(); + +/// How many times each call is timed. +const int _runs = 5; + +/// Expects [measured] to take at most [maxRatio] times as long as +/// [baseline], comparing the median of [_runs] timed runs of each, made in +/// turn after one untimed run of each. +/// +/// [what] names the two calls in the failure message. +void expectTimeRatio( + void Function() measured, + void Function() baseline, + num maxRatio, { + required String what, +}) { + baseline(); + measured(); + final baselineTimes = []; + final measuredTimes = []; + for (var run = 0; run < _runs; run++) { + baselineTimes.add(_time(baseline)); + measuredTimes.add(_time(measured)); + } + final int measuredMedian = _median(measuredTimes); + final int baselineMedian = _median(baselineTimes); + expect( + measuredMedian / baselineMedian, + lessThanOrEqualTo(maxRatio), + reason: + 'Median time of $what: $measuredMedian µs against ' + '$baselineMedian µs.', + ); +} + +/// The time [body] takes, in microseconds. +int _time(void Function() body) { + final watch = Stopwatch()..start(); + body(); + return watch.elapsedMicroseconds; +} + +int _median(List times) => (times.toList()..sort())[times.length ~/ 2]; diff --git a/dart/a2ui_agent/test/direct_json_test.dart b/dart/a2ui_agent/test/direct_json_test.dart index 4475824b8c..80b09e2258 100644 --- a/dart/a2ui_agent/test/direct_json_test.dart +++ b/dart/a2ui_agent/test/direct_json_test.dart @@ -497,6 +497,47 @@ void main() { ); }); + test('holds back a growing data model once its value nests too deep', () { + final DirectJsonParser reader = parser(); + expect( + reader.parseChunk( + '[${create('s')}, {"version": "v0.9", "updateDataModel": ' + '{"surfaceId": "s", "value": {"a": [1, ', + ), + isNotEmpty, + ); + expect(reader.parseChunk('[' * 60), isEmpty); + expect( + () => reader.parseChunk('${']' * 61}}}}]'), + throwsError(), + ); + }); + + test('checks a later message against the value a data model closed ' + 'with', () { + final DirectJsonParser reader = parser(); + expect( + reader.parseChunk( + '[${create('s')}, {"version": "v0.9", "updateDataModel": ' + '{"surfaceId": "s", "path": "/items", "value": [', + ), + isNotEmpty, + ); + expect(reader.parseChunk('1, 2]}}, '), isNotEmpty); + // The first item is a number, which has no property to set. + expect( + reader.parseChunk( + '{"version": "v0.9", "updateDataModel": {"surfaceId": "s", ' + '"path": "/items/0/name", "value": "x"}}', + ), + isEmpty, + ); + expect( + () => reader.parseChunk(']'), + throwsError(), + ); + }); + test('reads a block wrapped in a markdown fence', () { expect( stream(['\n``', '`json\n[${create('s')}]\n```']), diff --git a/dart/a2ui_core/CHANGELOG.md b/dart/a2ui_core/CHANGELOG.md index 2a7b9ead9f..661b5ad486 100644 --- a/dart/a2ui_core/CHANGELOG.md +++ b/dart/a2ui_core/CHANGELOG.md @@ -45,6 +45,12 @@ each batch of components, for an agent that checks one message's components in several batches as they stream. Its `earlier` argument takes the ids of the earlier batches. +- Exported `checkPathsAndRecursion`, the check of data-model paths and + nesting depth that `MessageProcessor` runs on each message under a + validation config. Its new `grownFrom` argument takes an earlier form of + the message that passed the check, and a list or map the two hold at the + same place is not checked again, so a message that grows as it streams is + checked for what it gained. - A `MessageProcessor` whose `validationConfig` allows dangling references, and orphan components or a missing root, as an agent checking a payload that is still arriving does, checks each `updateComponents` without diff --git a/dart/a2ui_core/README.md b/dart/a2ui_core/README.md index d80227743a..0746577a2c 100644 --- a/dart/a2ui_core/README.md +++ b/dart/a2ui_core/README.md @@ -130,3 +130,5 @@ dart analyze --fatal-infos dart format --output=none --set-exit-if-changed . dart test ``` + +`dart test` skips the tests tagged `performance`, which time `processMessages` on surfaces of two sizes for `conformance/core/message_processor_scaling.yaml`, since a slow or loaded machine can fail them. Run them with `dart test -P performance`. CI runs them in a job of their own. diff --git a/dart/a2ui_core/dart_test.yaml b/dart/a2ui_core/dart_test.yaml new file mode 100644 index 0000000000..227d216895 --- /dev/null +++ b/dart/a2ui_core/dart_test.yaml @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tags: + # Tests that time an operation, such as the shared scaling suites. A slow + # or loaded machine can fail them, so they are skipped unless the + # `performance` preset is selected, which runs only them, one test file at + # a time: `dart test -P performance`. + performance: + skip: "Timing test, run with `dart test -P performance`." + presets: {performance: {skip: false}} + +presets: + performance: + include_tags: performance + concurrency: 1 diff --git a/dart/a2ui_core/lib/a2ui_core.dart b/dart/a2ui_core/lib/a2ui_core.dart index aaeb9a073f..cbbddd98de 100644 --- a/dart/a2ui_core/lib/a2ui_core.dart +++ b/dart/a2ui_core/lib/a2ui_core.dart @@ -61,7 +61,8 @@ export 'src/resolution/node_resolver.dart'; export 'src/resolution/resolved_binding.dart'; export 'src/rpc/rpc_handler.dart'; // Payload validation. -export 'src/validation/component_graph.dart' show checkDuplicateComponentIds; +export 'src/validation/component_graph.dart' + show checkDuplicateComponentIds, checkPathsAndRecursion; export 'src/validation/schema_resolution.dart' show inlineLocalRefs, resolveSchemaRefs; export 'src/validation/validation_config.dart'; diff --git a/dart/a2ui_core/lib/src/validation/component_graph.dart b/dart/a2ui_core/lib/src/validation/component_graph.dart index 49d80e2e2f..38934e2032 100644 --- a/dart/a2ui_core/lib/src/validation/component_graph.dart +++ b/dart/a2ui_core/lib/src/validation/component_graph.dart @@ -360,15 +360,27 @@ bool _isFunctionCall( /// Checks data-model paths and nesting depth anywhere inside a message body. /// +/// [grownFrom], if given, is an earlier form of [data] that passed this +/// check, such as a message that a stream has since extended. A list or map +/// in [data] that is identical to the one at the same place in [grownFrom] +/// is not checked again, so checking a message that grew costs what it +/// gained. Neither may have been changed since [grownFrom] was checked. +/// /// Throws [A2uiValidationError] for a malformed path and [A2uiRecursionError] /// when nesting or chained function calls run past their caps. -void checkPathsAndRecursion(Object? data, {A2uiProtocolVersion? version}) { +void checkPathsAndRecursion( + Object? data, { + A2uiProtocolVersion? version, + Object? grownFrom, +}) { final bool usesDirectives = version != null ? version.isAtLeast(A2uiProtocolVersion.v1_0) : _inferUsesDirectives(data); + // [before] is what [grownFrom] holds at the place of [node], or null. void traverse( Object? node, + Object? before, int depth, int callDepth, { required bool inDataValue, @@ -380,17 +392,32 @@ void checkPathsAndRecursion(Object? data, {A2uiProtocolVersion? version}) { } if (node is List) { - for (final Object? item in node) { - traverse(item, depth + 1, callDepth, inDataValue: inDataValue); + if (identical(node, before)) return; + final List? items = before is List ? before : null; + for (var i = 0; i < node.length; i++) { + traverse( + node[i], + items != null && i < items.length ? items[i] : null, + depth + 1, + callDepth, + inDataValue: inDataValue, + ); } return; } - if (node is! Map) return; + if (node is! Map || identical(node, before)) return; + final Map? entries = before is Map ? before : null; if (inDataValue) { - for (final Object? value in node.values) { - traverse(value, depth + 1, callDepth, inDataValue: true); + for (final MapEntry entry in node.entries) { + traverse( + entry.value, + entries?[entry.key], + depth + 1, + callDepth, + inDataValue: true, + ); } return; } @@ -405,9 +432,11 @@ void checkPathsAndRecursion(Object? data, {A2uiProtocolVersion? version}) { "Invalid path syntax: '$udmPath'", ); } + final Object? udmBefore = entries?['updateDataModel']; for (final MapEntry entry in udm.entries) { traverse( entry.value, + udmBefore is Map ? udmBefore[entry.key] : null, depth + 2, callDepth, inDataValue: entry.key == 'value', @@ -437,6 +466,7 @@ void checkPathsAndRecursion(Object? data, {A2uiProtocolVersion? version}) { for (final MapEntry entry in object.entries) { traverse( entry.value, + entries?[entry.key], depth + 1, entry.key == 'args' ? callDepth + 1 : callDepth, inDataValue: false, @@ -445,8 +475,14 @@ void checkPathsAndRecursion(Object? data, {A2uiProtocolVersion? version}) { return; } - for (final Object? value in object.values) { - traverse(value, depth + 1, callDepth, inDataValue: false); + for (final MapEntry entry in object.entries) { + traverse( + entry.value, + entries?[entry.key], + depth + 1, + callDepth, + inDataValue: false, + ); } } @@ -457,16 +493,28 @@ void checkPathsAndRecursion(Object? data, {A2uiProtocolVersion? version}) { "Invalid path syntax: '$udmPath'", ); } - traverse(data.value, 2, 0, inDataValue: true); + traverse( + data.value, + grownFrom is UpdateDataModelMessage ? grownFrom.value : null, + 2, + 0, + inDataValue: true, + ); return; } if (data is AgentToRendererMessage) { - traverse(data.toJson(), 0, 0, inDataValue: false); + traverse( + data.toJson(), + grownFrom is AgentToRendererMessage ? grownFrom.toJson() : grownFrom, + 0, + 0, + inDataValue: false, + ); return; } - traverse(data, 0, 0, inDataValue: false); + traverse(data, grownFrom, 0, 0, inDataValue: false); } bool _inferUsesDirectives(Object? data) { diff --git a/dart/a2ui_core/test/component_graph_test.dart b/dart/a2ui_core/test/component_graph_test.dart index 8c1f26b89d..e62a492fc1 100644 --- a/dart/a2ui_core/test/component_graph_test.dart +++ b/dart/a2ui_core/test/component_graph_test.dart @@ -301,5 +301,82 @@ void main() { returnsNormally, ); }); + + group('with grownFrom', () { + /// A list nested past the depth limit, which fails the check wherever + /// it is. + List tooDeep() { + var value = []; + for (var i = 0; i < maxComponentDepth; i++) { + value = [value]; + } + return value; + } + + UpdateDataModelMessage items(List value) => + UpdateDataModelMessage( + version: 'v1.0', + surfaceId: 's1', + path: '/items', + value: value, + ); + + test('skips a list identical to the one at its place', () { + final List deep = tooDeep(); + expect( + () => checkPathsAndRecursion(items([deep, 1])), + throwsA(isA()), + ); + // grownFrom would not pass either, which shows that the list it + // shares is not read again. + expect( + () => checkPathsAndRecursion( + items([deep, 1]), + grownFrom: items([deep]), + ), + returnsNormally, + ); + }); + + test('skips the same way in a message written as JSON', () { + final List deep = tooDeep(); + Map json(List value) => { + 'version': 'v1.0', + 'updateDataModel': { + 'surfaceId': 's1', + 'path': '/items', + 'value': value, + }, + }; + expect( + () => checkPathsAndRecursion( + json([deep, 1]), + grownFrom: json([deep]), + ), + returnsNormally, + ); + }); + + test('checks what the message gained', () { + expect( + () => checkPathsAndRecursion( + items([1, tooDeep()]), + grownFrom: items([1]), + ), + throwsA(isA()), + ); + }); + + test('checks a list that grownFrom holds at another place', () { + final List deep = tooDeep(); + expect( + () => checkPathsAndRecursion( + items([1, deep]), + grownFrom: items([deep]), + ), + throwsA(isA()), + ); + }); + }); }); } diff --git a/dart/a2ui_core/test/conformance/message_processor_scaling_conformance_test.dart b/dart/a2ui_core/test/conformance/message_processor_scaling_conformance_test.dart new file mode 100644 index 0000000000..dde1212331 --- /dev/null +++ b/dart/a2ui_core/test/conformance/message_processor_scaling_conformance_test.dart @@ -0,0 +1,133 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Runs the shared `core/message_processor_scaling.yaml` suite, which times +/// [MessageProcessor.processMessages] on a payload written at two sizes and +/// compares the median times. +/// +/// The tests are tagged `performance`, which `dart_test.yaml` skips unless +/// the `performance` preset is selected, as in `dart test -P performance`. +@Tags(['performance']) +library; + +import 'dart:convert'; + +import 'package:a2ui_core/a2ui_core.dart'; +import 'package:test/test.dart'; + +import '../support/renderer_catalog.dart'; +import 'conformance_harness.dart'; + +void main() { + const suite = 'core/message_processor_scaling.yaml'; + final List cases = loadConformanceSuite(suite); + group('conformance $suite', () { + test('suite is not empty', () => expect(cases, isNotEmpty)); + for (final testCase in cases) { + test(testCase['name']! as String, () => _runCase(testCase)); + } + }); +} + +void _runCase(ConformanceTestCase testCase) { + expect(testCase['action'], 'measure_growth'); + expect(testCase['operation'], 'process_messages'); + final List> catalogs = [ + for (final Object? path in testCase['catalogPaths']! as List) + rendererCatalog( + readConformanceJson(path! as String), + protocolVersion: testCase['protocolVersion']! as String, + ), + ]; + final args = testCase['args']! as Map; + final ValidationConfig? config = switch (args['validation_config']) { + 'none' => null, + 'relaxed' => ValidationConfig.relaxed, + final Object? name => throw UnsupportedError( + 'Unknown validation config $name', + ), + }; + + /// Processes the payload at [size] on a new processor. + void Function() process(int size) { + final Object? messages = jsonDecode(writePayload(testCase, size)); + return () => MessageProcessor( + catalogs: catalogs, + validationConfig: config, + ).processMessages(messages); + } + + final [int smaller, int larger] = + (testCase['sizes']! as List).cast(); + expectTimeRatio( + process(larger), + process(smaller), + testCase['max_ratio']! as num, + what: 'processMessages at $larger against $smaller', + ); +} + +/// The payload of a timing case written at [size]; see `PayloadTemplate` in +/// `conformance_schema.json`. +String writePayload(ConformanceTestCase testCase, int size) => [ + for (final Object? segment in testCase['payload']! as List) + switch (segment) { + final String text => text, + {'repeat': final String text} => [ + for (var i = 0; i < size; i++) text.replaceAll('{i}', '$i'), + ].join((segment as Map)['separator'] as String? ?? ''), + _ => throw ArgumentError.value(segment, 'segment'), + }, + ].join(); + +/// How many times each call is timed. +const int _runs = 5; + +/// Expects [measured] to take at most [maxRatio] times as long as +/// [baseline], comparing the median of [_runs] timed runs of each, made in +/// turn after one untimed run of each. +/// +/// [what] names the two calls in the failure message. +void expectTimeRatio( + void Function() measured, + void Function() baseline, + num maxRatio, { + required String what, +}) { + baseline(); + measured(); + final baselineTimes = []; + final measuredTimes = []; + for (var run = 0; run < _runs; run++) { + baselineTimes.add(_time(baseline)); + measuredTimes.add(_time(measured)); + } + final int measuredMedian = _median(measuredTimes); + final int baselineMedian = _median(baselineTimes); + expect( + measuredMedian / baselineMedian, + lessThanOrEqualTo(maxRatio), + reason: 'Median time of $what: $measuredMedian µs against ' + '$baselineMedian µs.', + ); +} + +/// The time [body] takes, in microseconds. +int _time(void Function() body) { + final watch = Stopwatch()..start(); + body(); + return watch.elapsedMicroseconds; +} + +int _median(List times) => (times.toList()..sort())[times.length ~/ 2]; diff --git a/python/a2ui_core/tests/test_conformance.py b/python/a2ui_core/tests/test_conformance.py index caa18d2717..b1bec867ef 100644 --- a/python/a2ui_core/tests/test_conformance.py +++ b/python/a2ui_core/tests/test_conformance.py @@ -84,6 +84,11 @@ " not have access to. Pending v1.0 catalogs for those renderers." ), "core/node_resolution.yaml": "The Python core has no node resolution layer.", + "core/message_processor_scaling.yaml": ( + "The cases time processing and depend on the machine, so they need a" + " harness that runs them apart from the other tests, which the Python" + " core does not have." + ), } # Root core conformance directory resolution @@ -618,6 +623,10 @@ def test_conformance_suite(test_id: str, rel_path: str, case: dict[str, Any]) -> validate_evaluate_function_case(case) elif action == "dispatch_action": validate_dispatch_action_case(case) + elif ( + action == "measure_growth" and rel_path == "core/message_processor_scaling.yaml" + ): + pytest.skip(UNRUNNABLE_SUITES[rel_path]) else: pytest.fail( f"Action '{action}' has no handler in the core Python harness." diff --git a/typescript/core/web_core/tests/conformance/conformance_test.mjs b/typescript/core/web_core/tests/conformance/conformance_test.mjs index 2725227b6a..92c2947c13 100644 --- a/typescript/core/web_core/tests/conformance/conformance_test.mjs +++ b/typescript/core/web_core/tests/conformance/conformance_test.mjs @@ -271,6 +271,11 @@ const UNIMPLEMENTED_ACTIONS = new Map([ 'web_core does not generate the agent_to_renderer schema from its own models', ], ['validate_common_type', 'web_core has no per-definition validators for the common types'], + [ + 'measure_growth', + 'the timing cases depend on the machine and need a runner that keeps them apart from the other tests, which web_core does not have', + ], + ['measure_overhead', 'web_core has no streaming inference parser'], ]); function findYamlFiles(dir) { From e7dc79dc5e60d80840f01ad02aeeb950ac3c3b4b Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Mon, 19 Oct 2026 07:09:02 -0700 Subject: [PATCH 2/7] Update json_reader.dart --- .../src/inference_formats/direct_json/json_reader.dart | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dart/a2ui_agent/lib/src/inference_formats/direct_json/json_reader.dart b/dart/a2ui_agent/lib/src/inference_formats/direct_json/json_reader.dart index 7a6ec72cf3..1e8c33ca6d 100644 --- a/dart/a2ui_agent/lib/src/inference_formats/direct_json/json_reader.dart +++ b/dart/a2ui_agent/lib/src/inference_formats/direct_json/json_reader.dart @@ -109,6 +109,11 @@ class PartialReadCache { final _ReadState _straight = _ReadState(); } +/// What the reads of a payload keep for the next read. +/// +/// An offset is the index of a character in the text read, counting from 0 +/// at its first character, so the offset after a closing bracket is the +/// index of the character that follows it. class _ReadState { /// Each list or map that closed, by the offset of its opening bracket, /// with the offset after its closing one. @@ -118,8 +123,9 @@ class _ReadState { /// items that closed and the offset after the last of them. final Map, int)> open = {}; - /// The messages of the list of messages that closed, with the offset after - /// the last of them, or null before the list holds one. + /// For a payload written as a list of messages, the messages whose JSON + /// object has closed, and the offset after the last of them, where the + /// next read goes on. Null until a read stops inside the list. (List, int)? messages; } From ab5c2af296f85dff990dbbce40f1d025db972a48 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Mon, 19 Oct 2026 07:12:47 -0700 Subject: [PATCH 3/7] Update conformance_schema.json --- conformance/conformance_schema.json | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/conformance/conformance_schema.json b/conformance/conformance_schema.json index fb2b962c63..20faaa367f 100644 --- a/conformance/conformance_schema.json +++ b/conformance/conformance_schema.json @@ -1392,14 +1392,28 @@ "baseline": {"const": "compile"}, "chunk_size": {"type": "integer", "minimum": 1}, "payload": {"$ref": "#/$defs/PayloadTemplate"}, - "size": {"type": "integer", "minimum": 1, "description": "The size n the payload is written at."}, + "size": { + "type": "integer", + "minimum": 1, + "description": "The size n the payload is written at." + }, "max_ratio": { "type": "number", "exclusiveMinimum": 1, "description": "The largest the median time of `operation` may be, as a multiple of the median time of `baseline`." } }, - "required": ["name", "action", "args", "operation", "baseline", "chunk_size", "payload", "size", "max_ratio"], + "required": [ + "name", + "action", + "args", + "operation", + "baseline", + "chunk_size", + "payload", + "size", + "max_ratio" + ], "additionalProperties": false }, "CreateFormatTest": { From b4d7fe6b2198d980097821f3c4b3184fcb215707 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Mon, 19 Oct 2026 07:47:16 -0700 Subject: [PATCH 4/7] - --- conformance/agent/direct_json/scaling.yaml | 26 ++++---- conformance/agent/express/scaling.yaml | 11 ++-- .../core/message_processor_scaling.yaml | 13 ++-- .../inference_formats/direct_json/parser.dart | 63 +++++++++++-------- 4 files changed, 64 insertions(+), 49 deletions(-) diff --git a/conformance/agent/direct_json/scaling.yaml b/conformance/agent/direct_json/scaling.yaml index 3bca57af6f..7ef80bd504 100644 --- a/conformance/agent/direct_json/scaling.yaml +++ b/conformance/agent/direct_json/scaling.yaml @@ -20,19 +20,21 @@ # while the time it takes to stream a payload grows with the square of the # payload's size. # -# A `measure_growth` case times one operation on a payload written at two -# sizes, n and 4n, and fails when the larger takes more than `max_ratio` times -# as long as the smaller. A cost that grows in proportion to the payload takes -# about 4 times as long at 4n, and one that grows with its square about 16 -# times, so a limit of 8 tells the two apart with room for noise. CI machines -# differ in speed and load too much for a limit in milliseconds, but two times -# taken on the same machine moments apart can be compared. +# A `measure_growth` case times one operation on a payload written at the two +# `sizes`, n and 4n. Its `max_ratio` is the largest value allowed for the time +# at 4n divided by the time at n, and the case fails above it. A cost that +# grows in proportion to the payload gives a ratio of about 4, and one that +# grows with its square a ratio of about 16, so a `max_ratio` of 8 tells the +# two apart with room for noise. CI machines differ in speed and load too much +# for a limit in milliseconds, but two times taken on the same machine moments +# apart can be compared. # -# `measure_overhead` times streaming a payload against compiling it whole, and -# fails when streaming takes more than `max_ratio` times as long. Streaming -# reads what each chunk adds and checks what it changes, so it does more work -# than compiling, but that work should stay a fixed multiple of it. The limit -# is loose and catches a parser that does far more on each chunk than the chunk +# A `measure_overhead` case times streaming a payload against compiling it +# whole, at one `size`. Its `max_ratio` is the largest value allowed for the +# time to stream divided by the time to compile. Streaming reads what each +# chunk adds and checks what it changes, so it does more work than compiling, +# but that work should stay a fixed multiple of it. A `max_ratio` of 10 is +# loose, and catches a parser that does far more on each chunk than the chunk # needs. # # `payload` is a template, written out at each size as `PayloadTemplate` in diff --git a/conformance/agent/express/scaling.yaml b/conformance/agent/express/scaling.yaml index 6dc4838648..24ec176594 100644 --- a/conformance/agent/express/scaling.yaml +++ b/conformance/agent/express/scaling.yaml @@ -14,11 +14,12 @@ # How the time to compile an Express block grows with the block. # -# A `measure_growth` case times `compile` on a block written at two sizes, n -# and 4n, and fails when the larger takes more than `max_ratio` times as long -# as the smaller. `agent/direct_json/scaling.yaml` explains the measure, how a -# harness times a case, and why the cases run apart from the other tests. The -# Express parser does not stream, so there is nothing else to time. +# A `measure_growth` case times `compile` on a block written at two sizes, n and +# 4n. Its `max_ratio` is the largest value allowed for the time at 4n divided by +# the time at n. `agent/direct_json/scaling.yaml` explains the measure and the +# limit of 8, how a harness times a case, and why the cases run apart from the +# other tests. The Express parser does not stream, so there is nothing else to +# time. - name: test_scaling_compile_express_v10 description: Compiling a block of n components takes about four times as long at 4n. diff --git a/conformance/core/message_processor_scaling.yaml b/conformance/core/message_processor_scaling.yaml index 2a4ea662e3..353c84dd48 100644 --- a/conformance/core/message_processor_scaling.yaml +++ b/conformance/core/message_processor_scaling.yaml @@ -19,12 +19,13 @@ # with the square of the surface's size when the surface arrives in many small # messages, as it does from an agent that streams. # -# A `measure_growth` case processes a payload written at two sizes, n and 4n, -# and fails when the larger takes more than `max_ratio` times as long as the -# smaller. `agent/direct_json/scaling.yaml` explains the measure, how a harness -# times a case, and why the cases run apart from the other tests. Each timed run -# creates a processor holding the catalogs of `catalogPaths` and hands it the -# payload's messages in one `processMessages` call. +# A `measure_growth` case processes a payload written at two sizes, n and 4n. +# Its `max_ratio` is the largest value allowed for the time at 4n divided by the +# time at n. `agent/direct_json/scaling.yaml` explains the measure and the limit +# of 8, how a harness times a case, and why the cases run apart from the other +# tests. Each timed run creates a processor holding the catalogs of +# `catalogPaths` and hands it the payload's messages in one `processMessages` +# call. - name: test_scaling_process_single_component_messages_relaxed_v10 description: Processing n messages of one component each, under a validation config, takes about four times as long at 4n, since each message is checked against what it changes rather than the whole surface. diff --git a/dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart b/dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart index 0840c74c58..ff3791b6ec 100644 --- a/dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart +++ b/dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart @@ -522,16 +522,20 @@ class _StreamedChecks { AgentToRendererMessage _check(Object? envelope, {required bool closed}) { final AgentToRendererMessage? growing = _growing; final String? rest = growing == null ? null : _restOf(envelope); - switch (growing) { - case UpdateComponentsMessage() when rest != null && rest == _growingJson: - return _checkAdded(growing, envelope! as Map); - case UpdateDataModelMessage() when rest != null && rest == _growingJson: - return _checkGrown( - growing, - envelope! as Map, - closed: closed, - ); - case final AgentToRendererMessage growing + switch ((growing, envelope)) { + case ( + final UpdateComponentsMessage growing, + final Map message, + ) + when rest != null && rest == _growingJson: + return _checkAdded(growing, message); + case ( + final UpdateDataModelMessage growing, + final Map message, + ) + when rest != null && rest == _growingJson: + return _checkGrown(growing, message, closed: closed); + case (final AgentToRendererMessage growing, _) when rest == null && jsonEncode(envelope) == _growingJson: return growing; } @@ -592,8 +596,7 @@ class _StreamedChecks { Map envelope, { required bool closed, }) { - final Object? value = - (envelope['updateDataModel']! as Map)['value']; + final Object? value = _dataModelBodyOf(envelope)!['value']; final message = _sameJson(value, growing.value) ? growing : UpdateDataModelMessage( @@ -629,26 +632,34 @@ Map? _bodyOf(Object? envelope) => switch (envelope) { _ => null, }; +/// The body of [envelope] if it is an `updateDataModel` with a value, or +/// null. +Map? _dataModelBodyOf(Object? envelope) => switch (envelope) { + {'updateDataModel': final Map body} + when body.containsKey('value') => + body, + _ => null, +}; + /// The JSON of [envelope] less what it grows by: the components of an /// `updateComponents` with a list of components, or the value of an -/// `updateDataModel` that has one. Null for any other message. +/// `updateDataModel` that has one. Null for any other message, and for an +/// envelope that is not a `Map`. String? _restOf(Object? envelope) { - String without(Map message, String type, String field) => + if (envelope is! Map) return null; + String without(String type, Map body, String field) => jsonEncode({ - ...message, - type: {...message[type]! as Map}..remove(field), + ...envelope, + type: {...body}..remove(field), }); - return switch (envelope) { - final Map message when _bodyOf(message) != null => without( - message, - 'updateComponents', - 'components', - ), - final Map message && {'updateDataModel': {'value': _}} => - without(message, 'updateDataModel', 'value'), - _ => null, - }; + if (_bodyOf(envelope) case final Map body) { + return without('updateComponents', body, 'components'); + } + if (_dataModelBodyOf(envelope) case final Map body) { + return without('updateDataModel', body, 'value'); + } + return null; } /// Whether [a] and [b] are the same JSON value. A list or map that is the From b664d4a9ed25d8c2c7a6ee4fcd6f501ae6217432 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Mon, 19 Oct 2026 10:42:10 -0700 Subject: [PATCH 5/7] Update dart/a2ui_core/lib/src/validation/component_graph.dart Co-authored-by: Greg Spencer Signed-off-by: Polina Cherkasova --- dart/a2ui_core/lib/src/validation/component_graph.dart | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dart/a2ui_core/lib/src/validation/component_graph.dart b/dart/a2ui_core/lib/src/validation/component_graph.dart index 38934e2032..268904bab1 100644 --- a/dart/a2ui_core/lib/src/validation/component_graph.dart +++ b/dart/a2ui_core/lib/src/validation/component_graph.dart @@ -466,7 +466,11 @@ void checkPathsAndRecursion( for (final MapEntry entry in object.entries) { traverse( entry.value, - entries?[entry.key], + entries != null && + (entry.key != 'args' || + _isFunctionCall(entries, usesDirectives: usesDirectives)) + ? entries[entry.key] + : null, depth + 1, entry.key == 'args' ? callDepth + 1 : callDepth, inDataValue: false, From f7fc6e1a5eaaa057c8c5f2b69af828cb8de3d278 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Mon, 19 Oct 2026 10:42:20 -0700 Subject: [PATCH 6/7] Update dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart Co-authored-by: Greg Spencer Signed-off-by: Polina Cherkasova --- .../lib/src/inference_formats/direct_json/parser.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart b/dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart index ff3791b6ec..acbc3c5b57 100644 --- a/dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart +++ b/dart/a2ui_agent/lib/src/inference_formats/direct_json/parser.dart @@ -533,7 +533,7 @@ class _StreamedChecks { final UpdateDataModelMessage growing, final Map message, ) - when rest != null && rest == _growingJson: + when growing.hasValue && rest != null && rest == _growingJson: return _checkGrown(growing, message, closed: closed); case (final AgentToRendererMessage growing, _) when rest == null && jsonEncode(envelope) == _growingJson: From 9cceabe083f7eea518852388a1cf4f7efb2e95aa Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Mon, 19 Oct 2026 11:50:13 -0700 Subject: [PATCH 7/7] - --- conformance/agent/direct_json/compiler.yaml | 3 + conformance/agent/direct_json/decompiler.yaml | 3 + .../agent/direct_json/response_parser.yaml | 9 +- .../agent/direct_json/response_streaming.yaml | 188 +++++++++++++++--- conformance/agent/express/compiler.yaml | 3 + conformance/agent/express/decompiler.yaml | 3 + 6 files changed, 184 insertions(+), 25 deletions(-) diff --git a/conformance/agent/direct_json/compiler.yaml b/conformance/agent/direct_json/compiler.yaml index 413374c044..9bc24e815f 100644 --- a/conformance/agent/direct_json/compiler.yaml +++ b/conformance/agent/direct_json/compiler.yaml @@ -58,6 +58,9 @@ # v1.0. A case states no version of its own: the version it runs under is the # one its catalog document declares. +# A compile case checks the block on its own, as `validate_payload` does: a +# surface that the block updates without creating is assumed to exist. + - name: test_compile_json_message_list description: A well formed message list compiles to those messages. action: compile diff --git a/conformance/agent/direct_json/decompiler.yaml b/conformance/agent/direct_json/decompiler.yaml index 0f9c7c8187..e04e296efe 100644 --- a/conformance/agent/direct_json/decompiler.yaml +++ b/conformance/agent/direct_json/decompiler.yaml @@ -36,6 +36,9 @@ # v1.0. A case states no version of its own: the version it runs under is the # one its catalog document declares. +# A round trip checks the recompiled block on its own, as `validate_payload` +# does: a surface that the block updates without creating is assumed to exist. + - name: test_decompile_json_messages description: Messages are written back out as the JSON a model would have emitted, and compiling that returns them. action: decompile diff --git a/conformance/agent/direct_json/response_parser.yaml b/conformance/agent/direct_json/response_parser.yaml index 569bef689b..b4b057ef82 100644 --- a/conformance/agent/direct_json/response_parser.yaml +++ b/conformance/agent/direct_json/response_parser.yaml @@ -56,6 +56,9 @@ # v1.0. A case states no version of its own: the version it runs under is the # one its catalog document declares. +# A response is parsed as one chat turn, against a renderer that holds no +# surfaces yet, so a case that updates a surface also creates it. + # --- Unwrapping a response --- - name: test_unwrap_text_before_between_and_after_blocks @@ -213,11 +216,15 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" input: |- Here is the surface. - [{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Text", "text": "Hello"}]}}] + [{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}, {"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Text", "text": "Hello"}]}}] Tell me if you want it changed. expect: - text: "Here is the surface." - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - version: "v1.0" updateComponents: surfaceId: "s1" diff --git a/conformance/agent/direct_json/response_streaming.yaml b/conformance/agent/direct_json/response_streaming.yaml index 6f4724a33d..9e433981fc 100644 --- a/conformance/agent/direct_json/response_streaming.yaml +++ b/conformance/agent/direct_json/response_streaming.yaml @@ -102,6 +102,11 @@ # version" run them against v0.8, v0.9 and v1.0 catalogs alike. The v0.8 and # v0.9 cases use `streaming_catalog_v0_8.json` and `streaming_catalog_v0_9.json`. +# A stream is read as one chat turn, against a renderer that holds no surfaces +# yet. A case that updates a surface it does not create starts with a block +# that creates it, since a renderer rejects an update to a surface it does not +# hold. + # --- Text as it arrives --- - name: test_chunk_text_emitted_as_it_arrives @@ -125,6 +130,13 @@ format: direct_json catalog: "test_data/catalogs/simplified_catalog_v1_0.json" steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: | Here is the surface. expect: @@ -231,6 +243,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: [] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateComponents": {"surfaceId": "s1",' expect: [] - input: ' "components": [{"id": "root", "component": "Text", "text": "Hello"}]}}]' @@ -253,6 +272,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: ["greeting"] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateDataModel": {"surfaceId": "s1", "value": {"greeting": "Hel' expect: - a2ui: @@ -278,6 +304,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: ["text"] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Text", "text": "Hello", "variant": "cap' expect: [] - input: 'tion"}]}}]' @@ -301,6 +334,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: ["value"] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateDataModel": {"surfaceId": "s1", "path": "/greeting", "value": "Hel' expect: - a2ui: @@ -326,6 +366,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: ["text"] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Text", "text": "Hel' expect: [] - input: 'lo"' @@ -352,6 +399,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: ["text"] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Card", "child": "body"}, {"id": "body", "component": "Text", "text": "Hel' expect: - a2ui: @@ -410,6 +464,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: [] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Text", "text": "Hel' expect: [] - input: 'lo"}]}}]' @@ -432,6 +493,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: [] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Text", "text": ""}]}}]' @@ -453,6 +521,13 @@ format: direct_json catalog: "test_data/catalogs/simplified_catalog_v1_0.json" steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: | First the greeting. expect: @@ -548,6 +623,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: ["text"] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Text", "text": {"@path": "/user' expect: [] - input: '/name"}}]}}]' @@ -571,6 +653,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: ["text"] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Te' expect: [] - input: 'xt", "text": "Hello"}]}}]' @@ -593,6 +682,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: ["text"] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Card", "child": "bo' expect: [] - input: 'dy"}, {"id": "body", "component": "Text", "text": "Hello"}]}}]' @@ -618,6 +714,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: ["text"] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Column", "children": ["one", "tw' expect: [] - input: 'o"]}, {"id": "one", "component": "Text", "text": "One"}, {"id": "two", "component": "Text", "text": "Two"}]}}]' @@ -646,6 +749,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: ["text"] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Column", "children": {"componentId": "row"' expect: [] - input: ', "path": "/items"}}, {"id": "row", "component": "Text", "text": "Item"}]}}]' @@ -744,6 +854,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: [] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateDataModel": {"surfaceId": "s1", "value": {"items": {' expect: - a2ui: @@ -783,6 +900,13 @@ catalog: "test_data/catalogs/simplified_catalog_v1_0.json" progressive_keys: [] steps: + - input: '[{"version": "v1.0", "createSurface": {"surfaceId": "s1", "catalogId": "conformance/simplified"}}]' + expect: + - a2ui: + - version: "v1.0" + createSurface: + surfaceId: "s1" + catalogId: "conformance/simplified" - input: '[{"version": "v1.0", "updateDataModel": {"surfaceId": "s1", "value": {"title": "Top", "items": {"item1"' expect: [] - input: ': {"name": "Item 1"}}}}}]' @@ -1289,7 +1413,7 @@ root: "root" - name: test_stream_update_before_create_surface_fails_v09 - description: An updateComponents before its surface's createSurface is emitted while the block arrives, and the block raises a validation error when it closes, since the surface did not exist yet. + description: An updateComponents for a surface that does not exist yet is held back, since a renderer would reject it, and the block raises a validation error when it closes. action: parse_chunk args: format: direct_json @@ -1299,15 +1423,7 @@ - input: "[" expect: [] - input: '{"version": "v0.9", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Text", "text": "hi"}]}}, ' - expect: - - a2ui: - - version: "v0.9" - updateComponents: - surfaceId: "s1" - components: - - id: "root" - component: "Text" - text: "hi" + expect: [] - input: '{"version": "v0.9", "createSurface": {"catalogId": "test_catalog", "surfaceId": "s1"}}]' expect_error: category: "ValidationError" @@ -3061,7 +3177,7 @@ path: "/name" - name: test_stream_data_model_before_create_surface_fails_v09 - description: An updateDataModel before its surface's createSurface is emitted while the block arrives, and the block raises a validation error when it closes. + description: An updateDataModel for a surface that does not exist yet is held back, since a renderer would reject it, the messages after it wait with it, and the block raises a validation error when it closes. action: parse_chunk args: format: direct_json @@ -3071,20 +3187,9 @@ - input: "[" expect: [] - input: '{"version": "v0.9", "updateDataModel": {"surfaceId": "s1", "value": {"name": "Alice"}}}, ' - expect: - - a2ui: - - version: "v0.9" - updateDataModel: - surfaceId: "s1" - value: - name: Alice + expect: [] - input: '{"version": "v0.9", "createSurface": {"catalogId": "test_catalog", "surfaceId": "s1"}}, ' - expect: - - a2ui: - - version: "v0.9" - createSurface: - surfaceId: "s1" - catalogId: "test_catalog" + expect: [] - input: '{"version": "v0.9", "updateComponents": {"surfaceId": "s1", "components": [{"id": "root", "component": "Text", "text": {"path": "/name"}}]}}]' expect_error: category: "ValidationError" @@ -3428,6 +3533,13 @@ catalog: "test_data/catalogs/streaming_catalog_v0_9.json" progressive_keys: [] steps: + - input: '[{"version": "v0.9", "createSurface": {"surfaceId": "default", "catalogId": "test_catalog"}}]' + expect: + - a2ui: + - version: "v0.9" + createSurface: + surfaceId: "default" + catalogId: "test_catalog" - input: '[ {"version": "v0.9", "updateDataModel": {"surfaceId": "default", "value": {"name": "John", "incomplete": ' expect: [] - input: '"done"}}}]' @@ -3484,6 +3596,13 @@ catalog: "test_data/catalogs/streaming_catalog_v0_9.json" progressive_keys: [] steps: + - input: '[{"version": "v0.9", "createSurface": {"surfaceId": "default", "catalogId": "test_catalog"}}]' + expect: + - a2ui: + - version: "v0.9" + createSurface: + surfaceId: "default" + catalogId: "test_catalog" - input: '[ {"version": "v0.9", "updateDataModel": {"surfaceId": "default", "value": {"infoLink": "[More Info](x)", "rat' expect: [] - input: 'ing": "5 star"' @@ -3549,6 +3668,13 @@ catalog: "test_data/catalogs/streaming_catalog_v0_9.json" progressive_keys: [] steps: + - input: '[{"version": "v0.9", "createSurface": {"surfaceId": "default", "catalogId": "test_catalog"}}]' + expect: + - a2ui: + - version: "v0.9" + createSurface: + surfaceId: "default" + catalogId: "test_catalog" - input: "[" expect: [] - input: '{"version": "v0.9", "updateDataModel": {"surfaceId": "default", "value": {"title": "Top Restaurants", "items": {' @@ -3596,6 +3722,13 @@ catalog: "test_data/catalogs/streaming_catalog_v0_9.json" progressive_keys: [] steps: + - input: '[{"version": "v0.9", "createSurface": {"surfaceId": "default", "catalogId": "test_catalog"}}]' + expect: + - a2ui: + - version: "v0.9" + createSurface: + surfaceId: "default" + catalogId: "test_catalog" - input: '[ {"version": "v0.9", "updateDataModel": {"surfaceId": "default", "value": {"title": "Top Restaurants","items": {"item1": {"name": "Food", "detail": "Spicy", "imageUrl":' expect: [] - input: ' "https://example.com/food.png"}}}}}]' @@ -3633,6 +3766,13 @@ catalog: "test_data/catalogs/streaming_catalog_v0_9.json" progressive_keys: [] steps: + - input: '[{"version": "v0.9", "createSurface": {"surfaceId": "default", "catalogId": "test_catalog"}}]' + expect: + - a2ui: + - version: "v0.9" + createSurface: + surfaceId: "default" + catalogId: "test_catalog" - input: '[ {"version": "v0.9", "updateDataModel": {"surfaceId": "default", "value": {"title": "Top Restaurants","items": {"item2": {"name": "Han Dynasty", "imageUrl": "http://localhost:10002/static/mapotofu.jpeg", "broken":' expect: [] - input: '"x"}}}}}]' diff --git a/conformance/agent/express/compiler.yaml b/conformance/agent/express/compiler.yaml index f1240830fd..1424b4d8ae 100644 --- a/conformance/agent/express/compiler.yaml +++ b/conformance/agent/express/compiler.yaml @@ -101,6 +101,9 @@ # v1.0. A case states no version of its own: the version it runs under is the # one its catalog document declares. +# A compile case checks the block on its own, as `validate_payload` does: a +# surface that the block updates without creating is assumed to exist. + - name: test_compile_express_minimal_root description: A block that assigns root compiles to one createSurface carrying that component. action: compile diff --git a/conformance/agent/express/decompiler.yaml b/conformance/agent/express/decompiler.yaml index e9bced6eb9..c8261a62e8 100644 --- a/conformance/agent/express/decompiler.yaml +++ b/conformance/agent/express/decompiler.yaml @@ -50,6 +50,9 @@ # v1.0. A case states no version of its own: the version it runs under is the # one its catalog document declares. +# A round trip checks the recompiled block on its own, as `validate_payload` +# does: a surface that the block updates without creating is assumed to exist. + - name: test_decompile_express_components description: A createSurface writes back out as the surface line and the component assignments it came from. action: decompile