diff --git a/docs/protocol.md b/docs/protocol.md index 9856caa..cffc3f5 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -10,6 +10,8 @@ A method receives the task URL and prompt. Follow the declared task kind: return Task requirements are data: expected markers, fact groups, forbidden output, minimum length, body coverage, structure and resource/file contracts are selected by the task. Missing reference coverage is unavailable, not zero or a perfect score. Binary task acceptance and text-quality diagnostics are separate. LexBench owns `score_attempt`, the shared `task_passed` predicate and file-byte acceptance. Browser Eval calls the same installed implementations. Other harnesses can use `lexbench_webfetch.evaluation.evaluate_response` or the `score` command without Browser Eval; disclose the benchmark/scorer revision when comparing scores. +When an adapter selects answer fields from a retained native response, validate that projection before scoring. `score_attempt` accepts `native_response`, `native_files` and `native_diagnostic` for this check; the execution harness owns persistence and hash verification. Declare every score-bearing body, metadata, file, title and final-URL source. Missing-field defaults may only be empty; they cannot supply an answer absent from the response. Direct `evaluate_response` and CLI inputs are the submitted answer itself, so their caller remains responsible for acquisition provenance. + ## Live sources and frozen evidence Input validation proves that supplied bytes match the frozen version. It does not prove that a live website still matches that version. A publishable live-site cohort must independently verify task-relevant reference scope before calls and again after the complete declared scope. If source content changes, preserve the attempt and investigate the shared cause; do not silently refresh references or omit failed cases. diff --git a/src/lexbench_webfetch/answer_projection.py b/src/lexbench_webfetch/answer_projection.py new file mode 100644 index 0000000..469bf37 --- /dev/null +++ b/src/lexbench_webfetch/answer_projection.py @@ -0,0 +1,312 @@ +"""Validate score-bearing fields against retained native answer bytes.""" +import json +import re +from urllib.parse import parse_qsl, quote, urljoin, urlsplit +from .paths import validate_http_url + + +def _projection_value(payload, pointer): + if not isinstance(pointer, list) or not pointer: + raise ValueError('Native answer projection requires a nonempty field pointer') + value = payload + try: + for key in pointer: + if isinstance(value, dict) and isinstance(key, str): + value = value[key] + elif isinstance(value, list) and type(key) is int and key >= 0: + value = value[key] + else: + raise ValueError('Native answer pointer is not a structural field selection') + except (KeyError, IndexError) as error: + raise ValueError('Native answer field is missing') from error + return value + + +def _attribute_record_urls(groups, base_url): + """Project declared HTML URL attributes without classifying their targets.""" + urls = [] + for group in groups if isinstance(groups, list) else []: + if not isinstance(group, dict): + continue + elements = group.get('results') + for element in elements if isinstance(elements, list) else []: + if not isinstance(element, dict): + continue + attributes = element.get('attributes') + for attribute in attributes if isinstance(attributes, list) else []: + if not isinstance(attribute, dict): + continue + name, value = attribute.get('name'), attribute.get('value') + if not isinstance(name, str) or not isinstance(value, str): + continue + name = name.casefold() + if name not in {'src', 'srcset', 'href', 'data-src', 'data-original'}: + continue + values = ([part.strip().split()[0] for part in value.split(',') if part.strip()] + if name == 'srcset' else [value]) + for raw in values: + try: + urls.append(urljoin(base_url, raw.strip())) + except ValueError: + # Retain malformed native values; the shared scorer, + # not a provider adapter, decides URL eligibility. + urls.append(raw) + return sorted(set(urls)) + + +SCORE_ANSWER_FIELDS = frozenset({'links', 'images', 'videos', 'media', 'extras'}) + + +def _selected(payload, spec): + # Missing native fields may mean no answer, never a synthesized answer. + if 'default' in spec and spec['default'] not in (None, '', [], {}): + raise ValueError('Native answer defaults must be empty') + try: + return _projection_value(payload, spec['pointer']) + except ValueError: + if 'default' not in spec: + raise + return spec['default'] + + +def _project_field(payload, spec): + value = _selected(payload, spec) + kind = spec.get('format', 'identity') + if kind == 'identity': + return value + if kind == 'html_attribute_urls': + return _attribute_record_urls(value, spec['base_url']) + if kind == 'named_links': + return [{'text': item['anchor'], 'href': item['url']} + for item in value if isinstance(item, dict) + and isinstance(item.get('anchor'), str) and item['anchor'].strip() + and isinstance(item.get('url'), str) and item['url'].strip()] + if kind == 'video_values': + values = [item for item in value if isinstance(item, (str, dict))] if isinstance(value, list) else [] + legacy = _selected(payload, spec['legacy']) + return values + ([legacy] if isinstance(legacy, str) and legacy else []) + raise ValueError('Unknown native answer field projection') + + +def require_native_projection_schema(declaration): + """Keep historical answer scope separate from schema-2 result context.""" + if declaration.get('schema_version') not in {1, 2}: + raise ValueError('Unsupported native answer projection') + if declaration['schema_version'] == 1 and 'result_fields' in declaration: + raise ValueError('Native result_fields require projection schema 2') + + +def project_native_answer(payload, declaration): + """Select declared native fields; never inspect references or URL meaning.""" + require_native_projection_schema(declaration) + kind = declaration.get('content_format') + if declaration.get('source_format') == 'bytes': + if declaration.get('answer_fields'): + raise ValueError('Native byte projection cannot declare structured answer fields') + if not isinstance(payload, bytes): + raise ValueError('Native byte projection requires bytes') + if kind == 'utf8': + return payload.decode('utf-8'), {} + if kind == 'empty': + return '', {} + raise ValueError('Unknown native byte projection') + if kind == 'empty': + content = '' + else: + value = _selected(payload, {'pointer': declaration.get('content_pointer'), + **({'default': ''} if declaration.get('optional_content') else {})}) + if kind == 'text' and isinstance(value, str): + content = value + elif kind == 'json': + content = json.dumps(value, ensure_ascii=False, separators=(',', ':')) + else: + raise ValueError('Native answer field differs from its declared type') + fields = {} + for name, spec in declaration.get('answer_fields', {}).items(): + if name not in SCORE_ANSWER_FIELDS: + raise ValueError('Unknown native answer metadata field') + fields[name] = _project_field(payload, spec) + return content, fields + + +def project_native_files(payload, declaration): + """Decode native delivered bytes; filenames and logs cannot supply answers.""" + import base64 + spec = declaration.get('files') + if spec is None: + return [] + value = _selected(payload, spec) if spec.get('format') != 'raw_bytes' else payload + if spec['format'] == 'raw_bytes' and isinstance(payload, bytes): + return [payload] if payload else [] + if spec['format'] == 'base64_file': + return [base64.b64decode(value, validate=True)] if value else [] + if spec['format'] == 'worker_files': + return [base64.b64decode(item['base64'], validate=True) for item in value] + raise ValueError('Unknown native file projection') + + +def main_document_url_from_fetch_log(stderr: str | bytes, requested_url: str) -> str | None: + """Replay the effective main-document URL from the same call's debug log.""" + pattern = re.compile( + rb"fetch request complete method=(?:GET|POST) url=(?P\S+) " + rb"final_url=(?P\S+) status=(?P\d{3})\b" + ) + + diagnostic = stderr.encode("utf-8") if isinstance(stderr, str) else stderr + navigation_pattern = re.compile( + rb"following pending location navigation asynchronously[^\n]* url=(?P\S+)" + ) + + def field(row: re.Match[bytes], name: str) -> str: + return row.group(name).decode("utf-8", "replace") + + def identity(value: str) -> tuple | None: + try: + validate_http_url(value) + parts = urlsplit(value) + return ( + parts.scheme.casefold(), + (parts.hostname or "").casefold(), + parts.port if parts.port is not None else (443 if parts.scheme.casefold() == "https" else 80), + re.sub(r"%[0-9a-fA-F]{2}", lambda match: match[0].upper(), + quote(parts.path, safe="/%:@!$&'()*+,;=")).rstrip("/") or "/", + sorted(parse_qsl(parts.query, keep_blank_values=True)), + ) + except ValueError: + return None + + requested_identity = identity(requested_url) + if requested_identity is None: + return None + rows = list(pattern.finditer(diagnostic)) + + # Legacy completion records do not identify redirect hops or request owners. + # Never infer the document from GET ordering or challenge query parameters. + row = next((candidate for candidate in rows + if identity(field(candidate, "url")) == requested_identity), None) + if row is None or 300 <= int(field(row, "status")) < 400: + return None + navigations = list(navigation_pattern.finditer(diagnostic)) + for index, navigation in enumerate(navigations): + boundary = navigations[index + 1].start() if index + 1 < len(navigations) else len(diagnostic) + committed = diagnostic.find( + b"committed main navigation before replacement realm bootstrap", + navigation.end(), boundary, + ) + if committed < 0: + continue + navigation_identity = identity(field(navigation, "url")) + if navigation_identity is None: + return None + matches = [candidate for candidate in rows + if navigation.end() <= candidate.start() < boundary + and identity(field(candidate, "url")) == navigation_identity] + if len(matches) != 1 or 300 <= int(field(matches[0], "status")) < 400: + return None + row = matches[0] + candidate = field(row, "final_url") + if identity(candidate) is None: + return None + try: + validate_http_url(candidate) + except ValueError: + return None + return candidate + + + +def project_native_result_fields(payload, declaration, *, request_url=None, + diagnostic=None, transport_receipt=None): + """Replay scored title/address from native fields or same-call evidence.""" + require_native_projection_schema(declaration) + fields = {'title': None, 'final_url': None} + for name, spec in declaration.get('result_fields', {}).items(): + if name not in fields: + raise ValueError('Unknown native result field') + kind = spec.get('format', 'identity') + if kind == 'request_url': + if name != 'final_url' or request_url is None: + raise ValueError('Native URL fallback requires its declared request URL') + value = request_url + elif kind == 'fetch_log_final_url': + if name != 'final_url' or diagnostic is None or request_url is None: + raise ValueError('Native final URL lacks its same-call diagnostic evidence') + value = main_document_url_from_fetch_log(diagnostic, request_url) + elif kind == 'transport_response_url': + if name != 'final_url' or not transport_receipt: + raise ValueError('Native final URL lacks its source transport receipt') + value = transport_receipt['response_url'] + else: + value = _selected(payload, {**spec, 'default': None}) + if kind == 'text_or_none': + value = value if isinstance(value, str) else None + elif kind != 'identity': + raise ValueError('Unknown native result field projection') + if spec.get('nonempty') and not value: + value = None + if not value and spec.get('fallback') == 'request_url': + if name != 'final_url' or request_url is None: + raise ValueError('Native URL fallback requires its declared request URL') + value = request_url + fields[name] = value + return fields + + +def require_native_answer_projection(content, metadata, raw_response, *, files=(), title=None, final_url=None, + request_url=None, diagnostic=None, transport_receipt=None): + """Reject every undeclared score-bearing input, then check exact provenance.""" + declaration = metadata.get('native_answer_projection') + present = {key for key in SCORE_ANSWER_FIELDS if metadata.get(key)} + if declaration is None: + if (present or files or title is not None + or (final_url is not None and final_url != request_url) + or (raw_response is not None and content)): + raise ValueError('Scored native answer has undeclared answer fields') + return + require_native_projection_schema(declaration) + if raw_response is None: + raise ValueError('Declared native answer lacks its retained raw response') + if present - set(declaration.get('answer_fields', {})): + raise ValueError('Scored native answer has undeclared answer fields') + allowed = metadata.get('native_route_receipt', {}).get('answer_fields') + if allowed is not None: + def check_source(spec): + pointer = spec.get('pointer') + if pointer: + name = '.'.join(str(key) for key in pointer if not isinstance(key, int)) + allowed_names = [field.replace('[]', '') for field in allowed] + if not any(name == field or name.startswith(field+'.') for field in allowed_names): + raise ValueError('Native projection reads outside its frozen answer fields') + if 'legacy' in spec: + check_source(spec['legacy']) + if declaration.get('source_format') == 'bytes': + required = 'original_bytes' if declaration.get('files') else 'stdout' + if required not in allowed: + raise ValueError('Native byte projection differs from its frozen answer fields') + else: + if declaration.get('content_format') != 'empty': + check_source({'pointer': declaration.get('content_pointer')}) + for spec in declaration.get('answer_fields', {}).values(): + check_source(spec) + if declaration.get('files'): + check_source(declaration['files']) + for spec in declaration.get('result_fields', {}).values(): + check_source(spec) + if spec.get('format') == 'fetch_log_final_url' and 'diagnostic.main_document_url' not in allowed: + raise ValueError('Native final URL conversion is outside its frozen sources') + payload = raw_response if declaration.get('source_format') == 'bytes' else json.loads(raw_response) + projected_result = project_native_result_fields(payload, declaration, request_url=request_url, + diagnostic=diagnostic, transport_receipt=transport_receipt) + if projected_result != {'title': title, 'final_url': final_url}: + raise ValueError('Scored title or final URL differs from its declared native projection') + if declaration.get('schema_version') == 2: + for spec in declaration.get('answer_fields', {}).values(): + if spec.get('format') == 'html_attribute_urls' and spec.get('base_url') != (final_url or request_url): + raise ValueError('Native resource base URL differs from its verified result URL') + expected, fields = project_native_answer(payload, declaration) + if content != expected or any(metadata.get(key) != value for key, value in fields.items()): + raise ValueError('Scored answer differs from its declared native field projection') + if [item['data'] for item in files] != project_native_files(payload, declaration): + raise ValueError('Scored native files differ from their declared byte projection') + + diff --git a/src/lexbench_webfetch/body_coverage.py b/src/lexbench_webfetch/body_coverage.py index 8127824..4bb4b19 100644 --- a/src/lexbench_webfetch/body_coverage.py +++ b/src/lexbench_webfetch/body_coverage.py @@ -6,6 +6,36 @@ from math import ceil +def native_markup_body_text(content, fallback): + """Read existing HTML layout in native Markdown, preserving literal code.""" + import html + from .dynamic_lists import visible_html_text + protected = {} + prefix = "NATIVEBODYLITERAL" + while prefix in content: + prefix += "X" + def protect(match): + token = prefix + str(len(protected)) + "END" + protected[token] = html.unescape(match[0]) + return token + masked = re.sub(r"(`+|~{3,}).*?\1|\\<[^>\n]+>", protect, content, flags=re.S) + if not re.search(r"<(?:table|thead|tbody|tfoot|tr|th|td|ul|ol|li|h[1-6]|p|div|section|article|br|span|strong|em|b|a)\b", masked, re.I): + return fallback + # Native converters can expose unfenced code samples as standalone + # table tags. Their original text remains authoritative; do not erase it + # by interpreting an ambiguous tag-only line as document structure. + if re.search(r"(?m)^[ \t]*[ \t]*$", masked): + return fallback + from bs4 import ParserRejectedMarkup + try: + visible = visible_html_text(masked, preserve_links=True) + except ParserRejectedMarkup: + return fallback + for token, literal in protected.items(): + visible = visible.replace(token, literal) + return visible + + def tokens(text): from .metrics import normalize_for_match return re.findall(r'[\u3400-\u9fff]|[^\W_\u3400-\u9fff]+', normalize_for_match(text)) diff --git a/src/lexbench_webfetch/composite_tasks.py b/src/lexbench_webfetch/composite_tasks.py index 11dc516..3a4ddc7 100644 --- a/src/lexbench_webfetch/composite_tasks.py +++ b/src/lexbench_webfetch/composite_tasks.py @@ -23,15 +23,63 @@ def _key(value: str) -> str: return re.sub(r"\s+", " ", value).strip(" *#>`") +def _column_scope(content: str, group: dict) -> str | None: + """Own a declared named list by its table column, not the adjacent column.""" + start, end = group.get("start_pattern"), group.get("end_pattern") + if group["kind"] != "named_link" or not start or not end: + return None + from .structured_tables import _pipe_cells + lines = content.splitlines() + def label(cell): + return re.sub(r"^#{1,6}\s+", "", visible_markdown_links(cell).strip()).strip(" *") + for index, line in enumerate(lines): + delimiter = "\t" if "\t" in line else "|" if "|" in line else None + if not delimiter: + continue + cells = _pipe_cells(line) if delimiter == "|" else line.split("\t") + if len(cells) < 2: + continue + starts = [i for i, cell in enumerate(cells) if re.fullmatch(start, label(cell), re.I)] + if not starts: + continue + if len(starts) != 1: + return "" + column = starts[0] + sibling_end = any(i != column and re.fullmatch(end, label(cell), re.I) + for i, cell in enumerate(cells)) + if not sibling_end and not re.search(end, "\n".join(lines[index + 1:]), re.I | re.M): + return "" + cursor = index + 1 + if delimiter == "|": + if cursor == len(lines): + return "" + divider = _pipe_cells(lines[cursor]) + if len(divider) != len(cells) or not all(re.fullmatch(r"\s*:?-{3,}:?\s*", value) for value in divider): + continue + cursor += 1 + owned = [cells[column]] + while cursor < len(lines) and lines[cursor].strip(): + values = _pipe_cells(lines[cursor]) if delimiter == "|" else lines[cursor].split("\t") + if len(values) != len(cells): + break + owned.append(values[column]) + cursor += 1 + return "\n".join(owned) + return None + + def _scope(content: str, group: dict) -> str: """Return one declared section, failing closed when a boundary is absent.""" + column = _column_scope(content, group) + if column is not None: + return column start = group.get("start_pattern") end = group.get("end_pattern") begin = 0 def boundary(pattern: str, value: str): matches = list(re.finditer(pattern, value, re.I | re.M)) - return next( + match = next( ( match for match in matches @@ -39,18 +87,32 @@ def boundary(pattern: str, value: str): ), matches[0] if matches else None, ) + if match is not None: + return match.start() + # Native Markdown can serialize adjacent reference-style controls + # without a space. Restore only that layout boundary, requiring the + # complete heading to match so prose cannot create a new section. + for line in re.finditer(r'(?m)^.*$', value): + spaced = re.sub(r'(?<=\])(?=(?:\\)?\[)', ' ', line[0]) + if spaced != line[0] and re.fullmatch(pattern, spaced, re.I | re.M): + return line.start() + if group['kind'] == 'field_report' and line[0].strip().startswith('|'): + cells = _pipe_cells(line[0]) + if len(cells) == 1 and re.fullmatch(pattern, cells[0].strip(), re.I | re.M): + return line.start() + return None if start: match = boundary(start, content) - if not match: + if match is None: return "" - begin = match.start() + begin = match finish = len(content) if end: match = boundary(end, content[begin:]) - if not match: + if match is None: return "" - finish = begin + match.start() + finish = begin + match return content[begin:finish] @@ -280,6 +342,75 @@ def _link_identity(url: str, group: dict) -> str | None: return identity +def _html_table_record_contexts(content: str, group: dict, base_url: str, allowed: set): + """Retain native row/cell boundaries lost by a text-only projection. + + One linked cell owns its row's fields. Multiple linked cells are separate + records; extra links nested in one cell do not create additional records. + """ + from .dynamic_lists import _scoped_html, visible_html_text + soup = _scoped_html(content) + contexts, table_ids = {}, set() + for row in soup.select('tr'): + owners = [] + for cell in row.select('td,th'): + if cell.find_parent('tr') is not row: + continue + identities = [] + for link in cell.select('a[href]'): + if link.find_parent(['td', 'th']) is not cell: + continue + identity = _link_identity(urljoin(base_url, link['href']), group) + if identity in allowed: + table_ids.add(identity) + if identity not in identities: + identities.append(identity) + if identities: + owners.append((cell, identities[0])) + for cell, identity in owners: + owner = row if len(owners) == 1 else cell + context = visible_html_text(str(owner), preserve_links=True) + contexts.setdefault(identity, []).append(context) + return table_ids, contexts + + +def _card_section_context(content: str, begin: int, end: int, title_start: int, + identity: str, group: dict, base_url: str) -> str: + """Stop an implicit card at a new peer section; keep owned subheadings.""" + context = content[begin:end] + first_line = content[begin:].split("\n", 1)[0] + own_heading = re.match(r"[ \t]{0,3}(#{1,6})[ \t]+", first_line) + level = len(own_heading[1]) if own_heading else 6 + links = markdown_links(content) + fence = None + offset = 0 + for line in context.splitlines(keepends=True): + marker = re.match(r"[ \t]{0,3}(`{3,}|~{3,})", line) + if marker: + value = marker[1] + if fence is None: + fence = value + elif (value[0] == fence[0] and len(value) >= len(fence) + and not line[marker.end():].strip()): + fence = None + elif fence is None: + heading = re.match(r"[ \t]{0,3}(#{1,6})[ \t]+", line) + absolute = begin + offset + owned_heading = heading and any( + absolute <= link.start < absolute + len(line) + and _link_identity(urljoin(group.get("link_base_url", base_url), link.destination), group) == identity + for link in links if not link.is_image) + if owned_heading: + level = min(level, len(heading[1])) + if (heading and begin + offset > title_start + and len(heading[1]) <= level + and not owned_heading + and not any(link.start <= absolute < link.end for link in links)): + return context[:offset] + offset += len(line) + return context + + def _html_field_surface(content: str, group: dict, base_url: str) -> tuple[str, str]: """Keep real DOM ownership while projecting visible links and field text.""" from bs4 import Comment, NavigableString, Tag @@ -351,7 +482,14 @@ def target_identities(node): node.insert_before(NavigableString('\n'+marker+'S'+str(index)+'\n')) node.insert_after(NavigableString('\n'+marker+'E'+str(index)+'\n')) for node in soup.find_all(['img','br']): - node.replace_with(NavigableString('\ufdd0')) + # HTML void elements cannot own following text. Some retained native + # HTML includes closing img/br tags; preserve text the parser nested + # under them while still omitting image alt text from record titles. + node.insert_before(NavigableString('\ufdd0')) + node.unwrap() + for anchor in soup.find_all('a', href=True): + if not anchor.get_text().replace('\ufdd0', '').strip(): + anchor.unwrap() return visible_html_text(str(soup), preserve_links=True).replace('\ufdd0', '\n\n'), marker @@ -372,7 +510,7 @@ def _record_owner_ranges(content: str, marker: str | None): return clean, ranges -def _named_link_group(content: str, group: dict, base_url: str, links: list | None, *, ownership_marker: str | None = None) -> dict: +def _named_link_group(content: str, group: dict, base_url: str, links: list | None, *, ownership_marker: str | None = None, native_html: str | None = None) -> dict: content, owner_ranges = _record_owner_ranges(content, ownership_marker) standard = { "version": 1, @@ -407,11 +545,27 @@ def _named_link_group(content: str, group: dict, base_url: str, links: list | No def is_record_start(link) -> bool: pattern = group.get("record_line_pattern") if not pattern: - return True + line_start = content.rfind("\n", 0, link.start) + 1 + prefix = content[line_start:link.start].strip() + prior = previous_targets[link.start] + if prior and prior[1] > line_start: + return True + same_target = bool(prior and prior[0] == _link_identity(urljoin(base_url, link.destination), group)) + return (same_target or serialized_link_card_fields(link) is not None or not prefix or _DATE_OR_TIME.fullmatch(prefix) is not None + or re.fullmatch(r"[\s#>*+\-\d.)|/:•·●▪◦,，、;；]*", prefix) is not None) line_start = content.rfind("\n", 0, link.start) + 1 line_end = content.find("\n", link.end) line_end = len(content) if line_end < 0 else line_end - return bool(re.search(pattern, content[line_start:line_end], re.I)) + line = content[line_start:line_end].lstrip() + date = _DATE_OR_TIME.match(line) + if date and re.match(r"[ \t]*#{1,6}[ \t]+", line[date.end():]): + line = line[date.end():].lstrip() + if re.fullmatch(r"(?P\*\*|__).+?(?P=mark)", link.label): + raw_link = content[link.start:link.end] + if line.strip() == raw_link: + return any(re.search(pattern, "#" * level + " " + line, re.I) + for level in range(1, 7)) + return bool(re.search(pattern, line, re.I)) # A card commonly places its title link, body paragraphs and date on # separate Markdown blocks. Own everything from the first qualifying @@ -451,7 +605,8 @@ def previous_scalar(link): return previous_start, scalar def visible_label(link): - label = link.label + card_fields = serialized_link_card_fields(link) + label = card_fields[0] if card_fields else link.label for image in reversed([item for item in markdown_links(label) if item.is_image]): label = label[:image.start] + label[image.end:] label = visible_markdown_links(label).strip() @@ -510,19 +665,34 @@ def owner_for(position): for link in markdown_links(content): destination = urljoin(group.get("link_base_url", base_url), link.destination) identity = _link_identity(destination, group) - if identity is None or link.is_image or not is_record_start(link): + if (identity is None or link.is_image + or (group.get('record_line_pattern') and not is_record_start(link))): continue entry = allowed_rows.get(identity) owner = owner_for(link.start) title = preferred_titles.get( (identity, owner), - entry["title"] if entry and owner is None else "", + entry["title"] if entry and owner is None and is_record_start(link) else "", ) # Even an unnamed target starts a different card. Its fields must # never complete the preceding named record. previous = previous_scalar(link) start = previous[0] if previous else link.start - candidates.append((identity, destination, title, start, link.start, owner)) + preceding = content.rfind("\n", 0, start) + 1 + if preceding > 0: + prior_start = content.rfind("\n", 0, preceding - 1) + 1 + prior = previous_targets[link.start] + if (prior is None or prior[0] == identity) and _DATE_OR_TIME.fullmatch(content[prior_start:preceding].strip()): + start = prior_start + if preceding > 0: + prior_start = content.rfind("\n", 0, preceding - 1) + 1 + prior_line = content[prior_start:preceding].strip() + if markdown_links(prior_line) and _DATE_OR_TIME.fullmatch(visible_markdown_links(prior_line).strip()) and all( + _link_identity(urljoin(base_url, prior.destination), group) == identity + and not visible_markdown_links(prior.label).strip() + for prior in markdown_links(prior_line)): + start = prior_start + candidates.append((identity, destination, title, start, link.start, owner, link.end)) owned = {} # Consecutive links to the same identity belong to one card (for # example, a title link followed by a linked date). A later duplicate @@ -533,10 +703,13 @@ def owner_for(position): if (not ordered or ordered[-1][0] != candidate[0] or ordered[-1][5] != candidate[5]): ordered.append(candidate) - for index, (identity, destination, label, start, link_start, owner) in enumerate(ordered): + for index, (identity, destination, label, start, link_start, owner, link_end) in enumerate(ordered): line_start = content.rfind("\n", 0, start) + 1 finish = ordered[index + 1][3] if index + 1 < len(ordered) else len(content) next_line = content.rfind("\n", 0, finish) + 1 if finish < len(content) else finish + if index and ordered[index - 1][6] > line_start: + line_start = start + next_line = max(next_line, link_end) if owner: begin, end = owner # The first link can have fields before it in the same card. @@ -559,8 +732,23 @@ def owner_for(position): cursor = max(cursor, end) pieces.append(content[cursor:next_line]) context = "\n\n".join(pieces) + context = _card_section_context(context, 0, len(context), max(0, link_start-line_start), identity, group, base_url) if label and _required_fields(context, label, field_specs=fields): owned[identity] = {"url": destination, "title": label, "context": context} + if native_html: + table_ids, table_contexts = _html_table_record_contexts( + native_html, group, base_url, set(allowed_rows), + ) + for identity in table_ids: + entry = allowed_rows[identity] + label = entry["title"] + context = next((value for value in table_contexts.get(identity, []) + if _required_fields(value, label, field_specs=fields)), None) + if context is None: + owned.pop(identity, None) + else: + owned[identity] = {"url": entry["url"], "title": label, + "context": context, "context_basis": "native_table_owner"} records = [row for identity, row in owned.items() if identity in allowed_rows] else: records = entries @@ -622,13 +810,124 @@ def _value_matches_spec(value: str, spec: dict, *, whole_cell: bool = False) -> return len(visible) >= minimum and any(character.isalpha() for character in visible) +def serialized_link_card_fields(link): + """Recover explicit inline card spans; URLs only corroborate visible text.""" + label = link.label + for image in reversed(markdown_links(label)): + if image.is_image: + label = label[:image.start] + " " + label[image.end:] + label = visible_markdown_links(label).strip() + explicit = re.fullmatch( + r"(?P.+),\s+(?P[^,\n]+,\s*[A-Z]{2})\s+" + r"(?P.+?)(?P\d{1,2}/\d{1,2}/\d{2,4},\s*\d{1,2}:\d{2}\s*[AP]M)(?:.*)", label) + if explicit: + name = re.sub(r"^(?:Find Tickets|Event Info)\s*", "", explicit['name']) + return name, [explicit['date'], explicit['venue'], explicit['city']] + times = list(re.finditer(r"\d{1,2}:\d{2}\s*[AP]M\b", label, re.I)) + if not times: + return None + end = times[-1].end() + tail = label[end:].strip() + compact = lambda value: re.sub(r"[^a-z0-9]", "", value.casefold()) + slugs = [compact(unquote(segment)) for segment in urlsplit(link.destination).path.split('/')] + cities = [match for match in re.finditer( + r"(?<=[a-z\s])(?=[A-Z][A-Za-z .'-]+,\s*[A-Z]{2}$)", tail) + if any(compact(tail[match.start():].rsplit(',', 1)[0]) in slug for slug in slugs)] + if not cities: + return None + city = cities[0] + name_venue, location = tail[:city.start()].strip(), tail[city.start():] + city_key = compact(location.rsplit(',', 1)[0]) + matches = [] + for segment in urlsplit(link.destination).path.split('/'): + slug = compact(unquote(segment)) + if city_key not in slug: + continue + # The URL cannot supply a missing name, date, place or city. It only + # distinguishes two values already present in the same link label. + slug = slug.split(city_key, 1)[0] + for boundary in re.finditer(r"(?<=\w)(?=\s|[^\w\s])", name_venue): + name = name_venue[:boundary.start()].strip() + venue = name_venue[boundary.start():].strip() + if (len(re.findall(r"\w+", name)) >= 2 and len(compact(name)) >= 8 + and compact(name) == slug and len(re.findall(r"[A-Za-z]+", venue)) >= 2): + matches.append((len(compact(name)), name, venue)) + if not matches: + return None + _, name, venue = max(matches) + return name, [label[:end], venue, location] + + def _context_lines(context: str, title: str) -> list[str]: title_key = _key(title) + for card in markdown_links(context): + if card.is_image: + continue + recovered = serialized_link_card_fields(card) + if recovered and title_key in {_key(recovered[0]), _key(visible_markdown_links(card.label))}: + return recovered[1] + for card in reversed(markdown_links(context)): + if (card.is_image or "\n" not in card.label + ): + continue + label = card.label + for image in reversed(markdown_links(label)): + if image.is_image: + label = label[:image.start] + " " + label[image.end:] + if _key(visible_markdown_links(label)) != title_key: + continue + header = re.match(r"[ \t]*(?P\*\*|__)(?P[^\n]+?)(?P=mark)(?:\\)*[ \t]*\n", label) + if not header: + # Do not leave a multiline Markdown destination in a field line: + # dates in the URL are not a delivered visible date field. + context = context[:card.start] + visible_markdown_links(label) + context[card.end:] + continue + name = re.sub(r"\\([_*!\[\]()`])", r"\1", header["title"]).strip() + identity_key = lambda value: re.sub(r"[\W_]", "", _key(value)) + identities = {identity_key(name), identity_key(name.rsplit("/", 1)[-1]), + identity_key(unquote(urlsplit(card.destination).path.rsplit("/", 1)[-1]))} + parts = [] + author_pending = False + for raw in label[header.end():].splitlines(): + value = re.sub(r"\\+$", "", raw).strip() + value = re.sub(r"\\([_*!\[\]()`])", r"\1", value) + if not value or value == "/" or identity_key(value) in identities: + continue + if value.casefold() == "by": + author_pending = True + continue + if author_pending: + value = "Author: " + value + author_pending = False + parts.append(value) + context = context[:card.start] + "\n".join(parts) + context[card.end:] lines = [] for raw in clean_reference_text(context).splitlines(): + for link in reversed(markdown_links(raw)): + card_fields = serialized_link_card_fields(link) + label = card_fields[0] if card_fields else link.label + for image in reversed(markdown_links(label)): + if image.is_image: + label = label[:image.start] + " " + label[image.end:] + if link.is_image or not label.strip(): + raw = raw[:link.start] + " " + raw[link.end:] + elif label != link.label: + native_link = raw[link.start:link.end].replace(link.label, label, 1) + raw = raw[:link.start] + native_link + raw[link.end:] if re.match(r"^\s*#{1,6}\s+", raw): continue + # The linked title identifies the record; it cannot provide a + # missing source or summary beside a date on the same native line. + if title_key: + for link in reversed(markdown_links(raw)): + if not link.is_image and _key(visible_markdown_links(link.label)) == title_key: + raw = raw[:link.start] + " " + raw[link.end:] line = visible_markdown_links(raw) + # A closed bold numeric span followed by its unit exposes two + # adjacent visible nodes. Preserve their boundary for typed fields. + # Escaped or unclosed markers are literal text, not this format. + line = re.sub(rf"(?<![\\\w*])\*\*({_AMOUNT})\*\*(?=[A-Za-z])", + r"\1 ", line) line = re.sub(r"(?<![\w*_])([*_]{1,3})([^\s*_](?:[^*_\n]*?[^\s*_])?)\1(?![\w*_])", r"\2", line) line = line.strip(" \t#>*|-_") if not line or _key(line) == title_key or re.fullmatch(r"https?://\S+", line): @@ -657,6 +956,22 @@ def _field_token_pattern(spec: dict): 'version': _VERSION}.get(spec['kind']) +def _text_role_value(value: str, spec: dict) -> str | None: + """Explicit metadata belongs to its declared role, not another text field.""" + roles = {"title", "name", "author", "creator", "owner", "source", "publisher", + "description", "summary", "date", "time", "version", "downloads", + "views", "likes", "count", "price", "rating", "size"} + match = re.fullmatch(r"([A-Za-z][A-Za-z ]*)[ \t]*[:：][ \t]*(.*)", value) + role = _key(match[1]) if match else None + payload = match[2].strip() if match else value + if re.fullmatch(r"[^\n]+?\s*/\s*by\s+[^\n]+", value, re.I): + role = "author" + payload = re.split(r"\s*/\s*by\s+", value, maxsplit=1, flags=re.I)[1] + if role in roles: + accepted = {_key(spec["id"].replace("_", " ")), + *(_key(alias) for alias in spec.get("aliases", []))} + return payload if role in accepted and payload else None + return value def _declared_aliases(field_specs: list[dict]) -> list[str]: return sorted( {alias for spec in field_specs for alias in spec.get("aliases", [])}, @@ -695,9 +1010,12 @@ def _owned_labeled_values( if value: values.append((match.group("label"), value, start, end)) return values - unpunctuated = re.match(rf"(?i)^\s*{label}\s+(.+)$", line) + unpunctuated = re.match( + rf"(?i)^\s*(?P<label>{names})(?:\s*\([^)]*\)\s*(?P<qualified>\S.*)" + rf"|\s+(?!\s*\()(?P<plain>\S.*))$", line) if unpunctuated: - value, start, end = _trimmed_value_span(line, *unpunctuated.span(2)) + group = 'qualified' if unpunctuated['qualified'] is not None else 'plain' + value, start, end = _trimmed_value_span(line, *unpunctuated.span(group)) return [(unpunctuated.group("label"), value, start, end)] if value else [] return [] @@ -737,16 +1055,40 @@ def _field_value( continue candidates.append((index, line, 0, len(line))) for index, value, start, end in candidates: + if spec["kind"] == "text" and not aliases: + value = _text_role_value(value, spec) + if value is None: + continue if token_pattern := _field_token_pattern(spec): for match in token_pattern.finditer(value): if _value_matches_spec(match[0], spec): return index, match[0], start + match.start(), start + match.end() continue - if _value_matches_spec(value, spec): + checked_spec = {**spec, "minimum_chars": 1} if aliases and spec["kind"] == "text" and "minimum_chars" not in spec else spec + if _value_matches_spec(value, checked_spec): return index, value, start, end return None +def _labeled_field_lines(lines, field_specs): + """Keep explicit label/value spans separate within one native card line.""" + aliases = sorted({alias for spec in field_specs for alias in spec.get("aliases", [])}, + key=len, reverse=True) + if not aliases: + return lines + label = r"(?:" + "|".join(re.escape(alias) for alias in aliases) + r")[ \t]*[:：]" + labels = re.compile(r"(?<!\w)" + label, re.I) + result = [] + for line in lines: + # Only a line that starts with a declared labeled field can contain + # more inline fields. A label mentioned inside prose is not metadata. + if re.match(label, line, re.I): + starts = [match.start() for match in labels.finditer(line)] + ends = starts[1:] + [len(line)] + result.extend(line[start:end].strip() for start, end in zip(starts, ends)) + else: + result.append(line) + return result def _canonical_field_report_value(value: str, spec: dict) -> str: """Keep the typed scalar from an owned field cell for diagnostics/fixed values.""" visible = visible_markdown_links(value).strip().strip("| ") @@ -770,11 +1112,25 @@ def _fixed_field_value_key(value: str, spec: dict) -> str: def _required_fields(context: str, title: str, *, field_specs: list[dict]) -> bool: """Require every declared field inside the record block which owns the link.""" - lines = _context_lines(context, title) + lines = _labeled_field_lines(_context_lines(context, title), field_specs) + separated = [] + patterns = [re.compile(spec["pattern"], re.I) for spec in field_specs + if spec["kind"] == "text" and spec.get("pattern") and not spec.get("aliases")] + for line in lines: + for pattern in patterns: + match = pattern.match(line) + if match and match.end() and (match.end() == len(line) or not line[match.end()].isalnum()): + rest = line[match.end():].strip(" |\t") + if rest: + separated.extend((match[0], rest)) + break + else: + separated.append(line) + lines = separated used: set[int] = set() typed_lines: set[int] = set() declared = _declared_aliases(field_specs) - for spec in field_specs: + for spec in sorted(field_specs, key=lambda field: not (field.get("aliases") or field["kind"] == "text" and field.get("pattern"))): typed = _field_token_pattern(spec) is not None result = _field_value( lines, @@ -993,7 +1349,15 @@ def _linked_table_group(content: str, group: dict, base_url: str) -> dict: def _field_report_group(content: str, group: dict) -> dict: - lines = _context_lines(content, "") + # A visible linked value still belongs to its preceding declared label. + # Preserve that explicit markup boundary without splitting plain words. + aliases = _declared_aliases(group["required_fields"]) + if aliases: + names = "|".join(re.escape(alias) for alias in aliases) + content = re.sub(rf"(?im)^(\s*(?:{names}))(?=\[)", r"\1 ", content) + lines = [re.sub(r"\s*\|\s*", " ", line).strip() + for line in _context_lines(content, "")] + lines = _labeled_field_lines(lines, group["required_fields"]) fields = {} used = set() declared = _declared_aliases(group["required_fields"]) @@ -1018,17 +1382,12 @@ def _field_report_group(content: str, group: dict) -> dict: passed = len(fields) == len(group["required_fields"]) and all( result["pass"] for result in fixed.values() ) - return { - "count": None, - "minimum": None, - "records": [{"fields": fields}], - "fixed_value_results": fixed, - "pass": passed, - } + return {"count": None, "minimum": None, "records": [{"fields": fields}], + "fixed_value_results": fixed, "pass": passed} def evaluate_composite_acceptance( - content: str, contract: dict, *, base_url: str, links: list | None = None, eligible: bool = True, table_content: str | None = None + content: str, contract: dict, *, base_url: str, links: list | None = None, eligible: bool = True, table_content: str | None = None, native_html: str | None = None ) -> dict: validate_composite_acceptance(contract) results = [] @@ -1045,6 +1404,7 @@ def evaluate_composite_acceptance( if group["kind"] == "named_link": result = _named_link_group( scoped, group, base_url, links if scoped == content else None, + native_html=native_html, ownership_marker=ownership_marker, ) elif group["kind"] == "linked_table": diff --git a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json index 378cdf5..c158c26 100644 --- a/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json +++ b/src/lexbench_webfetch/definitions/evaluation/scoring-regressions.json @@ -11,14 +11,27 @@ "url": "https://example.test/tree", "ordered_tree_reference": { "tag": "table", - "children": [{"tag": "tr", "children": [{"tag": "td", "text": "A", "children": []}]}] + "children": [ + { + "tag": "tr", + "children": [ + { + "tag": "td", + "text": "A", + "children": [] + } + ] + } + ] }, "ordered_tree_min_similarity": 1.0 }, "artifact": "{\"tag\":\"table\",\"children\":[{\"tag\":\"tr\",\"children\":[{\"tag\":\"td\",\"text\":\"A\",\"children\":[]}]}]}", "ok": true, "expected_task_pass": true, - "expected_metrics": {"ordered_tree_similarity": 1.0} + "expected_metrics": { + "ordered_tree_similarity": 1.0 + } }, { "id": "ordered-tree-missing-children-cannot-be-perfect", @@ -29,14 +42,27 @@ "url": "https://example.test/tree", "ordered_tree_reference": { "tag": "table", - "children": [{"tag": "tr", "children": [{"tag": "td", "text": "A", "children": []}]}] + "children": [ + { + "tag": "tr", + "children": [ + { + "tag": "td", + "text": "A", + "children": [] + } + ] + } + ] }, "ordered_tree_min_similarity": 1.0 }, "artifact": "{\"tag\":\"table\",\"children\":[]}", "ok": true, "expected_task_pass": false, - "expected_metrics": {"ordered_tree_similarity": 0.33333333333333337} + "expected_metrics": { + "ordered_tree_similarity": 0.33333333333333337 + } }, { "id": "wikimedia-thumbnail-host-migration-keeps-same-asset", @@ -47,12 +73,25 @@ "url": "https://en.wikipedia.org/wiki/Apollo_11", "task_kind": "resource", "content_score_applicable": false, - "expected_resources": [{"id": "moon", "urls": ["https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon.jpg/330px-A_Man_on_the_Moon.jpg?utm_source=en.wikipedia.org"]}] + "expected_resources": [ + { + "id": "moon", + "urls": [ + "https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon.jpg/330px-A_Man_on_the_Moon.jpg?utm_source=en.wikipedia.org" + ], + "aliases": [ + "https://thumb.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon.jpg/960px-A_Man_on_the_Moon.jpg?utm_campaign=parser" + ] + } + ] }, "artifact": "https://thumb.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon.jpg/960px-A_Man_on_the_Moon.jpg?utm_campaign=parser", "ok": true, "expected_task_pass": true, - "expected_metrics": {"resource_pass": true, "resource_hits": 1} + "expected_metrics": { + "resource_pass": true, + "resource_hits": 1 + } }, { "id": "wikimedia-thumbnail-host-migration-does-not-hide-wrong-asset", @@ -63,12 +102,25 @@ "url": "https://en.wikipedia.org/wiki/Apollo_11", "task_kind": "resource", "content_score_applicable": false, - "expected_resources": [{"id": "moon", "urls": ["https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon.jpg/330px-A_Man_on_the_Moon.jpg?utm_source=en.wikipedia.org"]}] + "expected_resources": [ + { + "id": "moon", + "urls": [ + "https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon.jpg/330px-A_Man_on_the_Moon.jpg?utm_source=en.wikipedia.org" + ], + "aliases": [ + "https://thumb.wikimedia.org/wikipedia/commons/thumb/4/41/A_Man_on_the_Moon.jpg/960px-A_Man_on_the_Moon.jpg?utm_campaign=parser" + ] + } + ] }, "artifact": "https://thumb.wikimedia.org/wikipedia/commons/thumb/4/42/A_Man_on_the_Moon.jpg/960px-A_Man_on_the_Moon.jpg?utm_campaign=parser", "ok": true, "expected_task_pass": false, - "expected_metrics": {"resource_pass": false, "resource_hits": 0} + "expected_metrics": { + "resource_pass": false, + "resource_hits": 0 + } }, { "id": "wikimedia-thumb-host-cannot-claim-original-file", @@ -79,12 +131,22 @@ "url": "https://en.wikipedia.org/wiki/Apollo_11", "task_kind": "resource", "content_score_applicable": false, - "expected_resources": [{"id": "original", "urls": ["https://upload.wikimedia.org/wikipedia/commons/4/41/A_Man_on_the_Moon.jpg"]}] + "expected_resources": [ + { + "id": "original", + "urls": [ + "https://upload.wikimedia.org/wikipedia/commons/4/41/A_Man_on_the_Moon.jpg" + ] + } + ] }, "artifact": "https://thumb.wikimedia.org/wikipedia/commons/4/41/A_Man_on_the_Moon.jpg", "ok": true, "expected_task_pass": false, - "expected_metrics": {"resource_pass": false, "resource_hits": 0} + "expected_metrics": { + "resource_pass": false, + "resource_hits": 0 + } }, { "id": "markdown-table-rule-does-not-break-visible-calendar-fact", @@ -93,12 +155,21 @@ "failure_mode": "false-negative", "case": { "url": "https://example.test/calendar", - "required_facts": [{"id": "calendar", "alternatives": ["December 2007 Sun Mon Tue Wed Thu Fri Sat"]}] + "required_facts": [ + { + "id": "calendar", + "alternatives": [ + "December 2007 Sun Mon Tue Wed Thu Fri Sat" + ] + } + ] }, "artifact": "| December 2007 | | | | | | |\n| ------------- | | --- | | --- | | --- |\n| Sun | Mon | Tue | Wed | Thu | Fri | Sat |", "ok": true, "expected_task_pass": true, - "expected_metrics": {"fact_coverage": 1.0} + "expected_metrics": { + "fact_coverage": 1.0 + } }, { "id": "markdown-table-rule-cannot-hide-wrong-calendar-fact", @@ -107,12 +178,21 @@ "failure_mode": "false-positive", "case": { "url": "https://example.test/calendar", - "required_facts": [{"id": "calendar", "alternatives": ["December 2007 Sun Mon Tue Wed Thu Fri Sat"]}] + "required_facts": [ + { + "id": "calendar", + "alternatives": [ + "December 2007 Sun Mon Tue Wed Thu Fri Sat" + ] + } + ] }, "artifact": "| December 2008 | | | | | | |\n| ------------- | | --- | | --- | | --- |\n| Sun | Mon | Tue | Wed | Thu | Fri | Sat |", "ok": true, "expected_task_pass": false, - "expected_metrics": {"fact_coverage": 0.0} + "expected_metrics": { + "fact_coverage": 0.0 + } }, { "id": "answer-title-supplies-required-fact", @@ -121,13 +201,23 @@ "failure_mode": "false-negative", "case": { "url": "https://example.test/page", - "required_facts": [{"id": "title", "alternatives": ["Exact page title"]}] + "required_facts": [ + { + "id": "title", + "alternatives": [ + "Exact page title" + ] + } + ] }, "artifact": "Body", "title": "Exact page title", "ok": true, "expected_task_pass": true, - "expected_metrics": {"fact_coverage": 1.0, "content_chars": 4} + "expected_metrics": { + "fact_coverage": 1.0, + "content_chars": 4 + } }, { "id": "answer-title-does-not-inflate-body-length", @@ -136,14 +226,25 @@ "failure_mode": "false-positive", "case": { "url": "https://example.test/page", - "required_facts": [{"id": "title", "alternatives": ["Exact page title"]}], + "required_facts": [ + { + "id": "title", + "alternatives": [ + "Exact page title" + ] + } + ], "min_content_chars": 20 }, "artifact": "Body", "title": "Exact page title", "ok": true, "expected_task_pass": false, - "expected_metrics": {"fact_coverage": 1.0, "length_pass": false, "content_chars": 4} + "expected_metrics": { + "fact_coverage": 1.0, + "length_pass": false, + "content_chars": 4 + } }, { "id": "balanced-multiline-markdown-keeps-visible-label", @@ -152,12 +253,21 @@ "failure_mode": "false-negative", "case": { "url": "https://example.test/page", - "required_facts": [{"id": "label", "alternatives": ["Download guide now"]}] + "required_facts": [ + { + "id": "label", + "alternatives": [ + "Download guide now" + ] + } + ] }, "artifact": "[Download **guide**\nnow]\n(/files/guide(v2).pdf?download=1 \"PDF copy\")", "ok": true, "expected_task_pass": true, - "expected_metrics": {"fact_coverage": 1.0} + "expected_metrics": { + "fact_coverage": 1.0 + } }, { "id": "explicit-named-links-form-records", @@ -171,16 +281,28 @@ "version": 1, "schema": "named_link", "minimum_records": 2, - "hosts": ["example.test"], + "hosts": [ + "example.test" + ], "entry_path": "/item/[0-9]+", "entry_context": "list_or_card", - "stable_text_anchors": [{"id": "heading", "alternatives": ["Catalog"]}] + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Catalog" + ] + } + ] } }, "artifact": "Catalog\n- [Alpha](https://example.test/item/1)\n- [Beta](https://example.test/item/2)", "ok": true, "expected_task_pass": true, - "expected_metrics": {"stable_structure_pass": true, "stable_content_pass": true} + "expected_metrics": { + "stable_structure_pass": true, + "stable_content_pass": true + } }, { "id": "prefix-field-belongs-to-following-product-card", @@ -193,14 +315,26 @@ "version": 1, "schema": "product", "minimum_records": 2, - "field_minimum_records": {"shipping": 2}, + "field_minimum_records": { + "shipping": 2 + }, "stable_text_anchors": [] } }, "artifact": "Free shipping\nAlpha\n$10\n\nFree shipping\nBeta\n$20", "ok": true, "expected_task_pass": true, - "expected_metrics": {"stable_structure_pass": true, "stable_field_results": [{"field": "shipping", "count": 2, "minimum": 2, "pass": true}]} + "expected_metrics": { + "stable_structure_pass": true, + "stable_field_results": [ + { + "field": "shipping", + "count": 2, + "minimum": 2, + "pass": true + } + ] + } }, { "id": "prefix-field-cannot-repair-previous-product-card", @@ -213,14 +347,26 @@ "version": 1, "schema": "product", "minimum_records": 2, - "field_minimum_records": {"shipping": 2}, + "field_minimum_records": { + "shipping": 2 + }, "stable_text_anchors": [] } }, "artifact": "Alpha\n$10\n\nFree shipping\nBeta\n$20", "ok": true, "expected_task_pass": false, - "expected_metrics": {"stable_structure_pass": false, "stable_field_results": [{"field": "shipping", "count": 1, "minimum": 2, "pass": false}]} + "expected_metrics": { + "stable_structure_pass": false, + "stable_field_results": [ + { + "field": "shipping", + "count": 1, + "minimum": 2, + "pass": false + } + ] + } }, { "id": "separate-names-and-bare-urls-are-not-records", @@ -234,17 +380,35 @@ "version": 1, "schema": "named_link", "minimum_records": 2, - "hosts": ["example.test"], + "hosts": [ + "example.test" + ], "entry_path": "/item/[0-9]+", "entry_context": "list_or_card", - "stable_text_anchors": [{"id": "heading", "alternatives": ["Catalog"]}] + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Catalog" + ] + } + ] } }, "artifact": "Catalog\nAlpha\nBeta\nhttps://example.test/item/1\nhttps://example.test/item/2", - "metadata": {"extras": {"links": ["https://example.test/item/1", "https://example.test/item/2"]}}, + "metadata": { + "extras": { + "links": [ + "https://example.test/item/1", + "https://example.test/item/2" + ] + } + }, "ok": true, "expected_task_pass": false, - "expected_metrics": {"stable_structure_pass": false} + "expected_metrics": { + "stable_structure_pass": false + } }, { "id": "complete-distributed-body-passes", @@ -261,15 +425,23 @@ "minimum_floor": 0.6, "minimum_region_passes": 2, "regions": [ - {"id": "opening", "text": "alpha bravo charlie delta"}, - {"id": "ending", "text": "echo foxtrot golf hotel"} + { + "id": "opening", + "text": "alpha bravo charlie delta" + }, + { + "id": "ending", + "text": "echo foxtrot golf hotel" + } ] } }, "artifact": "alpha bravo charlie delta echo foxtrot golf hotel", "ok": true, "expected_task_pass": true, - "expected_metrics": {"body_coverage_pass": true} + "expected_metrics": { + "body_coverage_pass": true + } }, { "id": "short-prefix-cannot-pass-full-body-task", @@ -286,15 +458,23 @@ "minimum_floor": 0.6, "minimum_region_passes": 2, "regions": [ - {"id": "opening", "text": "alpha bravo charlie delta"}, - {"id": "ending", "text": "echo foxtrot golf hotel"} + { + "id": "opening", + "text": "alpha bravo charlie delta" + }, + { + "id": "ending", + "text": "echo foxtrot golf hotel" + } ] } }, "artifact": "alpha bravo charlie delta", "ok": true, "expected_task_pass": false, - "expected_metrics": {"body_coverage_pass": false} + "expected_metrics": { + "body_coverage_pass": false + } }, { "id": "overlapping-body-regions-use-global-capacity", @@ -310,15 +490,24 @@ "minimum_floor": 0.6, "minimum_region_passes": 1, "regions": [ - {"id": "opening", "text": "abcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxAAAAAAAABBBBBBBB"}, - {"id": "ending", "text": "abcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxCCCCCCCCDDDDDDDD"} + { + "id": "opening", + "text": "abcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxAAAAAAAABBBBBBBB" + }, + { + "id": "ending", + "text": "abcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxCCCCCCCCDDDDDDDD" + } ] } }, "artifact": "abcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxabcdefAAAAAAAABBBBBBBBCCCCCCCCDDDDDDDD", "ok": true, "expected_task_pass": true, - "expected_metrics": {"body_coverage_pass": true, "body_target_regions": 1} + "expected_metrics": { + "body_coverage_pass": true, + "body_target_regions": 1 + } }, { "id": "one-body-fragment-instance-cannot-fill-two-regions", @@ -334,15 +523,23 @@ "minimum_floor": 0.6, "minimum_region_passes": 1, "regions": [ - {"id": "opening", "text": "abcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwx"}, - {"id": "ending", "text": "abcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwx"} + { + "id": "opening", + "text": "abcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwx" + }, + { + "id": "ending", + "text": "abcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwx" + } ] } }, "artifact": "abcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwxabcdefghijklmnopqrstuvwx", "ok": true, "expected_task_pass": false, - "expected_metrics": {"body_coverage_pass": false} + "expected_metrics": { + "body_coverage_pass": false + } }, { "id": "declared-truncation-cannot-pass-full-body-task", @@ -351,13 +548,24 @@ "failure_mode": "false-positive", "case": { "url": "https://example.test/article", - "required_facts": [{"id": "anchor", "alternatives": ["required anchor"]}], + "required_facts": [ + { + "id": "anchor", + "alternatives": [ + "required anchor" + ] + } + ], "min_content_chars": 20 }, "artifact": "required anchor and enough retained prefix text\n\n(Content truncated. Fetch a more specific URL or section for the full text.)", "ok": true, "expected_task_pass": false, - "expected_metrics": {"declared_truncation_hits": ["(Content truncated. Fetch a more specific URL or section for the full text.)"]} + "expected_metrics": { + "declared_truncation_hits": [ + "(Content truncated. Fetch a more specific URL or section for the full text.)" + ] + } }, { "id": "ordinary-truncation-prose-does-not-fail-body-task", @@ -366,13 +574,22 @@ "failure_mode": "false-negative", "case": { "url": "https://example.test/article", - "required_facts": [{"id": "anchor", "alternatives": ["required anchor"]}], + "required_facts": [ + { + "id": "anchor", + "alternatives": [ + "required anchor" + ] + } + ], "min_content_chars": 20 }, "artifact": "required anchor. This article explains how content truncated by a parser can be recovered.", "ok": true, "expected_task_pass": true, - "expected_metrics": {"declared_truncation_hits": []} + "expected_metrics": { + "declared_truncation_hits": [] + } }, { "id": "exact-body-regions-survive-declared-truncation-notice", @@ -386,13 +603,23 @@ "version": 1, "unit": "region_token_recall", "minimum_coverage": 0.5, - "regions": [{"id": "body", "text": "alpha bravo charlie delta"}] + "regions": [ + { + "id": "body", + "text": "alpha bravo charlie delta" + } + ] } }, "artifact": "alpha bravo charlie delta\n[Content truncated: showing 10 of 20 lines]", "ok": true, "expected_task_pass": true, - "expected_metrics": {"body_coverage_pass": true, "declared_truncation_hits": ["[Content truncated: showing 10 of 20 lines]"]} + "expected_metrics": { + "body_coverage_pass": true, + "declared_truncation_hits": [ + "[Content truncated: showing 10 of 20 lines]" + ] + } }, { "id": "reference-captcha-heading-is-legitimate-body-text", @@ -402,26 +629,25 @@ "case": { "url": "https://example.test/accessibility", "ground_truth_main_content": "Accessibility guidance\nCAPTCHA\nProvide an equivalent text alternative.", - "required_facts": [{"id": "guidance", "alternatives": ["Provide an equivalent text alternative"]}] + "required_facts": [ + { + "id": "guidance", + "alternatives": [ + "Provide an equivalent text alternative" + ] + } + ] }, "artifact": "Accessibility guidance\nCAPTCHA\nProvide an equivalent text alternative.", "ok": true, "expected_task_pass": true, - "expected_metrics": {"blocked_page": false, "blocked_page_hits": [], "blocked_page_reference_exemptions": ["CAPTCHA"]} - }, - { - "id": "standalone-captcha-without-reference-remains-blocked", - "category": "task-predicate", - "root_cause": "blocked_page_detection", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/accessibility", - "ground_truth_main_content": "Accessibility guidance" - }, - "artifact": "CAPTCHA", - "ok": true, - "expected_task_pass": false, - "expected_metrics": {"blocked_page": true, "blocked_page_hits": ["CAPTCHA"], "blocked_page_reference_exemptions": []} + "expected_metrics": { + "blocked_page": false, + "blocked_page_hits": [], + "blocked_page_reference_exemptions": [ + "CAPTCHA" + ] + } }, { "id": "complete-records-with-declared-excluded-ui-fail", @@ -430,7 +656,9 @@ "failure_mode": "false-positive", "case": { "url": "https://example.test/products", - "forbidden_any": ["Similar Products"], + "forbidden_any": [ + "Similar Products" + ], "acceptance_contract": { "version": 1, "schema": "product", @@ -441,7 +669,12 @@ "artifact": "Alpha\n$10.00\n\nBeta\n$20.00\n\nSimilar Products\nGamma\n$30.00", "ok": true, "expected_task_pass": false, - "expected_metrics": {"forbidden_hits": ["Similar Products"], "stable_structure_pass": true} + "expected_metrics": { + "forbidden_requirement_hits": [ + "Similar Products" + ], + "stable_structure_pass": true + } }, { "id": "target-only-records-pass-declared-exclusions", @@ -450,7 +683,9 @@ "failure_mode": "false-negative", "case": { "url": "https://example.test/products", - "forbidden_any": ["Similar Products"], + "forbidden_any": [ + "Similar Products" + ], "acceptance_contract": { "version": 1, "schema": "product", @@ -461,7 +696,10 @@ "artifact": "Alpha\n$10.00\n\nBeta\n$20.00", "ok": true, "expected_task_pass": true, - "expected_metrics": {"forbidden_hits": [], "stable_structure_pass": true} + "expected_metrics": { + "forbidden_requirement_hits": [], + "stable_structure_pass": true + } }, { "id": "glued-quote-fields-cannot-borrow-boundaries", @@ -475,13 +713,22 @@ "version": 1, "schema": "quote", "minimum_records": 1, - "stable_text_anchors": [{"id": "quote", "alternatives": ["The world as we have created it is a process of our thinking. by Albert Einstein Tags: change deep-thoughts thinking world"]}] + "stable_text_anchors": [ + { + "id": "quote", + "alternatives": [ + "The world as we have created it is a process of our thinking. by Albert Einstein Tags: change deep-thoughts thinking world" + ] + } + ] } }, "artifact": "\"The world as we have created it is a process of our thinking.\"byAlbert EinsteinTags:changedeep-thoughtsthinkingworld", "ok": true, "expected_task_pass": false, - "expected_metrics": {"stable_content_pass": false} + "expected_metrics": { + "stable_content_pass": false + } }, { "id": "nested-linked-image-is-explicit-resource", @@ -492,12 +739,21 @@ "url": "https://example.test/gallery", "task_kind": "resource", "content_score_applicable": false, - "expected_resources": [{"id": "image", "urls": ["/media/moon(large).jpg"]}] + "expected_resources": [ + { + "id": "image", + "urls": [ + "/media/moon(large).jpg" + ] + } + ] }, "artifact": "[![Moon](//cdn.example.test/thumb.jpg)](/media/moon(large).jpg)", "ok": true, "expected_task_pass": true, - "expected_metrics": {"resource_pass": true} + "expected_metrics": { + "resource_pass": true + } }, { "id": "input-url-echo-is-not-resource-discovery", @@ -508,12 +764,21 @@ "url": "https://example.test/media/moon.jpg", "task_kind": "resource", "content_score_applicable": false, - "expected_resources": [{"id": "image", "urls": ["https://example.test/media/moon.jpg"]}] + "expected_resources": [ + { + "id": "image", + "urls": [ + "https://example.test/media/moon.jpg" + ] + } + ] }, "artifact": "https://example.test/media/moon.jpg", "ok": true, "expected_task_pass": false, - "expected_metrics": {"resource_pass": false} + "expected_metrics": { + "resource_pass": false + } }, { "id": "same-entity-tracking-links-do-not-inflate-count", @@ -527,17 +792,28 @@ "version": 1, "schema": "named_link", "minimum_records": 2, - "hosts": ["amazon.example.test"], + "hosts": [ + "amazon.example.test" + ], "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", "entry_context": "list_or_card", - "stable_text_anchors": [{"id": "heading", "alternatives": ["Headphones"]}] + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Headphones" + ] + } + ] } }, "artifact": "Headphones\n- [Alpha](/alpha/dp/B012345678/ref=one)\n- [Alpha offer](/alpha/dp/B012345678/ref=two)", "ok": true, "expected_task_pass": false, - "expected_metrics": {"stable_structure_pass": false} + "expected_metrics": { + "stable_structure_pass": false + } }, { "id": "distinct-entity-links-meet-count", @@ -551,17 +827,28 @@ "version": 1, "schema": "named_link", "minimum_records": 2, - "hosts": ["amazon.example.test"], + "hosts": [ + "amazon.example.test" + ], "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", "entry_context": "list_or_card", - "stable_text_anchors": [{"id": "heading", "alternatives": ["Headphones"]}] + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Headphones" + ] + } + ] } }, "artifact": "Headphones\n- [Alpha](/alpha/dp/B012345678/ref=one)\n- [Beta](/beta/dp/B087654321/ref=two)", "ok": true, "expected_task_pass": true, - "expected_metrics": {"stable_structure_pass": true} + "expected_metrics": { + "stable_structure_pass": true + } }, { "id": "successful-empty-response-is-not-task-success", @@ -570,12 +857,22 @@ "failure_mode": "false-positive", "case": { "url": "https://example.test/page", - "required_facts": [{"id": "fact", "alternatives": ["Required text"]}] - }, - "artifact": "", - "ok": true, - "expected_task_pass": false, - "expected_metrics": {"content_pass": false, "content_check_coverage": 0.0} + "required_facts": [ + { + "id": "fact", + "alternatives": [ + "Required text" + ] + } + ] + }, + "artifact": "", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "content_pass": false, + "content_check_coverage": 0.0 + } }, { "id": "reference-scope-retains-main-content", @@ -584,12 +881,21 @@ "failure_mode": "false-negative", "case": { "url": "https://example.test/article", - "required_facts": [{"id": "main", "alternatives": ["Independent article text"]}] + "required_facts": [ + { + "id": "main", + "alternatives": [ + "Independent article text" + ] + } + ] }, "artifact": "Independent article text", "ok": true, "expected_task_pass": true, - "expected_metrics": {"fact_coverage": 1.0} + "expected_metrics": { + "fact_coverage": 1.0 + } }, { "id": "reference-scope-excludes-site-chrome", @@ -598,12 +904,21 @@ "failure_mode": "false-positive", "case": { "url": "https://example.test/article", - "required_facts": [{"id": "main", "alternatives": ["Independent article text"]}] + "required_facts": [ + { + "id": "main", + "alternatives": [ + "Independent article text" + ] + } + ] }, "artifact": "Cookie settings navigation", "ok": true, "expected_task_pass": false, - "expected_metrics": {"fact_coverage": 0.0} + "expected_metrics": { + "fact_coverage": 0.0 + } }, { "id": "complete-declared-record-passes-contract", @@ -622,7 +937,9 @@ "artifact": "Alpha phone\nA brighter display.\n\nBeta phone\nA different camera.", "ok": true, "expected_task_pass": true, - "expected_metrics": {"stable_structure_pass": true} + "expected_metrics": { + "stable_structure_pass": true + } }, { "id": "incomplete-record-fails-declared-contract", @@ -641,7 +958,9 @@ "artifact": "Alpha phone\n\nBeta phone", "ok": true, "expected_task_pass": false, - "expected_metrics": {"stable_structure_pass": false} + "expected_metrics": { + "stable_structure_pass": false + } }, { "id": "target-only-image-answer-rejects-unexpected-image", @@ -652,21 +971,37 @@ "url": "https://example.test/gallery", "task_kind": "resource", "content_score_applicable": false, - "expected_resources": [{"id": "hero", "urls": ["/media/hero.jpg"]}], + "expected_resources": [ + { + "id": "hero", + "urls": [ + "/media/hero.jpg" + ] + } + ], "resource_output_contract": { "candidate_scope": "image_answer_fields", "unexpected": "reject" } }, "artifact": "", - "metadata": {"extras": {"imageLinks": ["/media/hero.jpg", "/media/logo.png"]}}, + "metadata": { + "extras": { + "imageLinks": [ + "/media/hero.jpg", + "/media/logo.png" + ] + } + }, "ok": true, "expected_task_pass": false, "expected_metrics": { "resource_recall": 1.0, "resource_precision": 0.5, "resource_pass": false, - "unexpected_resource_urls": ["https://example.test/media/logo.png"] + "unexpected_resource_urls": [ + "https://example.test/media/logo.png" + ] } }, { @@ -678,11 +1013,17 @@ "url": "https://example.test/gallery", "task_kind": "resource", "content_score_applicable": false, - "expected_resources": [{ - "id": "hero", - "urls": ["/media/hero-large.jpg"], - "aliases": ["/media/hero-480w.jpg"] - }], + "expected_resources": [ + { + "id": "hero", + "urls": [ + "/media/hero-large.jpg" + ], + "aliases": [ + "/media/hero-480w.jpg" + ] + } + ], "resource_output_contract": { "candidate_scope": "image_answer_fields", "unexpected": "reject" @@ -705,13 +1046,23 @@ "failure_mode": "false-negative", "case": { "url": "https://example.test/article", - "required_facts": [{"id": "title", "alternatives": ["Fearless Concurrency"]}], + "required_facts": [ + { + "id": "title", + "alternatives": [ + "Fearless Concurrency" + ] + } + ], "min_content_chars": 20 }, "artifact": "Source navigation\nFearless Concurrency\nComplete article body.", "ok": true, "expected_task_pass": true, - "expected_metrics": {"fact_coverage": 1.0, "length_pass": true} + "expected_metrics": { + "fact_coverage": 1.0, + "length_pass": true + } }, { "id": "semantic-title-remains-required", @@ -720,13 +1071,23 @@ "failure_mode": "false-positive", "case": { "url": "https://example.test/article", - "required_facts": [{"id": "title", "alternatives": ["Fearless Concurrency"]}], + "required_facts": [ + { + "id": "title", + "alternatives": [ + "Fearless Concurrency" + ] + } + ], "min_content_chars": 20 }, "artifact": "Source navigation\nComplete unrelated article body.", "ok": true, "expected_task_pass": false, - "expected_metrics": {"fact_coverage": 0.0, "length_pass": true} + "expected_metrics": { + "fact_coverage": 0.0, + "length_pass": true + } }, { "id": "complete-records-allow-supplemental-ui", @@ -745,7 +1106,10 @@ "artifact": "Alpha shirt\n$10\n\nBeta shirt\n$20\n\nAdd to cart", "ok": true, "expected_task_pass": true, - "expected_metrics": {"stable_structure_pass": true, "stable_content_pass": true} + "expected_metrics": { + "stable_structure_pass": true, + "stable_content_pass": true + } }, { "id": "supplemental-ui-cannot-repair-incomplete-records", @@ -764,7 +1128,9 @@ "artifact": "Alpha shirt\n$10\n\nBeta shirt\n\nAdd to cart", "ok": true, "expected_task_pass": false, - "expected_metrics": {"stable_structure_pass": false} + "expected_metrics": { + "stable_structure_pass": false + } }, { "id": "quote-pagination-stays-outside-tags", @@ -778,16 +1144,23 @@ "version": 1, "schema": "quote", "minimum_records": 1, - "stable_text_anchors": [{ - "id": "quote", - "alternatives": ["\u201cA real quote.\u201d\nby Real Author\nTags: first second"] - }] + "stable_text_anchors": [ + { + "id": "quote", + "alternatives": [ + "“A real quote.”\nby Real Author\nTags: first second" + ] + } + ] } }, - "artifact": "\u201cA real quote.\u201d\nby Real Author\nTags: first second\nNext \u2192", + "artifact": "“A real quote.”\nby Real Author\nTags: first second\nNext →", "ok": true, "expected_task_pass": true, - "expected_metrics": {"stable_structure_pass": true, "stable_content_pass": true} + "expected_metrics": { + "stable_structure_pass": true, + "stable_content_pass": true + } }, { "id": "pagination-cannot-supply-a-missing-tag", @@ -801,32 +1174,23 @@ "version": 1, "schema": "quote", "minimum_records": 1, - "stable_text_anchors": [{ - "id": "quote", - "alternatives": ["\u201cA real quote.\u201d\nby Real Author\nTags: first next"] - }] + "stable_text_anchors": [ + { + "id": "quote", + "alternatives": [ + "“A real quote.”\nby Real Author\nTags: first next" + ] + } + ] } }, - "artifact": "\u201cA real quote.\u201d\nby Real Author\nTags: first\nNext \u2192", - "ok": true, - "expected_task_pass": false, - "expected_metrics": {"stable_structure_pass": true, "stable_content_pass": false} - }, - { - "id": "known-challenge-headings-block-complete-records", - "category": "task-predicate", - "root_cause": "blocked_page_detection", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/gallery", - "task_kind": "resource", - "content_score_applicable": false, - "expected_resources": [{"id": "hero", "urls": ["/hero.jpg"]}] - }, - "artifact": "W3 Performing security verification\n![Hero](/hero.jpg)", + "artifact": "“A real quote.”\nby Real Author\nTags: first\nNext →", "ok": true, "expected_task_pass": false, - "expected_metrics": {"resource_pass": true, "blocked_page": true} + "expected_metrics": { + "stable_structure_pass": true, + "stable_content_pass": false + } }, { "id": "challenge-phrase-in-prose-is-not-a-block-page", @@ -837,28 +1201,22 @@ "url": "https://example.test/gallery", "task_kind": "resource", "content_score_applicable": false, - "expected_resources": [{"id": "hero", "urls": ["/hero.jpg"]}] + "expected_resources": [ + { + "id": "hero", + "urls": [ + "/hero.jpg" + ] + } + ] }, "artifact": "This guide explains why a site may say Making sure you're not a bot.\n![Hero](/hero.jpg)", "ok": true, "expected_task_pass": true, - "expected_metrics": {"resource_pass": true, "blocked_page": false} - }, - { - "id": "unsplash-bot-heading-blocks-complete-records", - "category": "task-predicate", - "root_cause": "blocked_page_detection", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/gallery", - "task_kind": "resource", - "content_score_applicable": false, - "expected_resources": [{"id": "hero", "urls": ["/hero.jpg"]}] - }, - "artifact": "Making sure you're not a bot\n![Hero](/hero.jpg)", - "ok": true, - "expected_task_pass": false, - "expected_metrics": {"resource_pass": true, "blocked_page": true} + "expected_metrics": { + "resource_pass": true, + "blocked_page": false + } }, { "id": "complete-dynamic-products-pass-output-contract", @@ -877,7 +1235,10 @@ "artifact": "Alpha hoodie\n$52\n\nBeta hoodie\n$70", "ok": true, "expected_task_pass": true, - "expected_metrics": {"stable_complete_record_count": 2, "stable_structure_pass": true} + "expected_metrics": { + "stable_complete_record_count": 2, + "stable_structure_pass": true + } }, { "id": "empty-placeholder-products-fail-output-contract", @@ -896,7 +1257,10 @@ "artifact": "Loading\n\nLoading", "ok": true, "expected_task_pass": false, - "expected_metrics": {"stable_complete_record_count": 0, "stable_structure_pass": false} + "expected_metrics": { + "stable_complete_record_count": 0, + "stable_structure_pass": false + } }, { "id": "semantic-reference-fact-survives-visual-line-wrap", @@ -905,15 +1269,21 @@ "failure_mode": "false-negative", "case": { "url": "https://example.test/help", - "required_facts": [{ - "id": "purchase-requirement", - "alternatives": ["飞书 AI 版本需企业管理员另外购买"] - }] + "required_facts": [ + { + "id": "purchase-requirement", + "alternatives": [ + "飞书 AI 版本需企业管理员另外购买" + ] + } + ] }, "artifact": "增值信息：飞书 AI 版本需企业管理员另外购买。如需购买，请\n联系客服\n或前往\n飞书管理后台\n>\n资产管理。", "ok": true, "expected_task_pass": true, - "expected_metrics": {"fact_coverage": 1.0} + "expected_metrics": { + "fact_coverage": 1.0 + } }, { "id": "nearby-purchase-navigation-cannot-replace-semantic-fact", @@ -922,38 +1292,20 @@ "failure_mode": "false-positive", "case": { "url": "https://example.test/help", - "required_facts": [{ - "id": "purchase-requirement", - "alternatives": ["飞书 AI 版本需企业管理员另外购买"] - }] + "required_facts": [ + { + "id": "purchase-requirement", + "alternatives": [ + "飞书 AI 版本需企业管理员另外购买" + ] + } + ] }, "artifact": "如需购买，请联系客服或前往飞书管理后台 > 资产管理 > AI 使用管理 > 用量概览自助购买。", "ok": true, "expected_task_pass": false, - "expected_metrics": {"fact_coverage": 0.0} - }, - { - "id": "chinese-gate-heading-blocks-complete-records", - "category": "task-predicate", - "root_cause": "blocked_page_detection", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/catalog", - "task_kind": "structure", - "acceptance_contract": { - "version": 1, - "schema": "product", - "minimum_records": 2 - } - }, - "artifact": "![](/shield.png)验证一下，购物无忧\nAlpha shirt\n$10\n\nBeta shirt\n$20", - "ok": true, - "expected_task_pass": false, "expected_metrics": { - "blocked_page": true, - "stable_blocked_page": true, - "stable_structure_pass": true, - "stable_content_pass": false + "fact_coverage": 0.0 } }, { @@ -992,7 +1344,9 @@ "version": 1, "schema": "named_link", "minimum_records": 2, - "hosts": ["example.test"], + "hosts": [ + "example.test" + ], "entry_path": "/item/\\d+" } }, @@ -1016,7 +1370,9 @@ "version": 1, "schema": "named_link", "minimum_records": 2, - "hosts": ["example.test"], + "hosts": [ + "example.test" + ], "entry_path": "/item/\\d+" } }, @@ -1036,15 +1392,27 @@ "case": { "url": "https://example.test/python", "required_facts": [ - {"id": "level", "alternatives": ["high-level"]}, - {"id": "purpose", "alternatives": ["general-purpose programming language"]} + { + "id": "level", + "alternatives": [ + "high-level" + ] + }, + { + "id": "purpose", + "alternatives": [ + "general-purpose programming language" + ] + } ], "fact_coverage_threshold": 1.0 }, "artifact": "**Python** is a [high-level](/level), [general-purpose programming language](/purpose).", "ok": true, "expected_task_pass": true, - "expected_metrics": {"fact_coverage": 1.0} + "expected_metrics": { + "fact_coverage": 1.0 + } }, { "id": "one-semantic-fragment-cannot-replace-the-other", @@ -1054,29 +1422,27 @@ "case": { "url": "https://example.test/python", "required_facts": [ - {"id": "level", "alternatives": ["high-level"]}, - {"id": "purpose", "alternatives": ["general-purpose programming language"]} + { + "id": "level", + "alternatives": [ + "high-level" + ] + }, + { + "id": "purpose", + "alternatives": [ + "general-purpose programming language" + ] + } ], "fact_coverage_threshold": 1.0 }, "artifact": "Python is a high-level language.", "ok": true, "expected_task_pass": false, - "expected_metrics": {"fact_coverage": 0.5} - }, - { - "id": "markdown-emphasized-forbidden-status-heading-is-blocked", - "category": "task-predicate", - "root_cause": "blocked_page_detection", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/platform", - "required_facts": [] - }, - "artifact": "**Error: Forbidden / 403**", - "ok": true, - "expected_task_pass": false, - "expected_metrics": {"blocked_page": true} + "expected_metrics": { + "fact_coverage": 0.5 + } }, { "id": "relative-autolinks-and-markdown-labels-preserve-visible-anchor", @@ -1090,12 +1456,18 @@ "version": 1, "schema": "named_link", "minimum_records": 2, - "hosts": ["example.test"], + "hosts": [ + "example.test" + ], "entry_path": "/item/[0-9]+", - "stable_text_anchors": [{ - "id": "help", - "alternatives": ["Questions: /help/start, /help/advanced, or Support."] - }] + "stable_text_anchors": [ + { + "id": "help", + "alternatives": [ + "Questions: /help/start, /help/advanced, or Support." + ] + } + ] } }, "artifact": "Questions: </help/start>, </help/advanced>, or [Support](https://support.example.test/).\n[Alpha](/item/1)\n[Beta](/item/2)", @@ -1118,12 +1490,18 @@ "version": 1, "schema": "named_link", "minimum_records": 2, - "hosts": ["example.test"], + "hosts": [ + "example.test" + ], "entry_path": "/item/[0-9]+", - "stable_text_anchors": [{ - "id": "help", - "alternatives": ["Questions: /help/start, /help/advanced, or Support."] - }] + "stable_text_anchors": [ + { + "id": "help", + "alternatives": [ + "Questions: /help/start, /help/advanced, or Support." + ] + } + ] } }, "artifact": "Questions: </help/start>, or [Support](https://support.example.test/).\n[Alpha](/item/1)\n[Beta](/item/2)", @@ -1146,9 +1524,13 @@ "version": 1, "schema": "named_link", "minimum_records": 3, - "hosts": ["example.test"], + "hosts": [ + "example.test" + ], "entry_path": "/item/[0-9]+", - "record_name_exclusions": ["FreeTV"] + "record_name_exclusions": [ + "FreeTV" + ] } }, "artifact": "[FreeTV](/item/1)\n[FreeTV](/item/2)\n[FreeTV](/item/3)", @@ -1171,9 +1553,13 @@ "version": 1, "schema": "named_link", "minimum_records": 3, - "hosts": ["example.test"], + "hosts": [ + "example.test" + ], "entry_path": "/item/[0-9]+", - "record_name_exclusions": ["FreeTV"] + "record_name_exclusions": [ + "FreeTV" + ] } }, "artifact": "[FreeTV](/item/0)\n[Alpha Show](/item/1)\n[Beta Show](/item/2)\n[Gamma Show](/item/3)", @@ -1193,16 +1579,36 @@ "url": "https://example.test/current-page", "min_content_chars": 20, "required_facts": [ - {"id": "identity", "alternatives": ["Stable page identity"]}, - {"id": "purpose", "alternatives": ["Stable purpose statement"]} + { + "id": "identity", + "alternatives": [ + "Stable page identity" + ] + }, + { + "id": "purpose", + "alternatives": [ + "Stable purpose statement" + ] + } ], "acceptance_contract": { "version": 1, "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [ - {"id": "identity", "alternatives": ["Stable page identity"]}, - {"id": "purpose", "alternatives": ["Stable purpose statement"]} + { + "id": "identity", + "alternatives": [ + "Stable page identity" + ] + }, + { + "id": "purpose", + "alternatives": [ + "Stable purpose statement" + ] + } ] } }, @@ -1223,16 +1629,36 @@ "url": "https://example.test/current-page", "min_content_chars": 20, "required_facts": [ - {"id": "identity", "alternatives": ["Stable page identity"]}, - {"id": "purpose", "alternatives": ["Stable purpose statement"]} - ], - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - {"id": "identity", "alternatives": ["Stable page identity"]}, - {"id": "purpose", "alternatives": ["Stable purpose statement"]} + { + "id": "identity", + "alternatives": [ + "Stable page identity" + ] + }, + { + "id": "purpose", + "alternatives": [ + "Stable purpose statement" + ] + } + ], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "identity", + "alternatives": [ + "Stable page identity" + ] + }, + { + "id": "purpose", + "alternatives": [ + "Stable purpose statement" + ] + } ] } }, @@ -1257,7 +1683,12 @@ "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [ - {"id": "identity", "alternatives": ["Stable page identity"]} + { + "id": "identity", + "alternatives": [ + "Stable page identity" + ] + } ] } }, @@ -1282,7 +1713,12 @@ "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [ - {"id": "identity", "alternatives": ["Stable page identity"]} + { + "id": "identity", + "alternatives": [ + "Stable page identity" + ] + } ] } }, @@ -1304,15 +1740,32 @@ "task_kind": "body", "min_content_chars": 20, "required_facts": [ - {"id": "title", "alternatives": ["Document conversion demonstration"]}, - {"id": "table", "alternatives": ["Highlighter 2 colors"]}, - {"id": "note", "alternatives": ["Endnotes retain a backlink to the current position"]} + { + "id": "title", + "alternatives": [ + "Document conversion demonstration" + ] + }, + { + "id": "table", + "alternatives": [ + "Highlighter 2 colors" + ] + }, + { + "id": "note", + "alternatives": [ + "Endnotes retain a backlink to the current position" + ] + } ] }, "artifact": "Document conversion demonstration\nTables\nFootnotes & Endnotes\nLists and images", "ok": true, "expected_task_pass": false, - "expected_metrics": {"fact_coverage": 0.3333} + "expected_metrics": { + "fact_coverage": 0.3333 + } }, { "id": "document-content-landmarks-pass-with-complete-table-and-note-content", @@ -1324,15 +1777,32 @@ "task_kind": "body", "min_content_chars": 20, "required_facts": [ - {"id": "title", "alternatives": ["Document conversion demonstration"]}, - {"id": "table", "alternatives": ["Highlighter 2 colors"]}, - {"id": "note", "alternatives": ["Endnotes retain a backlink to the current position"]} + { + "id": "title", + "alternatives": [ + "Document conversion demonstration" + ] + }, + { + "id": "table", + "alternatives": [ + "Highlighter 2 colors" + ] + }, + { + "id": "note", + "alternatives": [ + "Endnotes retain a backlink to the current position" + ] + } ] }, "artifact": "Document conversion demonstration\n| ITEM | NEEDED |\n| --- | --- |\n| Highlighter | 2 colors |\nEndnotes retain a backlink to the current position", "ok": true, "expected_task_pass": true, - "expected_metrics": {"fact_coverage": 1.0} + "expected_metrics": { + "fact_coverage": 1.0 + } }, { "id": "markdown-table-cells-match-readable-content-landmarks", @@ -1343,13 +1813,20 @@ "url": "https://example.test/document.docx", "task_kind": "body", "required_facts": [ - {"id": "table", "alternatives": ["Highlighter 2 colors"]} + { + "id": "table", + "alternatives": [ + "Highlighter 2 colors" + ] + } ] }, "artifact": "| ITEM | NEEDED |\n| --- | --- |\n| Highlighter | 2 colors |", "ok": true, "expected_task_pass": true, - "expected_metrics": {"fact_coverage": 1.0} + "expected_metrics": { + "fact_coverage": 1.0 + } }, { "id": "single-literal-pipe-is-not-a-table-boundary", @@ -1360,13 +1837,20 @@ "url": "https://example.test/code", "task_kind": "body", "required_facts": [ - {"id": "expression", "alternatives": ["alpha beta"]} + { + "id": "expression", + "alternatives": [ + "alpha beta" + ] + } ] }, "artifact": "The literal expression is alpha | beta.", "ok": true, "expected_task_pass": false, - "expected_metrics": {"fact_coverage": 0.0} + "expected_metrics": { + "fact_coverage": 0.0 + } }, { "id": "script-state-anchor-is-not-readable-body", @@ -1380,14 +1864,22 @@ "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [ - {"id": "answer", "alternatives": ["Required readable answer"]} + { + "id": "answer", + "alternatives": [ + "Required readable answer" + ] + } ] } }, "artifact": "window.__S0__={\"text\":\"Required readable answer\",\"payload\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"};window.__S1__={};window.__S2__={};window.__S3__={};window.__S4__={};window.__S5__={};window.__S6__={};window.__S7__={};window.__S8__={};window.__S9__={};window.__S10__={};window.__S11__={};//# sourceURL=/bundle.js\n//# sourceURL=/1.js\n//# sourceURL=/2.js\n//# sourceURL=/3.js\n//# sourceURL=/4.js\n//# sourceURL=/5.js\n//# sourceURL=/6.js\n//# sourceURL=/7.js\n//# sourceURL=/8.js\n//# sourceURL=/9.js\n//# sourceURL=/10.js", "ok": true, "expected_task_pass": false, - "expected_metrics": {"script_payload_shell": true, "stable_task_pass": false} + "expected_metrics": { + "script_payload_shell": true, + "stable_task_pass": false + } }, { "id": "readable-javascript-article-with-code-still-passes", @@ -1401,22840 +1893,46365 @@ "schema": "body_text", "minimum_records": 0, "stable_text_anchors": [ - {"id": "identity", "alternatives": ["Reading JavaScript application state"]}, - {"id": "purpose", "alternatives": ["readable prose remains the task output"]} + { + "id": "identity", + "alternatives": [ + "Reading JavaScript application state" + ] + }, + { + "id": "purpose", + "alternatives": [ + "readable prose remains the task output" + ] + } ] } }, "artifact": "# Reading JavaScript application state\nThis article explains why readable prose remains the task output. It discusses serialization, hydration, accessibility, extraction boundaries, and testing in ordinary paragraphs.\n```js\nwindow.__STATE__ = {article: 'example'};\n```", "ok": true, "expected_task_pass": true, - "expected_metrics": {"script_payload_shell": false, "stable_task_pass": true} - }, - { - "id": "amazon-tracking-suffix-remains-a-product-detail", - "category": "acceptance-contract", - "root_cause": "amazon_product_tracking_path_suffix", - "failure_mode": "false-negative", - "case": { - "url": "https://www.amazon.example/search", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.amazon.example" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "stable_text_anchors": [ - { - "id": "query", - "alternatives": [ - "headphones" - ] - } - ] - } - }, - "artifact": "headphones\n- [Alpha headset](/alpha/dp/B012345678/ref=sr_1_1)\n- [Beta headset](/gp/aw/d/B087654321/ref=sr_1_2)", - "ok": true, - "expected_task_pass": true, - "expected_metrics": { - "stable_structure_pass": true, - "stable_unique_record_count": 2 - } - }, - { - "id": "amazon-non-ref-suffix-is-not-a-product-detail", - "category": "acceptance-contract", - "root_cause": "amazon_product_tracking_path_suffix", - "failure_mode": "false-positive", - "case": { - "url": "https://www.amazon.example/search", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.amazon.example" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "stable_text_anchors": [ - { - "id": "query", - "alternatives": [ - "headphones" - ] - } - ] + "expected_metrics": { + "script_payload_shell": false, + "stable_task_pass": true + } + }, + { + "id": "amazon-tracking-suffix-remains-a-product-detail", + "category": "acceptance-contract", + "root_cause": "amazon_product_tracking_path_suffix", + "failure_mode": "false-negative", + "case": { + "url": "https://www.amazon.example/search", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.amazon.example" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "stable_text_anchors": [ + { + "id": "query", + "alternatives": [ + "headphones" + ] } - }, - "artifact": "headphones\n- [Alpha review](/alpha/dp/B012345678/reviews)\n- [Beta offers](/gp/aw/d/B087654321/offers)", - "ok": true, - "expected_task_pass": false, - "expected_metrics": { - "stable_structure_pass": false, - "stable_unique_record_count": 0 - } - }, - { - "id": "dynamic-target-placeholder-cannot-pass", - "category": "body-scope", - "root_cause": "unrendered_dynamic_target_placeholder", - "failure_mode": "false-positive", - "case": { - "url": "https://example.com/table", - "task_kind": "body", - "required_facts": [ - { - "id": "title", - "alternatives": [ - "Virtual DOM" - ] - } - ], - "forbidden_any": [ - "Loading Example..." - ] - }, - "artifact": "Virtual DOM\nLoading Example...\nView Source", - "ok": true, - "expected_task_pass": false, - "expected_metrics": { - "forbidden_hits": [ - "Loading Example..." - ] + ] } + }, + "artifact": "headphones\n- [Alpha headset](/alpha/dp/B012345678/ref=sr_1_1)\n- [Beta headset](/gp/aw/d/B087654321/ref=sr_1_2)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_structure_pass": true, + "stable_unique_record_count": 2 + } }, { - "id": "completed-dynamic-target-can-pass", - "category": "body-scope", - "root_cause": "unrendered_dynamic_target_placeholder", - "failure_mode": "false-negative", - "case": { - "url": "https://example.com/table", - "task_kind": "body", - "required_facts": [ - { - "id": "title", - "alternatives": [ - "Virtual DOM" - ] - } - ], - "forbidden_any": [ - "Loading Example..." - ] - }, - "artifact": "Virtual DOM\nName Age\nAlice 31\nBob 42", - "ok": true, - "expected_task_pass": true, - "expected_metrics": { - "forbidden_hits": [] - } - }, - { - "id": "github-route-links-are-not-repositories", - "category": "acceptance-contract", - "root_cause": "github_route_as_repository", - "failure_mode": "false-positive", - "case": { - "url": "https://github.com/trending", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!trending(?:/|$)|sponsors(?:/|$))[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", - "stable_text_anchors": [ - { - "id": "title", - "alternatives": [ - "Trending" - ] - }, - { - "id": "authors", - "alternatives": [ - "Built by" - ] - } - ] - } - }, - "artifact": "Trending\nBuilt by\n- [Python](https://github.com/trending/python)\n- [Explore](https://github.com/sponsors/explore)", - "ok": true, - "expected_task_pass": false, - "expected_metrics": { - "stable_structure_pass": false, - "stable_unique_record_count": 0 - } - }, - { - "id": "github-owner-repository-links-are-records", - "category": "acceptance-contract", - "root_cause": "github_route_as_repository", - "failure_mode": "false-negative", - "case": { - "url": "https://github.com/trending", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!trending(?:/|$)|sponsors(?:/|$))[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", - "stable_text_anchors": [ - { - "id": "title", - "alternatives": [ - "Trending" - ] - }, - { - "id": "authors", - "alternatives": [ - "Built by" - ] - } - ] - } - }, - "artifact": "Trending\nBuilt by\n- [alpha / project](https://github.com/alpha/project)\n- [beta / tool](https://github.com/beta/tool)", - "ok": true, - "expected_task_pass": true, - "expected_metrics": { - "stable_structure_pass": true, - "stable_unique_record_count": 2 - } - }, - { - "id": "stable-contract-cannot-bypass-incomplete-body", - "category": "body-scope", - "root_cause": "stable_contract_body_conjunction", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/article", - "task_kind": "body", - "ground_truth_main_content": "Article alpha bravo charlie delta echo foxtrot golf hotel", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 2, - "regions": [ - { - "id": "opening", - "text": "alpha bravo charlie delta" - }, - { - "id": "ending", - "text": "echo foxtrot golf hotel" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "identity", - "alternatives": [ - "Article" - ] - } - ] - } - }, - "artifact": "Article alpha bravo charlie delta", - "ok": true, - "expected_task_pass": false, - "expected_metrics": { - "stable_task_pass": true, - "body_coverage_pass": false, - "content_pass": false - } - }, - { - "id": "stable-contract-and-complete-body-can-pass-together", - "category": "body-scope", - "root_cause": "stable_contract_body_conjunction", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/article", - "task_kind": "body", - "ground_truth_main_content": "Article alpha bravo charlie delta echo foxtrot golf hotel", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 2, - "regions": [ - { - "id": "opening", - "text": "alpha bravo charlie delta" - }, - { - "id": "ending", - "text": "echo foxtrot golf hotel" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "identity", - "alternatives": [ - "Article" - ] - } - ] - } - }, - "artifact": "Article alpha bravo charlie delta echo foxtrot golf hotel", - "ok": true, - "expected_task_pass": true, - "expected_metrics": { - "stable_task_pass": true, - "body_coverage_pass": true, - "content_pass": true - } - }, - { - "id": "unresolved-rule-cannot-be-counted-as-provider-pass", - "category": "acceptance-contract", - "root_cause": "rule_unresolved_exclusion", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/dynamic", - "task_kind": "body", - "expected_any": [ - "complete answer" - ], - "rule_review": { - "status": "unresolved", - "reason": "fewer_than_three_stable_texts" - } - }, - "artifact": "complete answer", - "ok": true, - "expected_task_pass": false, - "expected_metrics": { - "rule_review_status": "unresolved" - } - }, - { - "id": "resolved-rule-remains-eligible-for-provider-pass", - "category": "acceptance-contract", - "root_cause": "rule_unresolved_exclusion", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/dynamic", - "task_kind": "body", - "expected_any": [ - "complete answer" - ], - "rule_review": { - "status": "resolved" - } - }, - "artifact": "complete answer", - "ok": true, - "expected_task_pass": true, - "expected_metrics": { - "rule_review_status": "resolved" - } - }, - { - "id": "structured-regions-reject-a-missing-ending-section", - "category": "body-scope", - "root_cause": "body_region_structure_priority", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/guide", - "task_kind": "body", - "ground_truth_main_content": "Opening alpha bravo charlie delta. Ending echo foxtrot golf hotel.", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 2, - "regions": [ - { - "id": "opening", - "text": "Opening alpha bravo charlie delta" - }, - { - "id": "ending", - "text": "Ending echo foxtrot golf hotel" - } - ] - } - }, - "artifact": "Opening alpha bravo charlie delta.", - "ok": true, - "expected_task_pass": false, - "expected_metrics": { - "body_coverage_pass": false, - "content_pass": false - } - }, - { - "id": "structured-regions-accept-the-complete-body", - "category": "body-scope", - "root_cause": "body_region_structure_priority", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/guide", - "task_kind": "body", - "ground_truth_main_content": "Opening alpha bravo charlie delta. Ending echo foxtrot golf hotel.", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 2, - "regions": [ - { - "id": "opening", - "text": "Opening alpha bravo charlie delta" - }, - { - "id": "ending", - "text": "Ending echo foxtrot golf hotel" - } - ] - } - }, - "artifact": "Opening alpha bravo charlie delta. Ending echo foxtrot golf hotel.", - "ok": true, - "expected_task_pass": true, - "expected_metrics": { - "body_coverage_pass": true, - "content_pass": true - } - }, - { - "id": "variable-list-23-of-30-does-not-meet-80-percent-source-threshold", - "category": "acceptance-contract", - "root_cause": "source_observed_record_threshold", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/list", - "task_kind": "stable_records", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 24, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "record_count_policy": { - "list_type": "variable", - "source_observed_count": 30, - "required_ratio": 0.8, - "source_observation_counts": [ - 30, - 30, - 30 - ], - "source_evidence_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "reason": "Independent source observation before the provider call." - } - } - }, - "artifact": "[Item 1](https://example.test/item/1)\n[Item 2](https://example.test/item/2)\n[Item 3](https://example.test/item/3)\n[Item 4](https://example.test/item/4)\n[Item 5](https://example.test/item/5)\n[Item 6](https://example.test/item/6)\n[Item 7](https://example.test/item/7)\n[Item 8](https://example.test/item/8)\n[Item 9](https://example.test/item/9)\n[Item 10](https://example.test/item/10)\n[Item 11](https://example.test/item/11)\n[Item 12](https://example.test/item/12)\n[Item 13](https://example.test/item/13)\n[Item 14](https://example.test/item/14)\n[Item 15](https://example.test/item/15)\n[Item 16](https://example.test/item/16)\n[Item 17](https://example.test/item/17)\n[Item 18](https://example.test/item/18)\n[Item 19](https://example.test/item/19)\n[Item 20](https://example.test/item/20)\n[Item 21](https://example.test/item/21)\n[Item 22](https://example.test/item/22)\n[Item 23](https://example.test/item/23)", - "ok": true, - "expected_task_pass": false, - "expected_metrics": { - "stable_unique_record_count": 23, - "stable_minimum_records": 24, - "stable_task_pass": false - } - }, - { - "id": "variable-list-24-of-30-meets-source-threshold-without-claiming-completeness", - "category": "acceptance-contract", - "root_cause": "source_observed_record_threshold", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/list", - "task_kind": "stable_records", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 24, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "record_count_policy": { - "list_type": "variable", - "source_observed_count": 30, - "required_ratio": 0.8, - "source_observation_counts": [ - 30, - 30, - 30 - ], - "source_evidence_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "reason": "Independent source observation before the provider call." - } - } - }, - "artifact": "[Item 1](https://example.test/item/1)\n[Item 2](https://example.test/item/2)\n[Item 3](https://example.test/item/3)\n[Item 4](https://example.test/item/4)\n[Item 5](https://example.test/item/5)\n[Item 6](https://example.test/item/6)\n[Item 7](https://example.test/item/7)\n[Item 8](https://example.test/item/8)\n[Item 9](https://example.test/item/9)\n[Item 10](https://example.test/item/10)\n[Item 11](https://example.test/item/11)\n[Item 12](https://example.test/item/12)\n[Item 13](https://example.test/item/13)\n[Item 14](https://example.test/item/14)\n[Item 15](https://example.test/item/15)\n[Item 16](https://example.test/item/16)\n[Item 17](https://example.test/item/17)\n[Item 18](https://example.test/item/18)\n[Item 19](https://example.test/item/19)\n[Item 20](https://example.test/item/20)\n[Item 21](https://example.test/item/21)\n[Item 22](https://example.test/item/22)\n[Item 23](https://example.test/item/23)\n[Item 24](https://example.test/item/24)", - "ok": true, - "expected_task_pass": true, - "expected_metrics": { - "stable_unique_record_count": 24, - "stable_minimum_records": 24, - "stable_task_pass": true - } - }, - { - "id": "dynamic-final-page-mismatch-cannot-be-counted-as-a-pass", - "category": "acceptance-contract", - "root_cause": "dynamic_page_identity_alignment", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/start", - "evaluation_url": "https://example.test/start", - "expected_final_url": "https://example.test/en/catalog", - "target_language": "en-US", - "target_region": "US", - "track": "dynamic", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "identity", - "alternatives": [ - "Catalog" - ] - } - ] - } - }, - "artifact": "Catalog", - "ok": true, - "final_url": "https://example.test/fr/catalog", - "metadata": { - "route": "dynamic_selected_text", - "document_language": "fr" - }, - "expected_task_pass": false, - "expected_metrics": { - "page_identity_status": "mismatch", - "final_url_consistent": false, - "language_consistent": false - } - }, - { - "id": "dynamic-matching-final-page-and-language-remain-eligible", - "category": "acceptance-contract", - "root_cause": "dynamic_page_identity_alignment", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/start", - "evaluation_url": "https://example.test/start", - "expected_final_url": "https://example.test/en/catalog", - "target_language": "en-US", - "target_region": "US", - "track": "dynamic", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "identity", - "alternatives": [ - "Catalog" - ] - } - ] - } - }, - "artifact": "Catalog", - "ok": true, - "final_url": "https://example.test/en/catalog?session=123#items", - "metadata": { - "route": "dynamic_selected_text", - "document_language": "en" - }, - "expected_task_pass": true, - "expected_metrics": { - "page_identity_status": "verified", - "final_url_consistent": true, - "language_consistent": true - } - }, - { - "id": "dynamic-search-query-mismatch-is-a-different-page", - "category": "acceptance-contract", - "root_cause": "dynamic_page_query_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/search?q=Mario", - "evaluation_url": "https://example.test/search?q=Mario", - "expected_final_url": "https://example.test/search?q=Mario", - "target_language": "ja-JP", - "target_region": "JP", - "track": "dynamic", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "identity", - "alternatives": [ - "Search results" - ] - } - ] - } - }, - "artifact": "Search results", - "ok": true, - "final_url": "https://example.test/search?q=Zelda", - "metadata": { - "route": "dynamic_selected_text", - "document_language": "ja" - }, - "expected_task_pass": false, - "expected_metrics": { - "page_identity_status": "mismatch", - "final_url_consistent": false - } - }, - { - "id": "dynamic-matching-search-query-remains-eligible", - "category": "acceptance-contract", - "root_cause": "dynamic_page_query_identity", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/search?q=Mario", - "evaluation_url": "https://example.test/search?q=Mario", - "expected_final_url": "https://example.test/search?q=Mario", - "target_language": "ja-JP", - "target_region": "JP", - "track": "dynamic", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "identity", - "alternatives": [ - "Search results" - ] - } - ] - } - }, - "artifact": "Search results", - "ok": true, - "final_url": "https://example.test/search?q=Mario#results", - "metadata": { - "route": "dynamic_selected_text", - "document_language": "ja" - }, - "expected_task_pass": true, - "expected_metrics": { - "page_identity_status": "verified", - "final_url_consistent": true - } - }, - { - "id": "dynamic-source-document-language-mismatch-is-not-verified", - "category": "acceptance-contract", - "root_cause": "declared_document_language_vs_visible_language", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/rtl", - "evaluation_url": "https://example.test/rtl", - "expected_final_url": "https://example.test/rtl", - "target_language": "ar", - "expected_document_language": "en", - "target_region": "US", - "track": "dynamic", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "arabic-content", - "alternatives": [ - "الشخص" - ] - } - ] - } - }, - "artifact": "الشخص", - "ok": true, - "final_url": "https://example.test/rtl", - "metadata": { - "route": "dynamic_selected_text", - "document_language": "fr" - }, - "expected_task_pass": false, - "expected_metrics": { - "page_identity_status": "mismatch", - "language_consistent": false - } - }, - { - "id": "dynamic-arabic-visible-text-with-english-html-lang-remains-eligible", - "category": "acceptance-contract", - "root_cause": "declared_document_language_vs_visible_language", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/rtl", - "evaluation_url": "https://example.test/rtl", - "expected_final_url": "https://example.test/rtl", - "target_language": "ar", - "expected_document_language": "en", - "target_region": "US", - "track": "dynamic", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "arabic-content", - "alternatives": [ - "الشخص" - ] - } - ] - } - }, - "artifact": "الشخص", - "ok": true, - "final_url": "https://example.test/rtl", - "metadata": { - "route": "dynamic_selected_text", - "document_language": "en" - }, - "expected_task_pass": true, - "expected_metrics": { - "page_identity_status": "verified", - "language_consistent": true - } - }, - { - "id": "name-link-count-cannot-credit-an-invented-name-for-a-real-url", - "category": "acceptance-contract", - "root_cause": "frozen_source_name_link_pair_membership", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/list", - "task_kind": "stable_records", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "maximum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "source_record_pair_hashes": [ - "2802c3d298ecbed8c44da81bb83f66e6349bbf3062abed780e51293265c3ee72", - "75002de276f7af8bf713de877ebf72e6b939ded0e48805943254d7bd2b54e714" - ], - "source_record_pairs_sha256": "8a8eb383a10ea9338de82cd9f9d2b58f970191e3260dbacbd08a7e7f5f7335ea" - } - }, - "artifact": "[Invented item](https://example.test/item/1)\n[Item 2](https://example.test/item/2)", - "ok": true, - "expected_task_pass": false, - "expected_metrics": { - "stable_source_matched_record_count": 1, - "stable_source_mismatched_record_count": 1, - "stable_task_pass": false - } - }, - { - "id": "name-link-count-credits-two-independently-observed-name-url-pairs", - "category": "acceptance-contract", - "root_cause": "frozen_source_name_link_pair_membership", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/list", - "task_kind": "stable_records", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "maximum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "source_record_pair_hashes": [ - "2802c3d298ecbed8c44da81bb83f66e6349bbf3062abed780e51293265c3ee72", - "75002de276f7af8bf713de877ebf72e6b939ded0e48805943254d7bd2b54e714" - ], - "source_record_pairs_sha256": "8a8eb383a10ea9338de82cd9f9d2b58f970191e3260dbacbd08a7e7f5f7335ea" - } - }, - "artifact": "[Item 1](https://example.test/item/1)\n[Item 2](https://example.test/item/2)", - "ok": true, - "expected_task_pass": true, - "expected_metrics": { - "stable_source_matched_record_count": 2, - "stable_source_mismatched_record_count": 0, - "stable_task_pass": true - } - }, - { - "id": "volatile-body-stable-text-does-not-freeze-current-members", - "category": "acceptance-contract", - "root_cause": "volatile_body_snapshot_used_as_invariant_answer", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/live-events", - "task_kind": "body", - "ground_truth_main_content": "40 Events. Old event at 10:00", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "heading", - "alternatives": [ - "Events, Past Day" - ] - }, - { - "id": "sort", - "alternatives": [ - "Newest First" - ] - }, - { - "id": "settings", - "alternatives": [ - "Settings" - ] - } - ] - }, - "mutable_body_policy": { - "stable_annotation": { - "content_kind": "changing_public_body", - "structural_only_fields": [ - "mutable_full_text_snapshot" - ] - } - } - }, - "artifact": "Events, Past Day\n52 Events.\nNewest First\nSettings\nNew event at 11:00", - "ok": true, - "expected_task_pass": true, - "expected_metrics": { - "stable_task_pass": true, - "body_coverage_pass": true - } - }, - { - "id": "volatile-body-stable-text-still-rejects-missing-page-identity", - "category": "acceptance-contract", - "root_cause": "volatile_body_snapshot_used_as_invariant_answer", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/live-events", - "task_kind": "body", - "ground_truth_main_content": "40 Events. Old event at 10:00", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "heading", - "alternatives": [ - "Events, Past Day" - ] - }, - { - "id": "sort", - "alternatives": [ - "Newest First" - ] - }, - { - "id": "settings", - "alternatives": [ - "Settings" - ] - } - ] - }, - "mutable_body_policy": { - "stable_annotation": { - "content_kind": "changing_public_body", - "structural_only_fields": [ - "mutable_full_text_snapshot" - ] - } - } - }, - "artifact": "52 Events.\nNewest First\nSettings\nNew event at 11:00", - "ok": true, - "expected_task_pass": false, - "expected_metrics": { - "stable_task_pass": false, - "stable_anchor_hits": 2 - } - }, - { - "id": "normal-theme-control-is-not-a-block-page-marker", - "category": "acceptance-contract", - "root_cause": "normal_page_control_misclassified_as_forbidden_content", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/docs", - "task_kind": "body", - "forbidden_any": [ - "CAPTCHA", - "Access denied" - ], - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "identity", - "alternatives": [ - "Documentation" - ] - }, - { - "id": "install", - "alternatives": [ - "Install" - ] - }, - { - "id": "import", - "alternatives": [ - "Import" - ] - } - ] - } - }, - "artifact": "Documentation\nDark theme\nInstall\nImport", - "ok": true, - "expected_task_pass": true, - "expected_metrics": { - "stable_task_pass": true, - "forbidden_hits": [] - } - }, - { - "id": "removing-normal-theme-control-does-not-hide-real-block-page", - "category": "acceptance-contract", - "root_cause": "normal_page_control_misclassified_as_forbidden_content", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/docs", - "task_kind": "body", - "forbidden_any": [ - "CAPTCHA", - "Access denied" - ], - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "identity", - "alternatives": [ - "Documentation" - ] - }, - { - "id": "install", - "alternatives": [ - "Install" - ] - }, - { - "id": "import", - "alternatives": [ - "Import" - ] - } - ] + "id": "amazon-non-ref-suffix-is-not-a-product-detail", + "category": "acceptance-contract", + "root_cause": "amazon_product_tracking_path_suffix", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.example/search", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.amazon.example" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "stable_text_anchors": [ + { + "id": "query", + "alternatives": [ + "headphones" + ] } - }, - "artifact": "Documentation\nInstall\nImport\nCAPTCHA", - "ok": true, - "expected_task_pass": false, - "expected_metrics": { - "forbidden_hits": [ - "CAPTCHA" - ] + ] } + }, + "artifact": "headphones\n- [Alpha review](/alpha/dp/B012345678/reviews)\n- [Beta offers](/gp/aw/d/B087654321/offers)", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_structure_pass": false, + "stable_unique_record_count": 0 + } }, { - "id": "literal-footer-pipe-remains-matchable", - "category": "format-normalization", - "root_cause": "multi_pipe_prose_misclassified_as_markdown_table", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/pipe-text", - "task_kind": "body", - "required_facts": [ - { - "id": "required", - "alternatives": [ - "上海互联网举报中心 |" - ] - } - ] - }, - "artifact": "上海互联网举报中心 | 儿童色情信息举报专区 |", - "ok": true, - "expected_task_pass": true, - "expected_metrics": { - "fact_coverage": 1.0 - } - }, - { - "id": "multiple-literal-pipes-preserve-operators", - "category": "format-normalization", - "root_cause": "multi_pipe_prose_misclassified_as_markdown_table", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/pipe-text", - "task_kind": "body", - "required_facts": [ - { - "id": "required", - "alternatives": [ - "alpha beta gamma" - ] - } - ] - }, - "artifact": "alpha | beta | gamma", - "ok": true, - "expected_task_pass": false, - "expected_metrics": { - "fact_coverage": 0.0 - } - }, - { - "category": "format-normalization", - "root_cause": "multi_pipe_prose_misclassified_as_markdown_table", - "ok": true, - "artifact": "| Expression | Result |\n| --- | --- |\n| alpha \\| beta | gamma |", - "id": "escaped-table-pipe-keeps-literal-expression", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/pipe-table", - "task_kind": "body", - "required_facts": [ - { - "id": "expression", - "alternatives": [ - "alpha | beta" - ] - } + "id": "dynamic-target-placeholder-cannot-pass", + "category": "body-scope", + "root_cause": "unrendered_dynamic_target_placeholder", + "failure_mode": "false-positive", + "case": { + "url": "https://example.com/table", + "task_kind": "body", + "required_facts": [ + { + "id": "title", + "alternatives": [ + "Virtual DOM" ] - }, - "expected_task_pass": true - }, - { - "category": "format-normalization", - "root_cause": "multi_pipe_prose_misclassified_as_markdown_table", - "ok": true, - "artifact": "| Expression | Result |\n| --- | --- |\n| alpha \\| beta | gamma |", - "id": "escaped-table-pipe-cannot-be-removed", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/pipe-table", - "task_kind": "body", - "required_facts": [ - { - "id": "expression", - "alternatives": [ - "alpha beta" - ] - } + } + ], + "forbidden_any": [ + "Loading Example..." + ] + }, + "artifact": "Virtual DOM\nLoading Example...\nView Source", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "forbidden_requirement_hits": [ + "Loading Example..." + ] + } + }, + { + "id": "completed-dynamic-target-can-pass", + "category": "body-scope", + "root_cause": "unrendered_dynamic_target_placeholder", + "failure_mode": "false-negative", + "case": { + "url": "https://example.com/table", + "task_kind": "body", + "required_facts": [ + { + "id": "title", + "alternatives": [ + "Virtual DOM" ] - }, - "expected_task_pass": false - }, - { - "id": "dynamic-membership-new-entities-accepted", - "category": "acceptance-contract", - "root_cause": "dynamic_list_membership_not_identity", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "record_count_policy": { - "list_type": "variable", - "source_observed_count": 2, - "required_ratio": 0.8, - "minimum_records": 2, - "source_observation_counts": [ - 2, - 2, - 2 - ], - "source_evidence_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "reason": "Independent source count; list membership is not frozen." - } - } - }, - "artifact": "[New first title](/item/91)\n[New second title](/item/92)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-membership-wrong-path-rejected", - "category": "acceptance-contract", - "root_cause": "dynamic_list_membership_not_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "record_count_policy": { - "list_type": "variable", - "source_observed_count": 2, - "required_ratio": 0.8, - "minimum_records": 2, - "source_observation_counts": [ - 2, - 2, - 2 - ], - "source_evidence_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "reason": "Independent source count; list membership is not frozen." - } - } - }, - "artifact": "[New first title](/item/91)\n[New second title](/category/92)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-membership-empty-title-rejected", - "category": "acceptance-contract", - "root_cause": "dynamic_list_membership_not_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "record_count_policy": { - "list_type": "variable", - "source_observed_count": 2, - "required_ratio": 0.8, - "minimum_records": 2, - "source_observation_counts": [ - 2, - 2, - 2 - ], - "source_evidence_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "reason": "Independent source count; list membership is not frozen." - } - } - }, - "artifact": "[New first title](/item/91)\n[](/item/92)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-membership-short-list-rejected", - "category": "acceptance-contract", - "root_cause": "dynamic_list_membership_not_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "record_count_policy": { - "list_type": "variable", - "source_observed_count": 2, - "required_ratio": 0.8, - "minimum_records": 2, - "source_observation_counts": [ - 2, - 2, - 2 - ], - "source_evidence_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "reason": "Independent source count; list membership is not frozen." - } - } - }, - "artifact": "[New first title](/item/91)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-membership-duplicate-not-extra-item", - "category": "acceptance-contract", - "root_cause": "dynamic_list_membership_not_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "record_count_policy": { - "list_type": "variable", - "source_observed_count": 2, - "required_ratio": 0.8, - "minimum_records": 2, - "source_observation_counts": [ - 2, - 2, - 2 - ], - "source_evidence_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "reason": "Independent source count; list membership is not frozen." - } - } - }, - "artifact": "[New first title](/item/91)\n[New first title](/item/91)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "quake-complete-visible-cards", - "category": "structured-extraction", - "root_cause": "earthquake_record_field_ownership", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/earthquakes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "earthquake", - "minimum_records": 2 - } - }, - "artifact": "4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)\n10.0 km\n3.1\nNorth Valley\n2026-09-20T05:00:00Z\n7.4 km", - "ok": true, - "expected_task_pass": true - }, - { - "id": "quake-complete-markdown-cards", - "category": "structured-extraction", - "root_cause": "earthquake_record_field_ownership", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/earthquakes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "earthquake", - "minimum_records": 2 - } - }, - "artifact": "[4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)\n10.0 km](/event/1)\n[3.1\nNorth Valley\n2026-09-20T05:00:00Z\n7.4 km](/event/2)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "quake-table-fields", - "category": "structured-extraction", - "root_cause": "earthquake_record_field_ownership", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/earthquakes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "earthquake", - "minimum_records": 2 - } - }, - "artifact": "| magnitude | location | time | depth |\n| --- | --- | --- | --- |\n| 4.2 | Harbor Island | 2026-09-20T04:00:00Z | 10.0 km |\n| 3.1 | North Valley | 2026-09-20T05:00:00Z | 7.4 km |", - "ok": true, - "expected_task_pass": true - }, - { - "id": "quake-dates-alone", - "category": "structured-extraction", - "root_cause": "earthquake_record_field_ownership", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/earthquakes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "earthquake", - "minimum_records": 2 - } - }, - "artifact": "Earthquakes\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "quake-missing-depth", - "category": "structured-extraction", - "root_cause": "earthquake_record_field_ownership", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/earthquakes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "earthquake", - "minimum_records": 2 - } - }, - "artifact": "4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)\n3.1\nNorth Valley\n2026-09-20T05:00:00Z\n7.4 km", - "ok": true, - "expected_task_pass": false - }, - { - "id": "quake-split-card-must-not-borrow", - "category": "structured-extraction", - "root_cause": "earthquake_record_field_ownership", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/earthquakes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "earthquake", - "minimum_records": 2 - } - }, - "artifact": "[4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)](/event/1)\n[10.0 km\n3.1\nNorth Valley\n2026-09-20T05:00:00Z\n7.4 km](/event/2)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "quake-duplicate-not-extra-event", - "category": "structured-extraction", - "root_cause": "earthquake_record_field_ownership", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/earthquakes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "earthquake", - "minimum_records": 2 - } - }, - "artifact": "4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)\n10.0 km\n4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)\n10.0 km", - "ok": true, - "expected_task_pass": false - }, - { - "id": "quake-invalid-calendar", - "category": "structured-extraction", - "root_cause": "earthquake_record_field_ownership", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/earthquakes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "earthquake", - "minimum_records": 2 - } - }, - "artifact": "4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)\n10.0 km\n3.1\nNorth Valley\n2026-02-30T05:00:00Z\n7.4 km", - "ok": true, - "expected_task_pass": false - }, - { - "id": "body-single-source-occurrence-not-duplicated", - "category": "reference-evidence", - "root_cause": "collector_parent_child_text_duplication", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/api-docs", - "task_kind": "body", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "sentences", - "regions": [ - { - "id": "body-region-1", - "text": "client name identifies the application requesting authorization redirect addresses determine where an approved response can be delivered", - "source_text": "Client name identifies the application requesting authorization.\n\nRedirect addresses determine where an approved response can be delivered.", - "segmentation_method": "sentences" - }, - { - "id": "body-region-2", - "text": "allowed scopes specify the operations available to this application grant types describe the supported authentication flows the service returns the registered identifier and a secret for subsequent requests", - "source_text": "Allowed scopes specify the operations available to this application.\n\nGrant types describe the supported authentication flows.\n\nThe service returns the registered identifier and a secret for subsequent requests.", - "segmentation_method": "sentences" - } - ] - } - }, - "artifact": "Client name identifies the application requesting authorization. Redirect addresses determine where an approved response can be delivered. Allowed scopes specify the operations available to this application. Grant types describe the supported authentication flows. The service returns the registered identifier and a secret for subsequent requests.", - "ok": true, - "expected_task_pass": true - }, - { - "id": "body-missing-source-paragraph-rejected", - "category": "reference-evidence", - "root_cause": "collector_parent_child_text_duplication", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/api-docs", - "task_kind": "body", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "sentences", - "regions": [ - { - "id": "body-region-1", - "text": "client name identifies the application requesting authorization redirect addresses determine where an approved response can be delivered", - "source_text": "Client name identifies the application requesting authorization.\n\nRedirect addresses determine where an approved response can be delivered.", - "segmentation_method": "sentences" - }, - { - "id": "body-region-2", - "text": "allowed scopes specify the operations available to this application grant types describe the supported authentication flows the service returns the registered identifier and a secret for subsequent requests", - "source_text": "Allowed scopes specify the operations available to this application.\n\nGrant types describe the supported authentication flows.\n\nThe service returns the registered identifier and a secret for subsequent requests.", - "segmentation_method": "sentences" - } - ] - } - }, - "artifact": "Client name.", - "ok": true, - "expected_task_pass": false - }, - { - "id": "body-real-repeated-occurrences-preserved", - "category": "reference-evidence", - "root_cause": "collector_parent_child_text_duplication", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/api-docs", - "task_kind": "body", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "paragraphs", - "regions": [ - { - "id": "body-region-1", - "text": "client name identifies the application requesting authorization redirect addresses determine where an approved response can be delivered allowed scopes specify the operations available to this application grant types describe the supported authentication flows the service returns the registered identifier and a secret for subsequent requests", - "source_text": "Client name identifies the application requesting authorization. Redirect addresses determine where an approved response can be delivered. Allowed scopes specify the operations available to this application. Grant types describe the supported authentication flows. The service returns the registered identifier and a secret for subsequent requests.", - "segmentation_method": "paragraphs" - }, - { - "id": "body-region-2", - "text": "client name identifies the application requesting authorization redirect addresses determine where an approved response can be delivered allowed scopes specify the operations available to this application grant types describe the supported authentication flows the service returns the registered identifier and a secret for subsequent requests", - "source_text": "Client name identifies the application requesting authorization. Redirect addresses determine where an approved response can be delivered. Allowed scopes specify the operations available to this application. Grant types describe the supported authentication flows. The service returns the registered identifier and a secret for subsequent requests.", - "segmentation_method": "paragraphs" - } - ] - } - }, - "artifact": "Client name identifies the application requesting authorization. Redirect addresses determine where an approved response can be delivered. Allowed scopes specify the operations available to this application. Grant types describe the supported authentication flows. The service returns the registered identifier and a secret for subsequent requests.", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scoped-list-new-short-names", - "category": "structured-extraction", - "root_cause": "declared_list_scope_and_nonempty_names", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".main-list a", - "start_pattern": "Main chart", - "end_pattern": "Other recommendations" - }, - "stable_text_anchors": [ - { - "id": "footer", - "alternatives": [ - "Company address" - ] - } - ] - } - }, - "artifact": "Main chart\n[918](/item/1)\n[爱](/item/2)\nOther recommendations\nCompany address", - "ok": true, - "expected_task_pass": true - }, - { - "id": "scoped-list-sidebar-not-main", - "category": "structured-extraction", - "root_cause": "declared_list_scope_and_nonempty_names", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".main-list a", - "start_pattern": "Main chart", - "end_pattern": "Other recommendations" - }, - "stable_text_anchors": [ - { - "id": "footer", - "alternatives": [ - "Company address" - ] - } - ] - } - }, - "artifact": "Main chart\n[918](/item/1)\nOther recommendations\n[爱](/item/2)\nCompany address", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scoped-list-html-valid", - "category": "structured-extraction", - "root_cause": "declared_list_scope_and_nonempty_names", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".main-list a", - "start_pattern": "Main chart", - "end_pattern": "Other recommendations" - }, - "stable_text_anchors": [ - { - "id": "footer", - "alternatives": [ - "Company address" - ] - } - ] - } - }, - "artifact": "<div class=\"main-list\"><a href=\"/item/1\">918</a><a href=\"/item/2\">爱</a></div><footer>Company address</footer>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "scoped-list-html-sidebar", - "category": "structured-extraction", - "root_cause": "declared_list_scope_and_nonempty_names", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".main-list a", - "start_pattern": "Main chart", - "end_pattern": "Other recommendations" - }, - "stable_text_anchors": [ - { - "id": "footer", - "alternatives": [ - "Company address" - ] - } - ] - } - }, - "artifact": "<div class=\"main-list\"><a href=\"/item/1\">918</a></div><aside><a href=\"/item/2\">爱</a></aside><footer>Company address</footer>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scoped-list-empty-name", - "category": "structured-extraction", - "root_cause": "declared_list_scope_and_nonempty_names", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".main-list a", - "start_pattern": "Main chart", - "end_pattern": "Other recommendations" - }, - "stable_text_anchors": [ - { - "id": "footer", - "alternatives": [ - "Company address" - ] - } - ] - } - }, - "artifact": "Main chart\n[](/item/1)\n[爱](/item/2)\nCompany address", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scoped-list-absent-heading", - "category": "structured-extraction", - "root_cause": "declared_list_scope_and_nonempty_names", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".main-list a", - "start_pattern": "Main chart", - "end_pattern": "Other recommendations" - }, - "stable_text_anchors": [ - { - "id": "footer", - "alternatives": [ - "Company address" - ] - } - ] - } - }, - "artifact": "[918](/item/1)\n[爱](/item/2)\nCompany address", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scoped-list-hidden-html", - "category": "structured-extraction", - "root_cause": "declared_list_scope_and_nonempty_names", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".main-list a", - "start_pattern": "Main chart", - "end_pattern": "Other recommendations" - }, - "stable_text_anchors": [ - { - "id": "footer", - "alternatives": [ - "Company address" - ] - } - ] - } - }, - "artifact": "<div class=\"main-list\"><a href=\"/item/1\">918</a><a hidden href=\"/item/2\">爱</a></div><footer>Company address</footer>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "owned-sibling-title-detail-link", - "category": "structured-extraction", - "root_cause": "same_card_title_detail_pairing", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/song/detail\\.htm", - "identity_query_keys": [ - "songId" - ], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".card", - "title_selector": ".name", - "link_selector": "a.detail" - } - } - }, - "artifact": "<div class=\"card\"><p class=\"name\">Real song</p><a class=\"detail\" href=\"/song/detail.htm?songId=123\">Song information</a></div>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "owned-sibling-does-not-cross-card", - "category": "structured-extraction", - "root_cause": "same_card_title_detail_pairing", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/song/detail\\.htm", - "identity_query_keys": [ - "songId" - ], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".card", - "title_selector": ".name", - "link_selector": "a.detail" - } - } - }, - "artifact": "<div class=\"card\"><p class=\"name\">Real song</p></div><div class=\"card\"><a class=\"detail\" href=\"/song/detail.htm?songId=123\">Song information</a></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "owned-sibling-ambiguous-link", - "category": "structured-extraction", - "root_cause": "same_card_title_detail_pairing", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/song/detail\\.htm", - "identity_query_keys": [ - "songId" - ], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".card", - "title_selector": ".name", - "link_selector": "a.detail" - } - } - }, - "artifact": "<div class=\"card\"><p class=\"name\">Real song</p><a class=\"detail\" href=\"/song/detail.htm?songId=123\">Song information</a><a class=\"detail\" href=\"/song/detail.htm?songId=456\">Second song</a></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "owned-sibling-hidden-title", - "category": "structured-extraction", - "root_cause": "same_card_title_detail_pairing", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/song/detail\\.htm", - "identity_query_keys": [ - "songId" - ], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".card", - "title_selector": ".name", - "link_selector": "a.detail" - } + } + ], + "forbidden_any": [ + "Loading Example..." + ] + }, + "artifact": "Virtual DOM\nName Age\nAlice 31\nBob 42", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "forbidden_requirement_hits": [] + } + }, + { + "id": "github-route-links-are-not-repositories", + "category": "acceptance-contract", + "root_cause": "github_route_as_repository", + "failure_mode": "false-positive", + "case": { + "url": "https://github.com/trending", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!trending(?:/|$)|sponsors(?:/|$))[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + "stable_text_anchors": [ + { + "id": "title", + "alternatives": [ + "Trending" + ] + }, + { + "id": "authors", + "alternatives": [ + "Built by" + ] } - }, - "artifact": "<div class=\"card\"><p class=\"name\" hidden>Real song</p><a class=\"detail\" href=\"/song/detail.htm?songId=123\">Song information</a></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "song-detail-literal-dot-not-wildcard", - "category": "structured-extraction", - "root_cause": "same_card_title_detail_pairing", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/song/detail\\.htm", - "identity_query_keys": [ - "songId" - ], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".card", - "title_selector": ".name", - "link_selector": "a.detail" - } + ] + } + }, + "artifact": "Trending\nBuilt by\n- [Python](https://github.com/trending/python)\n- [Explore](https://github.com/sponsors/explore)", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_structure_pass": false, + "stable_unique_record_count": 0 + } + }, + { + "id": "github-owner-repository-links-are-records", + "category": "acceptance-contract", + "root_cause": "github_route_as_repository", + "failure_mode": "false-negative", + "case": { + "url": "https://github.com/trending", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!trending(?:/|$)|sponsors(?:/|$))[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + "stable_text_anchors": [ + { + "id": "title", + "alternatives": [ + "Trending" + ] + }, + { + "id": "authors", + "alternatives": [ + "Built by" + ] } - }, - "artifact": "<div class=\"card\"><p class=\"name\">Real song</p><a class=\"detail\" href=\"/song/detailXhtm?songId=123\">Song information</a></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "owned-title-equivalent-markdown", - "category": "structured-extraction", - "root_cause": "same_card_title_detail_pairing", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/chart", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/song/detail\\.htm", - "identity_query_keys": [ - "songId" - ], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".card", - "title_selector": ".name", - "link_selector": "a.detail" - } + ] + } + }, + "artifact": "Trending\nBuilt by\n- [alpha / project](https://github.com/alpha/project)\n- [beta / tool](https://github.com/beta/tool)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_structure_pass": true, + "stable_unique_record_count": 2 + } + }, + { + "id": "stable-contract-cannot-bypass-incomplete-body", + "category": "body-scope", + "root_cause": "stable_contract_body_conjunction", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "ground_truth_main_content": "Article alpha bravo charlie delta echo foxtrot golf hotel", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 2, + "regions": [ + { + "id": "opening", + "text": "alpha bravo charlie delta" + }, + { + "id": "ending", + "text": "echo foxtrot golf hotel" } + ] }, - "artifact": "[Real song](/song/detail.htm?songId=123)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "product-owned-price-visible", - "category": "structured-extraction", - "root_cause": "source_declared_price_availability", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/search", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "linked_product", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", - "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "price_patterns": [ - "\\$[0-9]+\\.[0-9]{2}" - ], - "price_unavailable_patterns": [ - "Click to see price" - ], - "entry_scope": { - "html_selector": ".product", - "title_selector": ".name", - "link_selector": "a.detail", - "price_selector": ".price" - } + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "identity", + "alternatives": [ + "Article" + ] } - }, - "artifact": "<div class=\"product\"><h2 class=\"name\">Headphones</h2><a class=\"detail\" href=\"/dp/B012345678\">Details</a><span class=\"price\">$29.99</span></div>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "product-explicit-source-price-unavailable", - "category": "structured-extraction", - "root_cause": "source_declared_price_availability", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/search", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "linked_product", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", - "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "price_patterns": [ - "\\$[0-9]+\\.[0-9]{2}" - ], - "price_unavailable_patterns": [ - "Click to see price" - ], - "entry_scope": { - "html_selector": ".product", - "title_selector": ".name", - "link_selector": "a.detail", - "price_selector": ".price" - } + ] + } + }, + "artifact": "Article alpha bravo charlie delta", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_task_pass": true, + "body_coverage_pass": false, + "content_pass": false + } + }, + { + "id": "stable-contract-and-complete-body-can-pass-together", + "category": "body-scope", + "root_cause": "stable_contract_body_conjunction", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "ground_truth_main_content": "Article alpha bravo charlie delta echo foxtrot golf hotel", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 2, + "regions": [ + { + "id": "opening", + "text": "alpha bravo charlie delta" + }, + { + "id": "ending", + "text": "echo foxtrot golf hotel" } + ] }, - "artifact": "<div class=\"product\"><h2 class=\"name\">Headphones</h2><a class=\"detail\" href=\"/dp/B012345678\">Details</a>Click to see price</div>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "product-price-simply-missing", - "category": "structured-extraction", - "root_cause": "source_declared_price_availability", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/search", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "linked_product", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", - "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "price_patterns": [ - "\\$[0-9]+\\.[0-9]{2}" - ], - "price_unavailable_patterns": [ - "Click to see price" - ], - "entry_scope": { - "html_selector": ".product", - "title_selector": ".name", - "link_selector": "a.detail", - "price_selector": ".price" - } + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "identity", + "alternatives": [ + "Article" + ] } - }, - "artifact": "<div class=\"product\"><h2 class=\"name\">Headphones</h2><a class=\"detail\" href=\"/dp/B012345678\">Details</a></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "product-price-cannot-borrow-from-neighbor", - "category": "structured-extraction", - "root_cause": "source_declared_price_availability", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/search", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "linked_product", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", - "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "price_patterns": [ - "\\$[0-9]+\\.[0-9]{2}" - ], - "price_unavailable_patterns": [ - "Click to see price" - ], - "entry_scope": { - "html_selector": ".product", - "title_selector": ".name", - "link_selector": "a.detail", - "price_selector": ".price" - } + ] + } + }, + "artifact": "Article alpha bravo charlie delta echo foxtrot golf hotel", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_task_pass": true, + "body_coverage_pass": true, + "content_pass": true + } + }, + { + "id": "unresolved-rule-cannot-be-counted-as-provider-pass", + "category": "acceptance-contract", + "root_cause": "rule_unresolved_exclusion", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/dynamic", + "task_kind": "body", + "expected_any": [ + "complete answer" + ], + "rule_review": { + "status": "unresolved", + "reason": "fewer_than_three_stable_texts" + } + }, + "artifact": "complete answer", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "rule_review_status": "unresolved" + } + }, + { + "id": "resolved-rule-remains-eligible-for-provider-pass", + "category": "acceptance-contract", + "root_cause": "rule_unresolved_exclusion", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/dynamic", + "task_kind": "body", + "expected_any": [ + "complete answer" + ], + "rule_review": { + "status": "resolved" + } + }, + "artifact": "complete answer", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "rule_review_status": "resolved" + } + }, + { + "id": "structured-regions-reject-a-missing-ending-section", + "category": "body-scope", + "root_cause": "body_region_structure_priority", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/guide", + "task_kind": "body", + "ground_truth_main_content": "Opening alpha bravo charlie delta. Ending echo foxtrot golf hotel.", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 2, + "regions": [ + { + "id": "opening", + "text": "Opening alpha bravo charlie delta" + }, + { + "id": "ending", + "text": "Ending echo foxtrot golf hotel" } - }, - "artifact": "<div class=\"product\"><h2 class=\"name\">Headphones</h2><a class=\"detail\" href=\"/dp/B012345678\">Details</a></div><div class=\"product\"><span class=\"price\">$29.99</span></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "product-unavailable-cannot-borrow", - "category": "structured-extraction", - "root_cause": "source_declared_price_availability", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/search", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "linked_product", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", - "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "price_patterns": [ - "\\$[0-9]+\\.[0-9]{2}" - ], - "price_unavailable_patterns": [ - "Click to see price" - ], - "entry_scope": { - "html_selector": ".product", - "title_selector": ".name", - "link_selector": "a.detail", - "price_selector": ".price" - } + ] + } + }, + "artifact": "Opening alpha bravo charlie delta.", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "body_coverage_pass": false, + "content_pass": false + } + }, + { + "id": "structured-regions-accept-the-complete-body", + "category": "body-scope", + "root_cause": "body_region_structure_priority", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/guide", + "task_kind": "body", + "ground_truth_main_content": "Opening alpha bravo charlie delta. Ending echo foxtrot golf hotel.", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 2, + "regions": [ + { + "id": "opening", + "text": "Opening alpha bravo charlie delta" + }, + { + "id": "ending", + "text": "Ending echo foxtrot golf hotel" } - }, - "artifact": "<div class=\"product\"><h2 class=\"name\">Headphones</h2><a class=\"detail\" href=\"/dp/B012345678\">Details</a></div><div class=\"product\">Click to see price</div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "product-price-markdown-equivalent", - "category": "structured-extraction", - "root_cause": "source_declared_price_availability", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/search", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "linked_product", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", - "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "price_patterns": [ - "\\$[0-9]+\\.[0-9]{2}" - ], - "price_unavailable_patterns": [ - "Click to see price" - ], - "entry_scope": { - "html_selector": ".product", - "title_selector": ".name", - "link_selector": "a.detail", - "price_selector": ".price" - } + ] + } + }, + "artifact": "Opening alpha bravo charlie delta. Ending echo foxtrot golf hotel.", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "body_coverage_pass": true, + "content_pass": true + } + }, + { + "id": "variable-list-23-of-30-does-not-meet-80-percent-source-threshold", + "category": "acceptance-contract", + "root_cause": "source_observed_record_threshold", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/list", + "task_kind": "stable_records", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 24, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "record_count_policy": { + "list_type": "variable", + "source_observed_count": 30, + "required_ratio": 0.8, + "source_observation_counts": [ + 30, + 30, + 30 + ], + "source_evidence_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "reason": "Independent source observation before the provider call." + } + } + }, + "artifact": "[Item 1](https://example.test/item/1)\n[Item 2](https://example.test/item/2)\n[Item 3](https://example.test/item/3)\n[Item 4](https://example.test/item/4)\n[Item 5](https://example.test/item/5)\n[Item 6](https://example.test/item/6)\n[Item 7](https://example.test/item/7)\n[Item 8](https://example.test/item/8)\n[Item 9](https://example.test/item/9)\n[Item 10](https://example.test/item/10)\n[Item 11](https://example.test/item/11)\n[Item 12](https://example.test/item/12)\n[Item 13](https://example.test/item/13)\n[Item 14](https://example.test/item/14)\n[Item 15](https://example.test/item/15)\n[Item 16](https://example.test/item/16)\n[Item 17](https://example.test/item/17)\n[Item 18](https://example.test/item/18)\n[Item 19](https://example.test/item/19)\n[Item 20](https://example.test/item/20)\n[Item 21](https://example.test/item/21)\n[Item 22](https://example.test/item/22)\n[Item 23](https://example.test/item/23)", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_unique_record_count": 23, + "stable_minimum_records": 24, + "stable_task_pass": false + } + }, + { + "id": "variable-list-24-of-30-meets-source-threshold-without-claiming-completeness", + "category": "acceptance-contract", + "root_cause": "source_observed_record_threshold", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/list", + "task_kind": "stable_records", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 24, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "record_count_policy": { + "list_type": "variable", + "source_observed_count": 30, + "required_ratio": 0.8, + "source_observation_counts": [ + 30, + 30, + 30 + ], + "source_evidence_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "reason": "Independent source observation before the provider call." + } + } + }, + "artifact": "[Item 1](https://example.test/item/1)\n[Item 2](https://example.test/item/2)\n[Item 3](https://example.test/item/3)\n[Item 4](https://example.test/item/4)\n[Item 5](https://example.test/item/5)\n[Item 6](https://example.test/item/6)\n[Item 7](https://example.test/item/7)\n[Item 8](https://example.test/item/8)\n[Item 9](https://example.test/item/9)\n[Item 10](https://example.test/item/10)\n[Item 11](https://example.test/item/11)\n[Item 12](https://example.test/item/12)\n[Item 13](https://example.test/item/13)\n[Item 14](https://example.test/item/14)\n[Item 15](https://example.test/item/15)\n[Item 16](https://example.test/item/16)\n[Item 17](https://example.test/item/17)\n[Item 18](https://example.test/item/18)\n[Item 19](https://example.test/item/19)\n[Item 20](https://example.test/item/20)\n[Item 21](https://example.test/item/21)\n[Item 22](https://example.test/item/22)\n[Item 23](https://example.test/item/23)\n[Item 24](https://example.test/item/24)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_unique_record_count": 24, + "stable_minimum_records": 24, + "stable_task_pass": true + } + }, + { + "id": "dynamic-final-page-mismatch-cannot-be-counted-as-a-pass", + "category": "acceptance-contract", + "root_cause": "dynamic_page_identity_alignment", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/start", + "evaluation_url": "https://example.test/start", + "expected_final_url": "https://example.test/en/catalog", + "target_language": "en-US", + "target_region": "US", + "track": "dynamic", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "identity", + "alternatives": [ + "Catalog" + ] } - }, - "artifact": "[Headphones](/dp/B012345678)\n$29.99", - "ok": true, - "expected_task_pass": true - }, - { - "id": "product-unavailable-markdown-equivalent", - "category": "structured-extraction", - "root_cause": "source_declared_price_availability", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/search", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "linked_product", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", - "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "price_patterns": [ - "\\$[0-9]+\\.[0-9]{2}" - ], - "price_unavailable_patterns": [ - "Click to see price" - ], - "entry_scope": { - "html_selector": ".product", - "title_selector": ".name", - "link_selector": "a.detail", - "price_selector": ".price" - } + ] + } + }, + "artifact": "Catalog", + "ok": true, + "final_url": "https://example.test/fr/catalog", + "metadata": { + "route": "dynamic_selected_text", + "document_language": "fr" + }, + "expected_task_pass": false, + "expected_metrics": { + "page_identity_status": "mismatch", + "final_url_consistent": false, + "language_consistent": false + } + }, + { + "id": "dynamic-matching-final-page-and-language-remain-eligible", + "category": "acceptance-contract", + "root_cause": "dynamic_page_identity_alignment", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/start", + "evaluation_url": "https://example.test/start", + "expected_final_url": "https://example.test/en/catalog", + "target_language": "en-US", + "target_region": "US", + "track": "dynamic", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "identity", + "alternatives": [ + "Catalog" + ] } - }, - "artifact": "[Headphones](/dp/B012345678)\nClick to see price", - "ok": true, - "expected_task_pass": true - }, - { - "id": "currency-whitespace-accepts-source-format", - "category": "acceptance-contract", - "root_cause": "overescaped_source_field_patterns", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/list", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/item/[0-9]+", - "page_patterns": [ - "₹\\s*[0-9]" - ] + ] + } + }, + "artifact": "Catalog", + "ok": true, + "final_url": "https://example.test/en/catalog?session=123#items", + "metadata": { + "route": "dynamic_selected_text", + "document_language": "en" + }, + "expected_task_pass": true, + "expected_metrics": { + "page_identity_status": "verified", + "final_url_consistent": true, + "language_consistent": true + } + }, + { + "id": "dynamic-search-query-mismatch-is-a-different-page", + "category": "acceptance-contract", + "root_cause": "dynamic_page_query_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/search?q=Mario", + "evaluation_url": "https://example.test/search?q=Mario", + "expected_final_url": "https://example.test/search?q=Mario", + "target_language": "ja-JP", + "target_region": "JP", + "track": "dynamic", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "identity", + "alternatives": [ + "Search results" + ] } - }, - "artifact": "[Actual item](/item/1)\n₹ 1499", - "ok": true, - "expected_task_pass": true - }, - { - "id": "currency-whitespace-rejects-non-field", - "category": "acceptance-contract", - "root_cause": "overescaped_source_field_patterns", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/list", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/item/[0-9]+", - "page_patterns": [ - "₹\\s*[0-9]" - ] + ] + } + }, + "artifact": "Search results", + "ok": true, + "final_url": "https://example.test/search?q=Zelda", + "metadata": { + "route": "dynamic_selected_text", + "document_language": "ja" + }, + "expected_task_pass": false, + "expected_metrics": { + "page_identity_status": "mismatch", + "final_url_consistent": false + } + }, + { + "id": "dynamic-matching-search-query-remains-eligible", + "category": "acceptance-contract", + "root_cause": "dynamic_page_query_identity", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/search?q=Mario", + "evaluation_url": "https://example.test/search?q=Mario", + "expected_final_url": "https://example.test/search?q=Mario", + "target_language": "ja-JP", + "target_region": "JP", + "track": "dynamic", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "identity", + "alternatives": [ + "Search results" + ] } - }, - "artifact": "[Actual item](/item/1)\nRs unknown", - "ok": true, - "expected_task_pass": false - }, - { - "id": "counter-word-boundaries-accepts-source-format", - "category": "acceptance-contract", - "root_cause": "overescaped_source_field_patterns", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/list", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/item/[0-9]+", - "page_patterns": [ - "\\b[0-9][0-9,]* points\\b" - ] + ] + } + }, + "artifact": "Search results", + "ok": true, + "final_url": "https://example.test/search?q=Mario#results", + "metadata": { + "route": "dynamic_selected_text", + "document_language": "ja" + }, + "expected_task_pass": true, + "expected_metrics": { + "page_identity_status": "verified", + "final_url_consistent": true + } + }, + { + "id": "dynamic-source-document-language-mismatch-is-not-verified", + "category": "acceptance-contract", + "root_cause": "declared_document_language_vs_visible_language", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/rtl", + "evaluation_url": "https://example.test/rtl", + "expected_final_url": "https://example.test/rtl", + "target_language": "ar", + "expected_document_language": "en", + "target_region": "US", + "track": "dynamic", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "arabic-content", + "alternatives": [ + "الشخص" + ] } - }, - "artifact": "[Actual item](/item/1)\n123 points", - "ok": true, - "expected_task_pass": true - }, - { - "id": "counter-word-boundaries-rejects-non-field", - "category": "acceptance-contract", - "root_cause": "overescaped_source_field_patterns", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/list", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/item/[0-9]+", - "page_patterns": [ - "\\b[0-9][0-9,]* points\\b" - ] + ] + } + }, + "artifact": "الشخص", + "ok": true, + "final_url": "https://example.test/rtl", + "metadata": { + "route": "dynamic_selected_text", + "document_language": "fr" + }, + "expected_task_pass": false, + "expected_metrics": { + "page_identity_status": "mismatch", + "language_consistent": false + } + }, + { + "id": "dynamic-arabic-visible-text-with-english-html-lang-remains-eligible", + "category": "acceptance-contract", + "root_cause": "declared_document_language_vs_visible_language", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/rtl", + "evaluation_url": "https://example.test/rtl", + "expected_final_url": "https://example.test/rtl", + "target_language": "ar", + "expected_document_language": "en", + "target_region": "US", + "track": "dynamic", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "arabic-content", + "alternatives": [ + "الشخص" + ] } - }, - "artifact": "[Actual item](/item/1)\n123 appointments", - "ok": true, - "expected_task_pass": false - }, - { - "id": "api-summary-reference-complete", - "category": "reference-evidence", - "root_cause": "api_operation_reference_omission", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/api", - "task_kind": "body", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "paragraphs", - "regions": [ - { - "id": "body-region-1", - "text": "get animals list registered animals and their current attributes post animals register a new animal using the submitted data delete animals id remove the animal identified by its identifier", - "source_text": "GET /animals List registered animals and their current attributes.\n\nPOST /animals Register a new animal using the submitted data.\n\nDELETE /animals/{id} Remove the animal identified by its identifier.", - "segmentation_method": "paragraphs" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "api_operation", - "minimum_records": 3 + ] + } + }, + "artifact": "الشخص", + "ok": true, + "final_url": "https://example.test/rtl", + "metadata": { + "route": "dynamic_selected_text", + "document_language": "en" + }, + "expected_task_pass": true, + "expected_metrics": { + "page_identity_status": "verified", + "language_consistent": true + } + }, + { + "id": "name-link-count-cannot-credit-an-invented-name-for-a-real-url", + "category": "acceptance-contract", + "root_cause": "frozen_source_name_link_pair_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/list", + "task_kind": "stable_records", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "maximum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "source_record_pair_hashes": [ + "2802c3d298ecbed8c44da81bb83f66e6349bbf3062abed780e51293265c3ee72", + "75002de276f7af8bf713de877ebf72e6b939ded0e48805943254d7bd2b54e714" + ], + "source_record_pairs_sha256": "8a8eb383a10ea9338de82cd9f9d2b58f970191e3260dbacbd08a7e7f5f7335ea" + } + }, + "artifact": "[Invented item](https://example.test/item/1)\n[Item 2](https://example.test/item/2)", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_source_matched_record_count": 1, + "stable_source_mismatched_record_count": 1, + "stable_task_pass": false + } + }, + { + "id": "name-link-count-credits-two-independently-observed-name-url-pairs", + "category": "acceptance-contract", + "root_cause": "frozen_source_name_link_pair_membership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/list", + "task_kind": "stable_records", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "maximum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "source_record_pair_hashes": [ + "2802c3d298ecbed8c44da81bb83f66e6349bbf3062abed780e51293265c3ee72", + "75002de276f7af8bf713de877ebf72e6b939ded0e48805943254d7bd2b54e714" + ], + "source_record_pairs_sha256": "8a8eb383a10ea9338de82cd9f9d2b58f970191e3260dbacbd08a7e7f5f7335ea" + } + }, + "artifact": "[Item 1](https://example.test/item/1)\n[Item 2](https://example.test/item/2)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_source_matched_record_count": 2, + "stable_source_mismatched_record_count": 0, + "stable_task_pass": true + } + }, + { + "id": "volatile-body-stable-text-does-not-freeze-current-members", + "category": "acceptance-contract", + "root_cause": "volatile_body_snapshot_used_as_invariant_answer", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/live-events", + "task_kind": "body", + "ground_truth_main_content": "40 Events. Old event at 10:00", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Events, Past Day" + ] + }, + { + "id": "sort", + "alternatives": [ + "Newest First" + ] + }, + { + "id": "settings", + "alternatives": [ + "Settings" + ] } + ] }, - "artifact": "GET /animals List registered animals and their current attributes.\n\nPOST /animals Register a new animal using the submitted data.\n\nDELETE /animals/{id} Remove the animal identified by its identifier.", - "ok": true, - "expected_task_pass": true - }, - { - "id": "api-paths-alone-not-summaries", - "category": "reference-evidence", - "root_cause": "api_operation_reference_omission", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/api", - "task_kind": "body", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "paragraphs", - "regions": [ - { - "id": "body-region-1", - "text": "get animals list registered animals and their current attributes post animals register a new animal using the submitted data delete animals id remove the animal identified by its identifier", - "source_text": "GET /animals List registered animals and their current attributes.\n\nPOST /animals Register a new animal using the submitted data.\n\nDELETE /animals/{id} Remove the animal identified by its identifier.", - "segmentation_method": "paragraphs" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "api_operation", - "minimum_records": 3 - } - }, - "artifact": "/animals\n/animals/{id}\nAPI documentation", - "ok": true, - "expected_task_pass": false - }, - { - "id": "api-missing-operation-summary", - "category": "reference-evidence", - "root_cause": "api_operation_reference_omission", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/api", - "task_kind": "body", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "paragraphs", - "regions": [ - { - "id": "body-region-1", - "text": "get animals list registered animals and their current attributes post animals register a new animal using the submitted data delete animals id remove the animal identified by its identifier", - "source_text": "GET /animals List registered animals and their current attributes.\n\nPOST /animals Register a new animal using the submitted data.\n\nDELETE /animals/{id} Remove the animal identified by its identifier.", - "segmentation_method": "paragraphs" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "api_operation", - "minimum_records": 3 - } - }, - "artifact": "GET /animals List registered animals and their current attributes.\n\nPOST /animals Register a new animal using the submitted data.\n\nDELETE /animals/{id} ", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-real-row-shape", - "category": "structured-extraction", - "root_cause": "table_record_field_ownership", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" - } - } - }, - "artifact": "The table below has 1000 rows\n1Ada Example20female4blue05/12/2025\n2Bob Example70male3green22/05/2026", - "ok": true, - "expected_task_pass": true - }, - { - "id": "table-date-list-not-records", - "category": "structured-extraction", - "root_cause": "table_record_field_ownership", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" - } - } - }, - "artifact": "The table below has 1000 rows\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-missing-cell-not-complete", - "category": "structured-extraction", - "root_cause": "table_record_field_ownership", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" - } - } - }, - "artifact": "The table below has 1000 rows\n1Ada Example20female4blue05/12/2025\n2Bob Example70male322/05/2026", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-invalid-calendar-not-complete", - "category": "structured-extraction", - "root_cause": "table_record_field_ownership", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" - } - } - }, - "artifact": "The table below has 1000 rows\n1Ada Example20female4blue05/12/2025\n2Bob Example70male3green31/02/2026", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-duplicate-id-not-extra-record", - "category": "structured-extraction", - "root_cause": "table_record_field_ownership", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" - } - } - }, - "artifact": "The table below has 1000 rows\n1Ada Example20female4blue05/12/2025\n1Bob Example70male3green22/05/2026", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-other-section-cannot-fill", - "category": "structured-extraction", - "root_cause": "table_record_field_ownership", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" - } - } - }, - "artifact": "The table below has 1000 rows\n1Ada Example20female4blue05/12/2025\nView Source\n2Bob Example70male3green22/05/2026", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-markdown-row-ownership", - "category": "structured-extraction", - "root_cause": "table_record_field_ownership", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" - } - } - }, - "artifact": "The table below has 1000 rows\n| ID | Name | Progress | Gender | Rating | Favourite Color | Date Of Birth |\n| --- | --- | --- | --- | --- | --- | --- |\n| 1 | Ada Example | 20 | female | 4 | blue | 05/12/2025 |\n| 2 | Bob Example | 70 | male | 3 | green | 22/05/2026 |", - "ok": true, - "expected_task_pass": true - }, - { - "id": "body-reference-excludes-icon-font-name", - "category": "reference-evidence", - "root_cause": "body_reference_nontext_controls", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/virtual-list", - "task_kind": "body", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "sentences", - "regions": [ - { - "id": "body-region-1", - "text": "basic virtual list displays the initial mounted options only records present after automatic loading belong to this observation scrolling loads additional records outside this scope visible option names and their positions form the body of this example", - "source_text": "Basic virtual list displays the initial mounted options.\n\nOnly records present after automatic loading belong to this observation; scrolling loads additional records outside this scope.\n\nVisible option names and their positions form the body of this example.", - "segmentation_method": "sentences" - } - ] - } - }, - "artifact": "Basic virtual list displays the initial mounted options. Only records present after automatic loading belong to this observation; scrolling loads additional records outside this scope. Visible option names and their positions form the body of this example.", - "ok": true, - "expected_task_pass": true - }, - { - "id": "body-icon-label-is-not-real-body", - "category": "reference-evidence", - "root_cause": "body_reference_nontext_controls", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/virtual-list", - "task_kind": "body", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "sentences", - "regions": [ - { - "id": "body-region-1", - "text": "basic virtual list displays the initial mounted options only records present after automatic loading belong to this observation scrolling loads additional records outside this scope visible option names and their positions form the body of this example", - "source_text": "Basic virtual list displays the initial mounted options.\n\nOnly records present after automatic loading belong to this observation; scrolling loads additional records outside this scope.\n\nVisible option names and their positions form the body of this example.", - "segmentation_method": "sentences" - } - ] - } - }, - "artifact": "code expand_more content_copy", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scoped-visible-news-title", - "category": "structured-extraction", - "root_cause": "scoped_visible_record_names", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/article/\\d+$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main a" - } - } - }, - "artifact": "<main><a style=\"display: grid; opacity: 1\" href=\"/article/1\">A new headline</a></main>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "scoped-hidden-news-ancestor", - "category": "structured-extraction", - "root_cause": "scoped_visible_record_names", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/article/\\d+$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main a" - } - } - }, - "artifact": "<main><div style=\"DISPLAY: none !important\"><a href=\"/article/1\">Hidden headline</a></div></main>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scoped-hidden-news-own-style", - "category": "structured-extraction", - "root_cause": "scoped_visible_record_names", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/article/\\d+$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main a" - } - } - }, - "artifact": "<main><a style=\"visibility:hidden\" href=\"/article/1\">Hidden headline</a></main>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scoped-news-image-alt-is-not-title", - "category": "structured-extraction", - "root_cause": "scoped_visible_record_names", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/article/\\d+$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main a" - } - } - }, - "artifact": "<main><h2>Product recommendations</h2><a href=\"/article/1\"><img alt=\"A news headline\" src=\"x.png\"></a></main>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scoped-news-hidden-copy-not-extra-record", - "category": "structured-extraction", - "root_cause": "scoped_visible_record_names", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/article/\\d+$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main a" - } - } - }, - "artifact": "<main><a href=\"/article/1\">Visible headline</a><div style=\"opacity:0\"><a href=\"/article/2\">Hidden headline</a></div></main>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "first-news-before-sidebar", - "category": "structured-extraction", - "root_cause": "declared_list_scope_and_nonempty_names", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/article/\\d+$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main a", - "end_pattern": "^[ \\t#*]*产品推荐[ \\t*]*$" - } - } - }, - "artifact": "[New headline](/article/1)\n产品推荐\n[Another](/article/2)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "sidebar-cannot-fill-first-news", - "category": "structured-extraction", - "root_cause": "declared_list_scope_and_nonempty_names", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/article/\\d+$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main a", - "end_pattern": "^[ \\t#*]*产品推荐[ \\t*]*$" - } - } - }, - "artifact": "产品推荐\n[Another](/article/2)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "news-hotissue-route-is-valid", - "category": "structured-extraction", - "root_cause": "observed_article_route_aliases", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/(?:mnews/(?:hotissue/)?)?article/[0-9]+/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main a" - }, - "entry_identity_pattern": "/article/([0-9]+/[0-9]+)$" - } - }, - "artifact": "<main><a href=\"/mnews/hotissue/article/015/0005334180\">New headline</a></main>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "news-extra-route-component-is-invalid", - "category": "structured-extraction", - "root_cause": "observed_article_route_aliases", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/(?:mnews/(?:hotissue/)?)?article/[0-9]+/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main a" - }, - "entry_identity_pattern": "/article/([0-9]+/[0-9]+)$" - } - }, - "artifact": "<main><a href=\"/mnews/unrelated/article/015/0005334180\">New headline</a></main>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "news-alias-is-one-article", - "category": "structured-extraction", - "root_cause": "observed_article_route_aliases", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/(?:mnews/(?:hotissue/)?)?article/[0-9]+/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main a" - }, - "entry_identity_pattern": "/article/([0-9]+/[0-9]+)$" - } - }, - "artifact": "<main><a href=\"/article/015/0005334180\">Headline</a><a href=\"/mnews/hotissue/article/015/0005334180\">Headline</a></main>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "ordinary-label-not-screenreader-caption", - "category": "acceptance-contract", - "root_cause": "screenreader_only_reference", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/article/[0-9]+", - "name_validation": "nonempty", - "stable_text_anchors": [ - { - "id": "visible-search-label", - "alternatives": [ - "Recherche" - ] - } - ] - } - }, - "artifact": "Recherche\n[Actual headline](/article/1)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "unrelated-assistive-caption-does-not-replace-visible-label", - "category": "acceptance-contract", - "root_cause": "screenreader_only_reference", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/article/[0-9]+", - "name_validation": "nonempty", - "stable_text_anchors": [ - { - "id": "visible-search-label", - "alternatives": [ - "Recherche" - ] - } - ] - } - }, - "artifact": "Abonnement Newsletter\n[Actual headline](/article/1)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "visible-label-does-not-replace-news", - "category": "acceptance-contract", - "root_cause": "screenreader_only_reference", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/article/[0-9]+", - "name_validation": "nonempty", - "stable_text_anchors": [ - { - "id": "visible-search-label", - "alternatives": [ - "Recherche" - ] - } - ] - } - }, - "artifact": "Recherche", - "ok": true, - "expected_task_pass": false - }, - { - "id": "visible-title-with-aria-hidden-still-counts", - "category": "structured-extraction", - "root_cause": "aria_metadata_is_not_visual_visibility", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/article/\\d+$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main a" - } - } - }, - "artifact": "<main><a href=\"/article/1\"><span aria-hidden=\"true\">Actual headline</span></a></main>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "aria-hidden-title-with-display-none-does-not-count", - "category": "structured-extraction", - "root_cause": "aria_metadata_is_not_visual_visibility", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/news", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/article/\\d+$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main a" - } - } - }, - "artifact": "<main><a href=\"/article/1\"><span aria-hidden=\"true\" style=\"display:none\">Hidden headline</span></a></main>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "loaded-page-articles-html", - "category": "structured-extraction", - "root_cause": "initial_loaded_page_article_scope", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/article/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a[href]" - }, - "title_exclusion_patterns": [ - "Politique" - ] - } - }, - "artifact": "<header><a href=\"/article/1\">Independent climate dashboard</a></header><main><a href=\"/article/2\">New headline</a></main>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "loaded-page-articles-markdown", - "category": "structured-extraction", - "root_cause": "initial_loaded_page_article_scope", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/article/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a[href]" - }, - "title_exclusion_patterns": [ - "Politique" - ] - } - }, - "artifact": "[Independent climate dashboard](/article/1)\n[New headline](/article/2)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "loaded-carousel-records-count", - "category": "structured-extraction", - "root_cause": "initial_loaded_page_article_scope", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/article/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a[href]" - }, - "title_exclusion_patterns": [ - "Politique" - ] - } - }, - "artifact": "<main><div style=\"width:20px;overflow:hidden\"><div style=\"width:1000px\"><a href=\"/article/1\">First story</a><a href=\"/article/2\">Second story</a></div></div></main>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "category-label-is-not-article-name", - "category": "structured-extraction", - "root_cause": "initial_loaded_page_article_scope", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/article/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a[href]" - }, - "title_exclusion_patterns": [ - "Politique" - ] - } - }, - "artifact": "[Politique](/article/1)\n[New headline](/article/2)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "same-article-repeat-cannot-fill-page-count", - "category": "structured-extraction", - "root_cause": "initial_loaded_page_article_scope", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/article/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a[href]" - }, - "title_exclusion_patterns": [ - "Politique" - ] - } - }, - "artifact": "[First story](/article/1)\n[First story](/article/1?from=sidebar)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "page-article-wrong-path-cannot-fill-count", - "category": "structured-extraction", - "root_cause": "initial_loaded_page_article_scope", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.test" - ], - "entry_path": "/article/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a[href]" - }, - "title_exclusion_patterns": [ - "Politique" - ] - } - }, - "artifact": "[First story](/article/1)\n[Category](/category/2)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "redirected-relative-article-uses-response-origin", - "category": "structured-extraction", - "root_cause": "redirected_document_link_base", - "failure_mode": "false-negative", - "case": { - "url": "https://old.example.test/start", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "new.example.test" - ], - "entry_path": "/article/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a[href]" - } - } - }, - "artifact": "<main><a href=\"/article/1\">Actual headline</a></main>", - "final_url": "https://new.example.test/news/", - "ok": true, - "expected_task_pass": true - }, - { - "id": "wrong-redirect-cannot-borrow-original-origin", - "category": "structured-extraction", - "root_cause": "redirected_document_link_base", - "failure_mode": "false-positive", - "case": { - "url": "https://old.example.test/start", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "old.example.test" - ], - "entry_path": "/article/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a[href]" - } - } - }, - "artifact": "<main><a href=\"/article/1\">Actual headline</a></main>", - "final_url": "https://other.example.test/news/", - "ok": true, - "expected_task_pass": false - }, - { - "id": "first-scroller-new-items-pass", - "category": "acceptance-contract", - "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/scroller", - "task_kind": "body", - "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "virtualscroller renders large amounts of data efficiently", - "source_text": "virtualscroller renders large amounts of data efficiently", - "segmentation_method": "balanced_tokens_fallback" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "Item #\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".demo", - "root_index": 0, - "row_selector": ".items > div", - "cell_selector": ":scope", - "start_pattern": "(?m)^(?=Item #\\d+)", - "end_pattern": "(?m)^PREVIEW$", - "plain_row_pattern": "(?P<item>Item #\\d+)" - } - } - }, - "artifact": "VirtualScroller renders large amounts of data efficiently.\nItem #40\nItem #41", - "ok": true, - "expected_task_pass": true - }, - { - "id": "scroller-controls-not-required-body", - "category": "acceptance-contract", - "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/scroller", - "task_kind": "body", - "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "virtualscroller renders large amounts of data efficiently", - "source_text": "virtualscroller renders large amounts of data efficiently", - "segmentation_method": "balanced_tokens_fallback" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "Item #\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".demo", - "root_index": 0, - "row_selector": ".items > div", - "cell_selector": ":scope", - "start_pattern": "(?m)^(?=Item #\\d+)", - "end_pattern": "(?m)^PREVIEW$", - "plain_row_pattern": "(?P<item>Item #\\d+)" - } - } - }, - "artifact": "VirtualScroller renders large amounts of data efficiently.<div class=\"demo\"><div class=\"items\"><div>Item #40</div><div>Item #41</div></div></div>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "other-scroller-cannot-fill-first", - "category": "acceptance-contract", - "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/scroller", - "task_kind": "body", - "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "virtualscroller renders large amounts of data efficiently", - "source_text": "virtualscroller renders large amounts of data efficiently", - "segmentation_method": "balanced_tokens_fallback" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "Item #\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".demo", - "root_index": 0, - "row_selector": ".items > div", - "cell_selector": ":scope", - "start_pattern": "(?m)^(?=Item #\\d+)", - "end_pattern": "(?m)^PREVIEW$", - "plain_row_pattern": "(?P<item>Item #\\d+)" - } - } - }, - "artifact": "VirtualScroller renders large amounts of data efficiently.<div class=\"demo\"><div class=\"items\"><div>Item #40</div></div></div><div class=\"demo\"><div class=\"items\"><div>Item #41</div></div></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scroller-description-alone-fails", - "category": "acceptance-contract", - "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/scroller", - "task_kind": "body", - "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "virtualscroller renders large amounts of data efficiently", - "source_text": "virtualscroller renders large amounts of data efficiently", - "segmentation_method": "balanced_tokens_fallback" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "Item #\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".demo", - "root_index": 0, - "row_selector": ".items > div", - "cell_selector": ":scope", - "start_pattern": "(?m)^(?=Item #\\d+)", - "end_pattern": "(?m)^PREVIEW$", - "plain_row_pattern": "(?P<item>Item #\\d+)" - } - } - }, - "artifact": "VirtualScroller renders large amounts of data efficiently.\nPREVIEW", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scroller-duplicate-item-fails", - "category": "acceptance-contract", - "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/scroller", - "task_kind": "body", - "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "virtualscroller renders large amounts of data efficiently", - "source_text": "virtualscroller renders large amounts of data efficiently", - "segmentation_method": "balanced_tokens_fallback" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "Item #\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".demo", - "root_index": 0, - "row_selector": ".items > div", - "cell_selector": ":scope", - "start_pattern": "(?m)^(?=Item #\\d+)", - "end_pattern": "(?m)^PREVIEW$", - "plain_row_pattern": "(?P<item>Item #\\d+)" - } - } - }, - "artifact": "VirtualScroller renders large amounts of data efficiently.\nItem #40\nItem #40", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scroller-missing-description-fails", - "category": "acceptance-contract", - "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/scroller", - "task_kind": "body", - "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "virtualscroller renders large amounts of data efficiently", - "source_text": "virtualscroller renders large amounts of data efficiently", - "segmentation_method": "balanced_tokens_fallback" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "Item #\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".demo", - "root_index": 0, - "row_selector": ".items > div", - "cell_selector": ":scope", - "start_pattern": "(?m)^(?=Item #\\d+)", - "end_pattern": "(?m)^PREVIEW$", - "plain_row_pattern": "(?P<item>Item #\\d+)" - } - } - }, - "artifact": "Item #40\nItem #41", - "ok": true, - "expected_task_pass": false - }, - { - "id": "scroller-malformed-label-fails", - "category": "acceptance-contract", - "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/scroller", - "task_kind": "body", - "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "virtualscroller renders large amounts of data efficiently", - "source_text": "virtualscroller renders large amounts of data efficiently", - "segmentation_method": "balanced_tokens_fallback" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "Item #\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".demo", - "root_index": 0, - "row_selector": ".items > div", - "cell_selector": ":scope", - "start_pattern": "(?m)^(?=Item #\\d+)", - "end_pattern": "(?m)^PREVIEW$", - "plain_row_pattern": "(?P<item>Item #\\d+)" - } - } - }, - "artifact": "VirtualScroller renders large amounts of data efficiently.<div class=\"demo\"><div class=\"items\"><div>Item #40</div><div>Broken</div></div></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "html-code-example-does-not-change-plain-table-format", - "category": "structured-extraction", - "root_cause": "embedded_examples_are_not_document_format", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/scroller", - "task_kind": "body", - "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "virtualscroller renders large amounts of data efficiently", - "source_text": "virtualscroller renders large amounts of data efficiently", - "segmentation_method": "balanced_tokens_fallback" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "Item #\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".demo", - "root_index": 0, - "row_selector": ".items > div", - "cell_selector": ":scope", - "start_pattern": "(?m)^(?=Item #\\d+)", - "end_pattern": "(?m)^PREVIEW$", - "plain_row_pattern": "(?P<item>Item #\\d+)" - } - } - }, - "artifact": "VirtualScroller renders large amounts of data efficiently.\nItem #40\nItem #41\nPREVIEW\nUsage\n<div class=\"sample-code\">Example markup</div>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "fenced-example-cannot-fill-live-rows", - "category": "structured-extraction", - "root_cause": "embedded_examples_are_not_document_format", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/scroller", - "task_kind": "body", - "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "virtualscroller renders large amounts of data efficiently", - "source_text": "virtualscroller renders large amounts of data efficiently", - "segmentation_method": "balanced_tokens_fallback" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "Item #\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".demo", - "root_index": 0, - "row_selector": ".items > div", - "cell_selector": ":scope", - "start_pattern": "(?m)^(?=Item #\\d+)", - "end_pattern": "(?m)^PREVIEW$", - "plain_row_pattern": "(?P<item>Item #\\d+)" - } - } - }, - "artifact": "VirtualScroller renders large amounts of data efficiently.\n```html\n<div class=\"demo\"><div class=\"items\"><div>Item #40</div><div>Item #41</div></div></div>\n```", - "ok": true, - "expected_task_pass": false - }, - { - "id": "whole-fenced-data-answer-remains-valid", - "category": "structured-extraction", - "root_cause": "embedded_examples_are_not_document_format", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/scroller", - "task_kind": "body", - "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "virtualscroller renders large amounts of data efficiently", - "source_text": "virtualscroller renders large amounts of data efficiently", - "segmentation_method": "balanced_tokens_fallback" - } - ] - }, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "Item #\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".demo", - "root_index": 0, - "row_selector": ".items > div", - "cell_selector": ":scope", - "start_pattern": "(?m)^(?=Item #\\d+)", - "end_pattern": "(?m)^PREVIEW$", - "plain_row_pattern": "(?P<item>Item #\\d+)" - } - } - }, - "artifact": "```text\nVirtualScroller renders large amounts of data efficiently.\nItem #40\nItem #41\n```", - "ok": true, - "expected_task_pass": true - }, - { - "id": "fixed-demo-seven-of-eight-missing-0", - "category": "acceptance-contract", - "root_cause": "short_repeated_demo_body_uses_record_coverage", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/demo", - "task_kind": "body", - "ground_truth_main_content": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7", - "body_requirements": {}, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*[0-7]" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*" - }, - "stable_text_anchors": [ - { - "id": "preview", - "alternatives": [ - "PREVIEW" - ] - } - ] - } - }, - "artifact": "Item #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7\nPREVIEW", - "ok": true, - "expected_task_pass": true - }, - { - "id": "fixed-demo-seven-of-eight-missing-4", - "category": "acceptance-contract", - "root_cause": "short_repeated_demo_body_uses_record_coverage", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/demo", - "task_kind": "body", - "ground_truth_main_content": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7", - "body_requirements": {}, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*[0-7]" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*" - }, - "stable_text_anchors": [ - { - "id": "preview", - "alternatives": [ - "PREVIEW" - ] - } - ] - } - }, - "artifact": "Item #0\nItem #1\nItem #2\nItem #3\nItem #5\nItem #6\nItem #7\nPREVIEW", - "ok": true, - "expected_task_pass": true - }, - { - "id": "fixed-demo-seven-of-eight-missing-7", - "category": "acceptance-contract", - "root_cause": "short_repeated_demo_body_uses_record_coverage", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/demo", - "task_kind": "body", - "ground_truth_main_content": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7", - "body_requirements": {}, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*[0-7]" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*" - }, - "stable_text_anchors": [ - { - "id": "preview", - "alternatives": [ - "PREVIEW" - ] - } - ] - } - }, - "artifact": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nPREVIEW", - "ok": true, - "expected_task_pass": true - }, - { - "id": "fixed-demo-wrong-initial-indices", - "category": "acceptance-contract", - "root_cause": "short_repeated_demo_body_uses_record_coverage", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/demo", - "task_kind": "body", - "ground_truth_main_content": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7", - "body_requirements": {}, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*[0-7]" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*" - }, - "stable_text_anchors": [ - { - "id": "preview", - "alternatives": [ - "PREVIEW" - ] - } - ] - } - }, - "artifact": "Item #40\nItem #41\nItem #42\nItem #43\nItem #44\nItem #45\nItem #46\nItem #47\nPREVIEW", - "ok": true, - "expected_task_pass": false - }, - { - "id": "fixed-demo-toolbar-only", - "category": "acceptance-contract", - "root_cause": "short_repeated_demo_body_uses_record_coverage", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/demo", - "task_kind": "body", - "ground_truth_main_content": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7", - "body_requirements": {}, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*[0-7]" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*" - }, - "stable_text_anchors": [ - { - "id": "preview", - "alternatives": [ - "PREVIEW" - ] - } - ] - } - }, - "artifact": "PREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "fixed-demo-six-is-below-eighty-percent", - "category": "acceptance-contract", - "root_cause": "short_repeated_demo_body_uses_record_coverage", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/demo", - "task_kind": "body", - "ground_truth_main_content": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7", - "body_requirements": {}, - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*[0-7]" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*" - }, - "stable_text_anchors": [ - { - "id": "preview", - "alternatives": [ - "PREVIEW" - ] - } - ] - } - }, - "artifact": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nPREVIEW", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-token-and-auxiliary-leading-zero-postal", - "category": "structured-extraction", - "root_cause": "typed_table_values_and_auxiliary_text", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" - } - } - }, - "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr><tr><td>Bob Example</td><td>04362-9499</td><td>bob@example.test</td></tr></tbody></table>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "table-token-and-auxiliary-auxiliary-label", - "category": "structured-extraction", - "root_cause": "typed_table_values_and_auxiliary_text", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" - } - } - }, - "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test<span class=\"screen-reader-only\">opens a new window</span></td></tr><tr><td>Bob Example</td><td>04362-9499</td><td>bob@example.test</td></tr></tbody></table>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "table-token-and-auxiliary-invalid-postal", - "category": "structured-extraction", - "root_cause": "typed_table_values_and_auxiliary_text", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" - } - } - }, - "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr><tr><td>Bob Example</td><td>bad</td><td>bob@example.test</td></tr></tbody></table>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-token-and-auxiliary-missing-email", - "category": "structured-extraction", - "root_cause": "typed_table_values_and_auxiliary_text", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" - } - } - }, - "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr><tr><td>Bob Example</td><td>04362-9499</td></tr></tbody></table>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-token-and-auxiliary-duplicate-identity", - "category": "structured-extraction", - "root_cause": "typed_table_values_and_auxiliary_text", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" - } - } - }, - "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr></tbody></table>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-token-and-auxiliary-hidden-email-cannot-fill", - "category": "structured-extraction", - "root_cause": "typed_table_values_and_auxiliary_text", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" - } - } - }, - "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr><tr><td>Bob Example</td><td>04362-9499</td><td><span class=\"screen-reader-only\">bob@example.test</span></td></tr></tbody></table>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-token-and-auxiliary-later-table-cannot-fill", - "category": "structured-extraction", - "root_cause": "typed_table_values_and_auxiliary_text", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" - } - } - }, - "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr></tbody></table><table><tbody><tr><td>Bob Example</td><td>04362-9499</td><td>bob@example.test</td></tr></tbody></table>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-token-and-auxiliary-changed-realistic-values", - "category": "structured-extraction", - "root_cause": "typed_table_values_and_auxiliary_text", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" - } - } - }, - "artifact": "<table><tbody><tr><td>Carol New</td><td>12456</td><td>carol@example.test</td></tr><tr><td>Dana New</td><td>00888</td><td>dana@example.test</td></tr></tbody></table>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "table-multiline-complete", - "category": "structured-extraction", - "root_cause": "multiline_table_row_boundaries", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)", - "plain_multiline_row_pattern": "(?m)^(?P<name>[^\\d\\n]+)\\n(?P<postal>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" - } - } - }, - "artifact": "People\nAda Sample\n00123\nada@example.test\nEdit\nBob Example\n04362-9499\nbob@example.test\nEdit", - "ok": true, - "expected_task_pass": true - }, - { - "id": "table-multiline-missing-field", - "category": "structured-extraction", - "root_cause": "multiline_table_row_boundaries", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)", - "plain_multiline_row_pattern": "(?m)^(?P<name>[^\\d\\n]+)\\n(?P<postal>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" - } - } - }, - "artifact": "People\nAda Sample\n00123\nada@example.test\nEdit\nBob Example\n04362-9499\nEdit", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-multiline-cannot-cross-row", - "category": "structured-extraction", - "root_cause": "multiline_table_row_boundaries", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)", - "plain_multiline_row_pattern": "(?m)^(?P<name>[^\\d\\n]+)\\n(?P<postal>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" - } - } - }, - "artifact": "People\nAda Sample\n00123\nEdit\nBob Example\n04362-9499\nbob@example.test\nEdit", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-multiline-other-example", - "category": "structured-extraction", - "root_cause": "multiline_table_row_boundaries", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)", - "plain_multiline_row_pattern": "(?m)^(?P<name>[^\\d\\n]+)\\n(?P<postal>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" - } - } - }, - "artifact": "People\nAda Sample\n00123\nada@example.test\nEdit\nNext example\nBob Example\n04362-9499\nbob@example.test\nEdit", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-multiline-mixed-format", - "category": "structured-extraction", - "root_cause": "multiline_table_row_boundaries", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "postal", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "email" - ], - "table_scope": { - "root_selector": "table", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td", - "exclude_selector": ".screen-reader-only", - "start_pattern": "People\\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)", - "plain_multiline_row_pattern": "(?m)^(?P<name>[^\\d\\n]+)\\n(?P<postal>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" - } - } - }, - "artifact": "People\nAda Sample\n00123\nada@example.test\nEdit\nBob Example 04362-9499 bob@example.test", - "ok": true, - "expected_task_pass": true - }, - { - "id": "rsuite-complete-record-sixteen-complete", - "category": "acceptance-contract", - "root_cause": "volatile_demo_people_not_frozen_body", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 16, - "table_columns": [ - { - "name": "id", - "label": "Id", - "type": "integer", - "minimum": 1 - }, - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "gender", - "label": "Gender", - "type": "text", - "pattern": "(?i)female|male" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "maximum": 150 - }, - { - "name": "postcode", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "root_selector": ".rs-table", - "root_index": 0, - "row_selector": "[role=row]:not(.rs-table-row-header)", - "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", - "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", - "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", - "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", - "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" - } - }, - "body_requirements": {} - }, - "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n1\nNew\nPerson\nfemale\n35\n00123\nperson1@example.test\nEdit\n2\nNew\nPerson\nfemale\n35\n00123\nperson2@example.test\nEdit\n3\nNew\nPerson\nfemale\n35\n00123\nperson3@example.test\nEdit\n4\nNew\nPerson\nfemale\n35\n00123\nperson4@example.test\nEdit\n5\nNew\nPerson\nfemale\n35\n00123\nperson5@example.test\nEdit\n6\nNew\nPerson\nfemale\n35\n00123\nperson6@example.test\nEdit\n7\nNew\nPerson\nfemale\n35\n00123\nperson7@example.test\nEdit\n8\nNew\nPerson\nfemale\n35\n00123\nperson8@example.test\nEdit\n9\nNew\nPerson\nfemale\n35\n00123\nperson9@example.test\nEdit\n10\nNew\nPerson\nfemale\n35\n00123\nperson10@example.test\nEdit\n11\nNew\nPerson\nfemale\n35\n00123\nperson11@example.test\nEdit\n12\nNew\nPerson\nfemale\n35\n00123\nperson12@example.test\nEdit\n13\nNew\nPerson\nfemale\n35\n00123\nperson13@example.test\nEdit\n14\nNew\nPerson\nfemale\n35\n00123\nperson14@example.test\nEdit\n15\nNew\nPerson\nfemale\n35\n00123\nperson15@example.test\nEdit\n16\nNew\nPerson\nfemale\n35\n00123\nperson16@example.test\nEdit\nLoading...", - "ok": true, - "expected_task_pass": true - }, - { - "id": "rsuite-complete-record-fifteen-incomplete", - "category": "acceptance-contract", - "root_cause": "volatile_demo_people_not_frozen_body", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 16, - "table_columns": [ - { - "name": "id", - "label": "Id", - "type": "integer", - "minimum": 1 - }, - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "gender", - "label": "Gender", - "type": "text", - "pattern": "(?i)female|male" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "maximum": 150 - }, - { - "name": "postcode", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "root_selector": ".rs-table", - "root_index": 0, - "row_selector": "[role=row]:not(.rs-table-row-header)", - "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", - "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", - "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", - "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", - "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" - } - }, - "body_requirements": {} - }, - "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n1\nNew\nPerson\nfemale\n35\n00123\nperson1@example.test\nEdit\n2\nNew\nPerson\nfemale\n35\n00123\nperson2@example.test\nEdit\n3\nNew\nPerson\nfemale\n35\n00123\nperson3@example.test\nEdit\n4\nNew\nPerson\nfemale\n35\n00123\nperson4@example.test\nEdit\n5\nNew\nPerson\nfemale\n35\n00123\nperson5@example.test\nEdit\n6\nNew\nPerson\nfemale\n35\n00123\nperson6@example.test\nEdit\n7\nNew\nPerson\nfemale\n35\n00123\nperson7@example.test\nEdit\n8\nNew\nPerson\nfemale\n35\n00123\nperson8@example.test\nEdit\n9\nNew\nPerson\nfemale\n35\n00123\nperson9@example.test\nEdit\n10\nNew\nPerson\nfemale\n35\n00123\nperson10@example.test\nEdit\n11\nNew\nPerson\nfemale\n35\n00123\nperson11@example.test\nEdit\n12\nNew\nPerson\nfemale\n35\n00123\nperson12@example.test\nEdit\n13\nNew\nPerson\nfemale\n35\n00123\nperson13@example.test\nEdit\n14\nNew\nPerson\nfemale\n35\n00123\nperson14@example.test\nEdit\n15\nNew\nPerson\nfemale\n35\n00123\nperson15@example.test\nEdit\nLoading...", - "ok": true, - "expected_task_pass": false - }, - { - "id": "rsuite-complete-record-missing-email", - "category": "acceptance-contract", - "root_cause": "volatile_demo_people_not_frozen_body", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 16, - "table_columns": [ - { - "name": "id", - "label": "Id", - "type": "integer", - "minimum": 1 - }, - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "gender", - "label": "Gender", - "type": "text", - "pattern": "(?i)female|male" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "maximum": 150 - }, - { - "name": "postcode", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "root_selector": ".rs-table", - "root_index": 0, - "row_selector": "[role=row]:not(.rs-table-row-header)", - "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", - "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", - "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", - "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", - "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" - } - }, - "body_requirements": {} - }, - "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n1\nNew\nPerson\nfemale\n35\n00123\nperson1@example.test\nEdit\n2\nNew\nPerson\nfemale\n35\n00123\nperson2@example.test\nEdit\n3\nNew\nPerson\nfemale\n35\n00123\nperson3@example.test\nEdit\n4\nNew\nPerson\nfemale\n35\n00123\nperson4@example.test\nEdit\n5\nNew\nPerson\nfemale\n35\n00123\nperson5@example.test\nEdit\n6\nNew\nPerson\nfemale\n35\n00123\nperson6@example.test\nEdit\n7\nNew\nPerson\nfemale\n35\n00123\nperson7@example.test\nEdit\n8\nNew\nPerson\nfemale\n35\n00123\nperson8@example.test\nEdit\n9\nNew\nPerson\nfemale\n35\n00123\nperson9@example.test\nEdit\n10\nNew\nPerson\nfemale\n35\n00123\nperson10@example.test\nEdit\n11\nNew\nPerson\nfemale\n35\n00123\nperson11@example.test\nEdit\n12\nNew\nPerson\nfemale\n35\n00123\nperson12@example.test\nEdit\n13\nNew\nPerson\nfemale\n35\n00123\nperson13@example.test\nEdit\n14\nNew\nPerson\nfemale\n35\n00123\nperson14@example.test\nEdit\n15\nNew\nPerson\nfemale\n35\n00123\nperson15@example.test\nEdit\n16\nNew\nPerson\nfemale\n35\n00123\nEdit\nLoading...", - "ok": true, - "expected_task_pass": false - }, - { - "id": "rsuite-complete-record-duplicate", - "category": "acceptance-contract", - "root_cause": "volatile_demo_people_not_frozen_body", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 16, - "table_columns": [ - { - "name": "id", - "label": "Id", - "type": "integer", - "minimum": 1 - }, - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "gender", - "label": "Gender", - "type": "text", - "pattern": "(?i)female|male" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "maximum": 150 - }, - { - "name": "postcode", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "root_selector": ".rs-table", - "root_index": 0, - "row_selector": "[role=row]:not(.rs-table-row-header)", - "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", - "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", - "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", - "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", - "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" - } + "mutable_body_policy": { + "stable_annotation": { + "content_kind": "changing_public_body", + "structural_only_fields": [ + "mutable_full_text_snapshot" + ] + } + } + }, + "artifact": "Events, Past Day\n52 Events.\nNewest First\nSettings\nNew event at 11:00", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_task_pass": true, + "body_coverage_pass": true + } + }, + { + "id": "volatile-body-stable-text-still-rejects-missing-page-identity", + "category": "acceptance-contract", + "root_cause": "volatile_body_snapshot_used_as_invariant_answer", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/live-events", + "task_kind": "body", + "ground_truth_main_content": "40 Events. Old event at 10:00", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Events, Past Day" + ] }, - "body_requirements": {} - }, - "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n1\nNew\nPerson\nfemale\n35\n00123\nperson1@example.test\nEdit\n2\nNew\nPerson\nfemale\n35\n00123\nperson2@example.test\nEdit\n3\nNew\nPerson\nfemale\n35\n00123\nperson3@example.test\nEdit\n4\nNew\nPerson\nfemale\n35\n00123\nperson4@example.test\nEdit\n5\nNew\nPerson\nfemale\n35\n00123\nperson5@example.test\nEdit\n6\nNew\nPerson\nfemale\n35\n00123\nperson6@example.test\nEdit\n7\nNew\nPerson\nfemale\n35\n00123\nperson7@example.test\nEdit\n8\nNew\nPerson\nfemale\n35\n00123\nperson8@example.test\nEdit\n9\nNew\nPerson\nfemale\n35\n00123\nperson9@example.test\nEdit\n10\nNew\nPerson\nfemale\n35\n00123\nperson10@example.test\nEdit\n11\nNew\nPerson\nfemale\n35\n00123\nperson11@example.test\nEdit\n12\nNew\nPerson\nfemale\n35\n00123\nperson12@example.test\nEdit\n13\nNew\nPerson\nfemale\n35\n00123\nperson13@example.test\nEdit\n14\nNew\nPerson\nfemale\n35\n00123\nperson14@example.test\nEdit\n15\nNew\nPerson\nfemale\n35\n00123\nperson15@example.test\nEdit\n1\nNew\nPerson\nfemale\n35\n00123\nperson1@example.test\nEdit\nLoading...", - "ok": true, - "expected_task_pass": false - }, - { - "id": "rsuite-complete-record-invalid-postcode", - "category": "acceptance-contract", - "root_cause": "volatile_demo_people_not_frozen_body", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 16, - "table_columns": [ - { - "name": "id", - "label": "Id", - "type": "integer", - "minimum": 1 - }, - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "gender", - "label": "Gender", - "type": "text", - "pattern": "(?i)female|male" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "maximum": 150 - }, - { - "name": "postcode", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "root_selector": ".rs-table", - "root_index": 0, - "row_selector": "[role=row]:not(.rs-table-row-header)", - "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", - "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", - "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", - "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", - "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" - } + { + "id": "sort", + "alternatives": [ + "Newest First" + ] }, - "body_requirements": {} + { + "id": "settings", + "alternatives": [ + "Settings" + ] + } + ] }, - "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n1\nNew\nPerson\nfemale\n35\n00123\nperson1@example.test\nEdit\n2\nNew\nPerson\nfemale\n35\n00123\nperson2@example.test\nEdit\n3\nNew\nPerson\nfemale\n35\n00123\nperson3@example.test\nEdit\n4\nNew\nPerson\nfemale\n35\n00123\nperson4@example.test\nEdit\n5\nNew\nPerson\nfemale\n35\n00123\nperson5@example.test\nEdit\n6\nNew\nPerson\nfemale\n35\n00123\nperson6@example.test\nEdit\n7\nNew\nPerson\nfemale\n35\n00123\nperson7@example.test\nEdit\n8\nNew\nPerson\nfemale\n35\n00123\nperson8@example.test\nEdit\n9\nNew\nPerson\nfemale\n35\n00123\nperson9@example.test\nEdit\n10\nNew\nPerson\nfemale\n35\n00123\nperson10@example.test\nEdit\n11\nNew\nPerson\nfemale\n35\n00123\nperson11@example.test\nEdit\n12\nNew\nPerson\nfemale\n35\n00123\nperson12@example.test\nEdit\n13\nNew\nPerson\nfemale\n35\n00123\nperson13@example.test\nEdit\n14\nNew\nPerson\nfemale\n35\n00123\nperson14@example.test\nEdit\n15\nNew\nPerson\nfemale\n35\n00123\nperson15@example.test\nEdit\n16\nNew\nPerson\nfemale\n35\nbad\nperson16@example.test\nEdit\nLoading...", - "ok": true, - "expected_task_pass": false - }, - { - "id": "rsuite-complete-record-markdown", - "category": "acceptance-contract", - "root_cause": "volatile_demo_people_not_frozen_body", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 16, - "table_columns": [ - { - "name": "id", - "label": "Id", - "type": "integer", - "minimum": 1 - }, - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "gender", - "label": "Gender", - "type": "text", - "pattern": "(?i)female|male" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "maximum": 150 - }, - { - "name": "postcode", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "root_selector": ".rs-table", - "root_index": 0, - "row_selector": "[role=row]:not(.rs-table-row-header)", - "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", - "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", - "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", - "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", - "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" - } + "mutable_body_policy": { + "stable_annotation": { + "content_kind": "changing_public_body", + "structural_only_fields": [ + "mutable_full_text_snapshot" + ] + } + } + }, + "artifact": "52 Events.\nNewest First\nSettings\nNew event at 11:00", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_task_pass": false, + "stable_anchor_hits": 2 + } + }, + { + "id": "normal-theme-control-is-not-a-block-page-marker", + "category": "acceptance-contract", + "root_cause": "normal_page_control_misclassified_as_forbidden_content", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/docs", + "task_kind": "body", + "forbidden_any": [ + "CAPTCHA", + "Access denied" + ], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "identity", + "alternatives": [ + "Documentation" + ] }, - "body_requirements": {} - }, - "artifact": "| Id | First Name | Last Name | Gender | Age | Postcode | Email |\n| --- | --- | --- | --- | --- | --- | --- |\n| 1 | New | Person | female | 35 | 00123 | person1@example.test |\n| 2 | New | Person | female | 35 | 00123 | person2@example.test |\n| 3 | New | Person | female | 35 | 00123 | person3@example.test |\n| 4 | New | Person | female | 35 | 00123 | person4@example.test |\n| 5 | New | Person | female | 35 | 00123 | person5@example.test |\n| 6 | New | Person | female | 35 | 00123 | person6@example.test |\n| 7 | New | Person | female | 35 | 00123 | person7@example.test |\n| 8 | New | Person | female | 35 | 00123 | person8@example.test |\n| 9 | New | Person | female | 35 | 00123 | person9@example.test |\n| 10 | New | Person | female | 35 | 00123 | person10@example.test |\n| 11 | New | Person | female | 35 | 00123 | person11@example.test |\n| 12 | New | Person | female | 35 | 00123 | person12@example.test |\n| 13 | New | Person | female | 35 | 00123 | person13@example.test |\n| 14 | New | Person | female | 35 | 00123 | person14@example.test |\n| 15 | New | Person | female | 35 | 00123 | person15@example.test |\n| 16 | New | Person | female | 35 | 00123 | person16@example.test |", - "ok": true, - "expected_task_pass": true - }, - { - "id": "rsuite-complete-record-only-headers", - "category": "acceptance-contract", - "root_cause": "volatile_demo_people_not_frozen_body", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 16, - "table_columns": [ - { - "name": "id", - "label": "Id", - "type": "integer", - "minimum": 1 - }, - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "gender", - "label": "Gender", - "type": "text", - "pattern": "(?i)female|male" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "maximum": 150 - }, - { - "name": "postcode", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "root_selector": ".rs-table", - "root_index": 0, - "row_selector": "[role=row]:not(.rs-table-row-header)", - "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", - "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", - "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", - "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", - "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" - } + { + "id": "install", + "alternatives": [ + "Install" + ] }, - "body_requirements": {} - }, - "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-ordinal-date-real-date", - "category": "structured-extraction", - "root_cause": "calendar_validity_for_table_dates", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 1, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - } - ], - "record_identity_fields": [ - "name" - ], - "table_scope": { - "row_selector": "tbody tr", - "cell_selector": "td", - "start_pattern": "People\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" - } - } - }, - "artifact": "People\nNew Person 23rd Feb 26", - "ok": true, - "expected_task_pass": true - }, - { - "id": "table-ordinal-date-leap-date", - "category": "structured-extraction", - "root_cause": "calendar_validity_for_table_dates", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 1, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - } - ], - "record_identity_fields": [ - "name" - ], - "table_scope": { - "row_selector": "tbody tr", - "cell_selector": "td", - "start_pattern": "People\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" - } - } - }, - "artifact": "People\nNew Person 29th Feb 2024", - "ok": true, - "expected_task_pass": true - }, - { - "id": "table-ordinal-date-impossible-day", - "category": "structured-extraction", - "root_cause": "calendar_validity_for_table_dates", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 1, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - } - ], - "record_identity_fields": [ - "name" - ], - "table_scope": { - "row_selector": "tbody tr", - "cell_selector": "td", - "start_pattern": "People\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" - } - } - }, - "artifact": "People\nNew Person 31st Feb 26", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-ordinal-date-non-leap-date", - "category": "structured-extraction", - "root_cause": "calendar_validity_for_table_dates", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 1, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - } - ], - "record_identity_fields": [ - "name" - ], - "table_scope": { - "row_selector": "tbody tr", - "cell_selector": "td", - "start_pattern": "People\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" - } + { + "id": "import", + "alternatives": [ + "Import" + ] } - }, - "artifact": "People\nNew Person 29th Feb 2025", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-ordinal-date-wrong-ordinal", - "category": "structured-extraction", - "root_cause": "calendar_validity_for_table_dates", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 1, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - } - ], - "record_identity_fields": [ - "name" - ], - "table_scope": { - "row_selector": "tbody tr", - "cell_selector": "td", - "start_pattern": "People\n", - "end_pattern": "Next example", - "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" - } - } - }, - "artifact": "People\nNew Person 11st Jan 26", - "ok": true, - "expected_task_pass": false - }, - { - "id": "eui-complete-record-eight-complete", - "category": "acceptance-contract", - "root_cause": "volatile_eui_people_and_invisible_reference", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "github", - "label": "Github", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - }, - { - "name": "online", - "label": "Online", - "type": "text", - "pattern": "(?i)Online|Offline" - } - ], - "record_identity_fields": [ - "first_name", - "last_name", - "github" - ], - "table_scope": { - "root_selector": "table.euiTable", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > th, :scope > td", - "exclude_selector": "[class*=euiScreenReaderOnly],svg", - "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", - "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", - "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", - "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" - } - }, - "body_requirements": {} - }, - "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nHelen\nExample\nHelen_account\n23rd Feb 26\nOnline\nShow source", - "ok": true, - "expected_task_pass": true - }, - { - "id": "eui-complete-record-seven-below80", - "category": "acceptance-contract", - "root_cause": "volatile_eui_people_and_invisible_reference", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "github", - "label": "Github", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - }, - { - "name": "online", - "label": "Online", - "type": "text", - "pattern": "(?i)Online|Offline" - } - ], - "record_identity_fields": [ - "first_name", - "last_name", - "github" - ], - "table_scope": { - "root_selector": "table.euiTable", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > th, :scope > td", - "exclude_selector": "[class*=euiScreenReaderOnly],svg", - "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", - "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", - "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", - "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" - } - }, - "body_requirements": {} - }, - "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nShow source", - "ok": true, - "expected_task_pass": false - }, - { - "id": "eui-complete-record-impossible-date", - "category": "acceptance-contract", - "root_cause": "volatile_eui_people_and_invisible_reference", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "github", - "label": "Github", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - }, - { - "name": "online", - "label": "Online", - "type": "text", - "pattern": "(?i)Online|Offline" - } - ], - "record_identity_fields": [ - "first_name", - "last_name", - "github" - ], - "table_scope": { - "root_selector": "table.euiTable", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > th, :scope > td", - "exclude_selector": "[class*=euiScreenReaderOnly],svg", - "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", - "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", - "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", - "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" - } - }, - "body_requirements": {} - }, - "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nHelen\nExample\nHelen_account\n31st Feb 26\nOnline\nShow source", - "ok": true, - "expected_task_pass": false - }, - { - "id": "eui-complete-record-missing-status", - "category": "acceptance-contract", - "root_cause": "volatile_eui_people_and_invisible_reference", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "github", - "label": "Github", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - }, - { - "name": "online", - "label": "Online", - "type": "text", - "pattern": "(?i)Online|Offline" - } - ], - "record_identity_fields": [ - "first_name", - "last_name", - "github" - ], - "table_scope": { - "root_selector": "table.euiTable", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > th, :scope > td", - "exclude_selector": "[class*=euiScreenReaderOnly],svg", - "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", - "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", - "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", - "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" - } - }, - "body_requirements": {} - }, - "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nHelen\nExample\nHelen_account\n23rd Feb 26\nShow source", - "ok": true, - "expected_task_pass": false - }, - { - "id": "eui-complete-record-duplicates", - "category": "acceptance-contract", - "root_cause": "volatile_eui_people_and_invisible_reference", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "github", - "label": "Github", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - }, - { - "name": "online", - "label": "Online", - "type": "text", - "pattern": "(?i)Online|Offline" - } - ], - "record_identity_fields": [ - "first_name", - "last_name", - "github" - ], - "table_scope": { - "root_selector": "table.euiTable", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > th, :scope > td", - "exclude_selector": "[class*=euiScreenReaderOnly],svg", - "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", - "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", - "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", - "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" - } - }, - "body_requirements": {} - }, - "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nShow source", - "ok": true, - "expected_task_pass": false - }, - { - "id": "eui-complete-record-cannot-borrow-neighbor", - "category": "acceptance-contract", - "root_cause": "volatile_eui_people_and_invisible_reference", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "github", - "label": "Github", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - }, - { - "name": "online", - "label": "Online", - "type": "text", - "pattern": "(?i)Online|Offline" - } - ], - "record_identity_fields": [ - "first_name", - "last_name", - "github" - ], - "table_scope": { - "root_selector": "table.euiTable", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > th, :scope > td", - "exclude_selector": "[class*=euiScreenReaderOnly],svg", - "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", - "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", - "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", - "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" - } - }, - "body_requirements": {} - }, - "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\n23rd Feb 26\nOnline\nspare_account\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nHelen\nExample\nHelen_account\n23rd Feb 26\nOnline\nShow source", - "ok": true, - "expected_task_pass": false - }, - { - "id": "eui-complete-record-markdown", - "category": "acceptance-contract", - "root_cause": "volatile_eui_people_and_invisible_reference", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "github", - "label": "Github", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - }, - { - "name": "online", - "label": "Online", - "type": "text", - "pattern": "(?i)Online|Offline" - } - ], - "record_identity_fields": [ - "first_name", - "last_name", - "github" - ], - "table_scope": { - "root_selector": "table.euiTable", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > th, :scope > td", - "exclude_selector": "[class*=euiScreenReaderOnly],svg", - "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", - "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", - "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", - "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" - } - }, - "body_requirements": {} - }, - "artifact": "| First Name | Last Name | Github | Date of Birth | Online |\n| --- | --- | --- | --- | --- |\n| Ada | Example | Ada_account | 23rd Feb 26 | Online |\n| Bob | Example | Bob_account | 23rd Feb 26 | Online |\n| Carol | Example | Carol_account | 23rd Feb 26 | Online |\n| Dana | Example | Dana_account | 23rd Feb 26 | Online |\n| Emma | Example | Emma_account | 23rd Feb 26 | Online |\n| Frank | Example | Frank_account | 23rd Feb 26 | Online |\n| Grace | Example | Grace_account | 23rd Feb 26 | Online |\n| Helen | Example | Helen_account | 23rd Feb 26 | Online |", - "ok": true, - "expected_task_pass": true - }, - { - "id": "eui-complete-record-html", - "category": "acceptance-contract", - "root_cause": "volatile_eui_people_and_invisible_reference", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "github", - "label": "Github", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - }, - { - "name": "online", - "label": "Online", - "type": "text", - "pattern": "(?i)Online|Offline" - } - ], - "record_identity_fields": [ - "first_name", - "last_name", - "github" - ], - "table_scope": { - "root_selector": "table.euiTable", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > th, :scope > td", - "exclude_selector": "[class*=euiScreenReaderOnly],svg", - "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", - "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", - "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", - "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" - } + ] + } + }, + "artifact": "Documentation\nDark theme\nInstall\nImport", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_task_pass": true, + "forbidden_requirement_hits": [] + } + }, + { + "id": "removing-normal-theme-control-does-not-hide-real-block-page", + "category": "acceptance-contract", + "root_cause": "normal_page_control_misclassified_as_forbidden_content", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/docs", + "task_kind": "body", + "forbidden_any": [ + "CAPTCHA", + "Access denied" + ], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "identity", + "alternatives": [ + "Documentation" + ] }, - "body_requirements": {} - }, - "artifact": "<table class=\"euiTable\"><tbody><tr><td>Ada</td><td>Example</td><td>Ada_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Bob</td><td>Example</td><td>Bob_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Carol</td><td>Example</td><td>Carol_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Dana</td><td>Example</td><td>Dana_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Emma</td><td>Example</td><td>Emma_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Frank</td><td>Example</td><td>Frank_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Grace</td><td>Example</td><td>Grace_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Helen</td><td>Example</td><td>Helen_account</td><td>23rd Feb 26</td><td>Online</td></tr></tbody></table>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "eui-complete-record-later-table", - "category": "acceptance-contract", - "root_cause": "volatile_eui_people_and_invisible_reference", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/people", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "github", - "label": "Github", - "type": "text" - }, - { - "name": "birth", - "label": "Date of Birth", - "type": "date_ordinal_en" - }, - { - "name": "online", - "label": "Online", - "type": "text", - "pattern": "(?i)Online|Offline" - } - ], - "record_identity_fields": [ - "first_name", - "last_name", - "github" - ], - "table_scope": { - "root_selector": "table.euiTable", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > th, :scope > td", - "exclude_selector": "[class*=euiScreenReaderOnly],svg", - "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", - "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", - "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", - "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" - } + { + "id": "install", + "alternatives": [ + "Install" + ] }, - "body_requirements": {} - }, - "artifact": "<table class=\"euiTable\"><tbody><tr><td>Ada</td><td>Example</td><td>Ada_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Bob</td><td>Example</td><td>Bob_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Carol</td><td>Example</td><td>Carol_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Dana</td><td>Example</td><td>Dana_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Emma</td><td>Example</td><td>Emma_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Frank</td><td>Example</td><td>Frank_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Grace</td><td>Example</td><td>Grace_account</td><td>23rd Feb 26</td><td>Online</td></tr></tbody></table><table class=\"euiTable\"><tbody><tr><td>Helen</td><td>Example</td><td>Helen_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Iris</td><td>Example</td><td>Iris_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>James</td><td>Example</td><td>James_account</td><td>23rd Feb 26</td><td>Online</td></tr></tbody></table>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "fixed-demo-product-complete", - "category": "acceptance-contract", - "root_cause": "fixed_demo_product_field_correspondence", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/fixed-products", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "product", - "label": "Product", - "type": "text" - }, - { - "name": "category", - "label": "Category", - "type": "text" - }, - { - "name": "price", - "label": "Price", - "type": "currency_usd" - }, - { - "name": "status", - "label": "Status", - "type": "text" - } - ], - "record_identity_fields": [ - "product" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td:not(:nth-child(4))", - "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", - "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", - "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "product": "Bamboo Watch f230fh0g3", - "category": "Accessories", - "price": "$65", - "status": "In Stock" - }, - { - "product": "Black Watch nvklal433", - "category": "Accessories", - "price": "$72", - "status": "In Stock" - }, - { - "product": "Blue Band zz21cz3c1", - "category": "Fitness", - "price": "$79", - "status": "Low Stock" - }, - { - "product": "Blue T-Shirt 244wgerg2", - "category": "Clothing", - "price": "$29", - "status": "In Stock" - }, - { - "product": "Bracelet h456wer53", - "category": "Accessories", - "price": "$15", - "status": "In Stock" - }, - { - "product": "Brown Purse av2231fwg", - "category": "Accessories", - "price": "$120", - "status": "Out of Stock" - }, - { - "product": "Chakra Bracelet bib36pfvm", - "category": "Accessories", - "price": "$32", - "status": "Low Stock" - }, - { - "product": "Galaxy Earrings mbvjkgip5", - "category": "Accessories", - "price": "$34", - "status": "In Stock" - } - ], - "ignored_column_labels": [ - "Rating" - ] - } - } - }, - "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 Out of Stock\nChakra Bracelet bib36pfvm Accessories $32 Low Stock\nGalaxy Earrings mbvjkgip5 Accessories $34 In Stock\nPREVIEW", - "ok": true, - "expected_task_pass": true - }, - { - "id": "fixed-demo-product-seven", - "category": "acceptance-contract", - "root_cause": "fixed_demo_product_field_correspondence", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/fixed-products", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "product", - "label": "Product", - "type": "text" - }, - { - "name": "category", - "label": "Category", - "type": "text" - }, - { - "name": "price", - "label": "Price", - "type": "currency_usd" - }, - { - "name": "status", - "label": "Status", - "type": "text" - } - ], - "record_identity_fields": [ - "product" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td:not(:nth-child(4))", - "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", - "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", - "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "product": "Bamboo Watch f230fh0g3", - "category": "Accessories", - "price": "$65", - "status": "In Stock" - }, - { - "product": "Black Watch nvklal433", - "category": "Accessories", - "price": "$72", - "status": "In Stock" - }, - { - "product": "Blue Band zz21cz3c1", - "category": "Fitness", - "price": "$79", - "status": "Low Stock" - }, - { - "product": "Blue T-Shirt 244wgerg2", - "category": "Clothing", - "price": "$29", - "status": "In Stock" - }, - { - "product": "Bracelet h456wer53", - "category": "Accessories", - "price": "$15", - "status": "In Stock" - }, - { - "product": "Brown Purse av2231fwg", - "category": "Accessories", - "price": "$120", - "status": "Out of Stock" - }, - { - "product": "Chakra Bracelet bib36pfvm", - "category": "Accessories", - "price": "$32", - "status": "Low Stock" - }, - { - "product": "Galaxy Earrings mbvjkgip5", - "category": "Accessories", - "price": "$34", - "status": "In Stock" - } - ], - "ignored_column_labels": [ - "Rating" - ] - } - } - }, - "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 Out of Stock\nChakra Bracelet bib36pfvm Accessories $32 Low Stock\nPREVIEW", - "ok": true, - "expected_task_pass": true - }, - { - "id": "fixed-demo-product-six", - "category": "acceptance-contract", - "root_cause": "fixed_demo_product_field_correspondence", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/fixed-products", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "product", - "label": "Product", - "type": "text" - }, - { - "name": "category", - "label": "Category", - "type": "text" - }, - { - "name": "price", - "label": "Price", - "type": "currency_usd" - }, - { - "name": "status", - "label": "Status", - "type": "text" - } - ], - "record_identity_fields": [ - "product" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td:not(:nth-child(4))", - "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", - "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", - "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "product": "Bamboo Watch f230fh0g3", - "category": "Accessories", - "price": "$65", - "status": "In Stock" - }, - { - "product": "Black Watch nvklal433", - "category": "Accessories", - "price": "$72", - "status": "In Stock" - }, - { - "product": "Blue Band zz21cz3c1", - "category": "Fitness", - "price": "$79", - "status": "Low Stock" - }, - { - "product": "Blue T-Shirt 244wgerg2", - "category": "Clothing", - "price": "$29", - "status": "In Stock" - }, - { - "product": "Bracelet h456wer53", - "category": "Accessories", - "price": "$15", - "status": "In Stock" - }, - { - "product": "Brown Purse av2231fwg", - "category": "Accessories", - "price": "$120", - "status": "Out of Stock" - }, - { - "product": "Chakra Bracelet bib36pfvm", - "category": "Accessories", - "price": "$32", - "status": "Low Stock" - }, - { - "product": "Galaxy Earrings mbvjkgip5", - "category": "Accessories", - "price": "$34", - "status": "In Stock" - } - ], - "ignored_column_labels": [ - "Rating" - ] - } - } - }, - "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 Out of Stock\nPREVIEW", - "ok": true, - "expected_task_pass": false - }, - { - "id": "fixed-demo-product-missing-prices", - "category": "acceptance-contract", - "root_cause": "fixed_demo_product_field_correspondence", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/fixed-products", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "product", - "label": "Product", - "type": "text" - }, - { - "name": "category", - "label": "Category", - "type": "text" - }, - { - "name": "price", - "label": "Price", - "type": "currency_usd" - }, - { - "name": "status", - "label": "Status", - "type": "text" - } - ], - "record_identity_fields": [ - "product" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td:not(:nth-child(4))", - "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", - "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", - "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "product": "Bamboo Watch f230fh0g3", - "category": "Accessories", - "price": "$65", - "status": "In Stock" - }, - { - "product": "Black Watch nvklal433", - "category": "Accessories", - "price": "$72", - "status": "In Stock" - }, - { - "product": "Blue Band zz21cz3c1", - "category": "Fitness", - "price": "$79", - "status": "Low Stock" - }, - { - "product": "Blue T-Shirt 244wgerg2", - "category": "Clothing", - "price": "$29", - "status": "In Stock" - }, - { - "product": "Bracelet h456wer53", - "category": "Accessories", - "price": "$15", - "status": "In Stock" - }, - { - "product": "Brown Purse av2231fwg", - "category": "Accessories", - "price": "$120", - "status": "Out of Stock" - }, - { - "product": "Chakra Bracelet bib36pfvm", - "category": "Accessories", - "price": "$32", - "status": "Low Stock" - }, - { - "product": "Galaxy Earrings mbvjkgip5", - "category": "Accessories", - "price": "$34", - "status": "In Stock" - } - ], - "ignored_column_labels": [ - "Rating" - ] - } - } - }, - "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories In Stock\nBlack Watch nvklal433 Accessories In Stock\nBlue Band zz21cz3c1 Fitness Low Stock\nBlue T-Shirt 244wgerg2 Clothing In Stock\nBracelet h456wer53 Accessories In Stock\nBrown Purse av2231fwg Accessories Out of Stock\nChakra Bracelet bib36pfvm Accessories Low Stock\nGalaxy Earrings mbvjkgip5 Accessories In Stock\nPREVIEW", - "ok": true, - "expected_task_pass": false - }, - { - "id": "fixed-demo-product-wrong-prices", - "category": "acceptance-contract", - "root_cause": "fixed_demo_product_field_correspondence", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/fixed-products", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "product", - "label": "Product", - "type": "text" - }, - { - "name": "category", - "label": "Category", - "type": "text" - }, - { - "name": "price", - "label": "Price", - "type": "currency_usd" - }, - { - "name": "status", - "label": "Status", - "type": "text" - } - ], - "record_identity_fields": [ - "product" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td:not(:nth-child(4))", - "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", - "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", - "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "product": "Bamboo Watch f230fh0g3", - "category": "Accessories", - "price": "$65", - "status": "In Stock" - }, - { - "product": "Black Watch nvklal433", - "category": "Accessories", - "price": "$72", - "status": "In Stock" - }, - { - "product": "Blue Band zz21cz3c1", - "category": "Fitness", - "price": "$79", - "status": "Low Stock" - }, - { - "product": "Blue T-Shirt 244wgerg2", - "category": "Clothing", - "price": "$29", - "status": "In Stock" - }, - { - "product": "Bracelet h456wer53", - "category": "Accessories", - "price": "$15", - "status": "In Stock" - }, - { - "product": "Brown Purse av2231fwg", - "category": "Accessories", - "price": "$120", - "status": "Out of Stock" - }, - { - "product": "Chakra Bracelet bib36pfvm", - "category": "Accessories", - "price": "$32", - "status": "Low Stock" - }, - { - "product": "Galaxy Earrings mbvjkgip5", - "category": "Accessories", - "price": "$34", - "status": "In Stock" - } - ], - "ignored_column_labels": [ - "Rating" - ] - } - } - }, - "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $99999 In Stock\nBlack Watch nvklal433 Accessories $99999 In Stock\nBlue Band zz21cz3c1 Fitness $99999 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $99999 In Stock\nBracelet h456wer53 Accessories $99999 In Stock\nBrown Purse av2231fwg Accessories $99999 Out of Stock\nChakra Bracelet bib36pfvm Accessories $99999 Low Stock\nGalaxy Earrings mbvjkgip5 Accessories $99999 In Stock\nPREVIEW", - "ok": true, - "expected_task_pass": false - }, - { - "id": "fixed-demo-product-wrong-statuses", - "category": "acceptance-contract", - "root_cause": "fixed_demo_product_field_correspondence", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/fixed-products", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "product", - "label": "Product", - "type": "text" - }, - { - "name": "category", - "label": "Category", - "type": "text" - }, - { - "name": "price", - "label": "Price", - "type": "currency_usd" - }, - { - "name": "status", - "label": "Status", - "type": "text" - } - ], - "record_identity_fields": [ - "product" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td:not(:nth-child(4))", - "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", - "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", - "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "product": "Bamboo Watch f230fh0g3", - "category": "Accessories", - "price": "$65", - "status": "In Stock" - }, - { - "product": "Black Watch nvklal433", - "category": "Accessories", - "price": "$72", - "status": "In Stock" - }, - { - "product": "Blue Band zz21cz3c1", - "category": "Fitness", - "price": "$79", - "status": "Low Stock" - }, - { - "product": "Blue T-Shirt 244wgerg2", - "category": "Clothing", - "price": "$29", - "status": "In Stock" - }, - { - "product": "Bracelet h456wer53", - "category": "Accessories", - "price": "$15", - "status": "In Stock" - }, - { - "product": "Brown Purse av2231fwg", - "category": "Accessories", - "price": "$120", - "status": "Out of Stock" - }, - { - "product": "Chakra Bracelet bib36pfvm", - "category": "Accessories", - "price": "$32", - "status": "Low Stock" - }, - { - "product": "Galaxy Earrings mbvjkgip5", - "category": "Accessories", - "price": "$34", - "status": "In Stock" - } - ], - "ignored_column_labels": [ - "Rating" - ] - } - } - }, - "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 In Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 In Stock\nChakra Bracelet bib36pfvm Accessories $32 In Stock\nGalaxy Earrings mbvjkgip5 Accessories $34 In Stock\nPREVIEW", - "ok": true, - "expected_task_pass": false - }, - { - "id": "fixed-demo-product-equivalent-currency", - "category": "acceptance-contract", - "root_cause": "fixed_demo_product_field_correspondence", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/fixed-products", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "product", - "label": "Product", - "type": "text" - }, - { - "name": "category", - "label": "Category", - "type": "text" - }, - { - "name": "price", - "label": "Price", - "type": "currency_usd" - }, - { - "name": "status", - "label": "Status", - "type": "text" - } - ], - "record_identity_fields": [ - "product" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td:not(:nth-child(4))", - "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", - "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", - "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "product": "Bamboo Watch f230fh0g3", - "category": "Accessories", - "price": "$65", - "status": "In Stock" - }, - { - "product": "Black Watch nvklal433", - "category": "Accessories", - "price": "$72", - "status": "In Stock" - }, - { - "product": "Blue Band zz21cz3c1", - "category": "Fitness", - "price": "$79", - "status": "Low Stock" - }, - { - "product": "Blue T-Shirt 244wgerg2", - "category": "Clothing", - "price": "$29", - "status": "In Stock" - }, - { - "product": "Bracelet h456wer53", - "category": "Accessories", - "price": "$15", - "status": "In Stock" - }, - { - "product": "Brown Purse av2231fwg", - "category": "Accessories", - "price": "$120", - "status": "Out of Stock" - }, - { - "product": "Chakra Bracelet bib36pfvm", - "category": "Accessories", - "price": "$32", - "status": "Low Stock" - }, - { - "product": "Galaxy Earrings mbvjkgip5", - "category": "Accessories", - "price": "$34", - "status": "In Stock" - } - ], - "ignored_column_labels": [ - "Rating" - ] - } - } - }, - "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65.00 In Stock\nBlack Watch nvklal433 Accessories $72.00 In Stock\nBlue Band zz21cz3c1 Fitness $79.00 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29.00 In Stock\nBracelet h456wer53 Accessories $15.00 In Stock\nBrown Purse av2231fwg Accessories $120.00 Out of Stock\nChakra Bracelet bib36pfvm Accessories $32.00 Low Stock\nGalaxy Earrings mbvjkgip5 Accessories $34.00 In Stock\nPREVIEW", - "ok": true, - "expected_task_pass": true - }, - { - "id": "fixed-demo-product-markdown-stars", - "category": "acceptance-contract", - "root_cause": "fixed_demo_product_field_correspondence", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/fixed-products", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "product", - "label": "Product", - "type": "text" - }, - { - "name": "category", - "label": "Category", - "type": "text" - }, - { - "name": "price", - "label": "Price", - "type": "currency_usd" - }, - { - "name": "status", - "label": "Status", - "type": "text" - } - ], - "record_identity_fields": [ - "product" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td:not(:nth-child(4))", - "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", - "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", - "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "product": "Bamboo Watch f230fh0g3", - "category": "Accessories", - "price": "$65", - "status": "In Stock" - }, - { - "product": "Black Watch nvklal433", - "category": "Accessories", - "price": "$72", - "status": "In Stock" - }, - { - "product": "Blue Band zz21cz3c1", - "category": "Fitness", - "price": "$79", - "status": "Low Stock" - }, - { - "product": "Blue T-Shirt 244wgerg2", - "category": "Clothing", - "price": "$29", - "status": "In Stock" - }, - { - "product": "Bracelet h456wer53", - "category": "Accessories", - "price": "$15", - "status": "In Stock" - }, - { - "product": "Brown Purse av2231fwg", - "category": "Accessories", - "price": "$120", - "status": "Out of Stock" - }, - { - "product": "Chakra Bracelet bib36pfvm", - "category": "Accessories", - "price": "$32", - "status": "Low Stock" - }, - { - "product": "Galaxy Earrings mbvjkgip5", - "category": "Accessories", - "price": "$34", - "status": "In Stock" - } - ], - "ignored_column_labels": [ - "Rating" - ] - } - } - }, - "artifact": "| Product | Category | Price | Rating | Status |\n| --- | --- | --- | --- | --- |\n| Bamboo Watch f230fh0g3 | Accessories | $65 | ★★★ | In Stock |\n| Black Watch nvklal433 | Accessories | $72 | ★★★ | In Stock |\n| Blue Band zz21cz3c1 | Fitness | $79 | ★★★ | Low Stock |\n| Blue T-Shirt 244wgerg2 | Clothing | $29 | ★★★ | In Stock |\n| Bracelet h456wer53 | Accessories | $15 | ★★★ | In Stock |\n| Brown Purse av2231fwg | Accessories | $120 | ★★★ | Out of Stock |\n| Chakra Bracelet bib36pfvm | Accessories | $32 | ★★★ | Low Stock |\n| Galaxy Earrings mbvjkgip5 | Accessories | $34 | ★★★ | In Stock |", - "ok": true, - "expected_task_pass": true - }, - { - "id": "fixed-demo-product-duplicates", - "category": "acceptance-contract", - "root_cause": "fixed_demo_product_field_correspondence", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/fixed-products", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 7, - "table_columns": [ - { - "name": "product", - "label": "Product", - "type": "text" - }, - { - "name": "category", - "label": "Category", - "type": "text" - }, - { - "name": "price", - "label": "Price", - "type": "currency_usd" - }, - { - "name": "status", - "label": "Status", - "type": "text" - } - ], - "record_identity_fields": [ - "product" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td:not(:nth-child(4))", - "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", - "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", - "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "product": "Bamboo Watch f230fh0g3", - "category": "Accessories", - "price": "$65", - "status": "In Stock" - }, - { - "product": "Black Watch nvklal433", - "category": "Accessories", - "price": "$72", - "status": "In Stock" - }, - { - "product": "Blue Band zz21cz3c1", - "category": "Fitness", - "price": "$79", - "status": "Low Stock" - }, - { - "product": "Blue T-Shirt 244wgerg2", - "category": "Clothing", - "price": "$29", - "status": "In Stock" - }, - { - "product": "Bracelet h456wer53", - "category": "Accessories", - "price": "$15", - "status": "In Stock" - }, - { - "product": "Brown Purse av2231fwg", - "category": "Accessories", - "price": "$120", - "status": "Out of Stock" - }, - { - "product": "Chakra Bracelet bib36pfvm", - "category": "Accessories", - "price": "$32", - "status": "Low Stock" - }, - { - "product": "Galaxy Earrings mbvjkgip5", - "category": "Accessories", - "price": "$34", - "status": "In Stock" - } - ], - "ignored_column_labels": [ - "Rating" - ] - } - } - }, - "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 Out of Stock\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nPREVIEW", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-scoped-games-jp-new-games", - "category": "acceptance-contract", - "root_cause": "nintendo_real_card_routes_not_historical_membership", - "failure_mode": "false-negative", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)" - } - } - }, - "artifact": "<div class=\"nc3-c-softCard\"><a href=\"https://store-jp.nintendo.com/item/software/D70010000999991\"><span class=\"nc3-c-softCard__name\">Future Game Alpha</span></a></div><div class=\"nc3-c-softCard\"><a href=\"https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc\"><span class=\"nc3-c-softCard__name\">Future Game Beta</span></a></div>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "nintendo-scoped-games-jp-markdown", - "category": "acceptance-contract", - "root_cause": "nintendo_real_card_routes_not_historical_membership", - "failure_mode": "false-negative", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)" - } - } - }, - "artifact": "[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "nintendo-scoped-games-jp-empty-title", - "category": "acceptance-contract", - "root_cause": "nintendo_real_card_routes_not_historical_membership", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)" - } - } - }, - "artifact": "<div class=\"nc3-c-softCard\"><a href=\"https://store-jp.nintendo.com/item/software/D70010000999991\"><span class=\"nc3-c-softCard__name\"></span></a></div><div class=\"nc3-c-softCard\"><a href=\"https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc\"><span class=\"nc3-c-softCard__name\">Future Game Beta</span></a></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-scoped-games-jp-duplicates", - "category": "acceptance-contract", - "root_cause": "nintendo_real_card_routes_not_historical_membership", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)" - } - } - }, - "artifact": "<div class=\"nc3-c-softCard\"><a href=\"https://store-jp.nintendo.com/item/software/D70010000999991\"><span class=\"nc3-c-softCard__name\">Future Game Alpha</span></a></div><div class=\"nc3-c-softCard\"><a href=\"https://store-jp.nintendo.com/item/software/D70010000999991\"><span class=\"nc3-c-softCard__name\">Future Game Alpha</span></a></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-scoped-games-jp-navigation-only", - "category": "acceptance-contract", - "root_cause": "nintendo_real_card_routes_not_historical_membership", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)" - } - } - }, - "artifact": "<nav><a href=\"https://store-jp.nintendo.com/item/software/D70010000999991\">Future Game Alpha</a><a href=\"https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc\">Future Game Beta</a></nav>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-scoped-games-jp-unrelated-path", - "category": "acceptance-contract", - "root_cause": "nintendo_real_card_routes_not_historical_membership", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)" - } - } - }, - "artifact": "<div class=\"nc3-c-softCard\"><a href=\"https://www.nintendo.com/help\"><span class=\"nc3-c-softCard__name\">Help</span></a></div><div class=\"nc3-c-softCard\"><a href=\"https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc\"><span class=\"nc3-c-softCard__name\">Future Game Beta</span></a></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-scoped-games-es-new-games", - "category": "acceptance-contract", - "root_cause": "nintendo_real_card_routes_not_historical_membership", - "failure_mode": "false-negative", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com" - ], - "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".searchresult_row", - "title_selector": ".page-title-text", - "link_selector": "a[href]:has(.page-title-text)" - } - } - }, - "artifact": "<div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html\"><span class=\"page-title-text\">Future Game Alpha</span></a></div><div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Juegos-de-dispositivo-inteligente/Another-Game-999992.html\"><span class=\"page-title-text\">Future Game Beta</span></a></div>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "nintendo-scoped-games-es-markdown", - "category": "acceptance-contract", - "root_cause": "nintendo_real_card_routes_not_historical_membership", - "failure_mode": "false-negative", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com" - ], - "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".searchresult_row", - "title_selector": ".page-title-text", - "link_selector": "a[href]:has(.page-title-text)" - } - } - }, - "artifact": "[Future Game Alpha](https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html)\n[Future Game Beta](https://www.nintendo.com/es-es/Juegos/Juegos-de-dispositivo-inteligente/Another-Game-999992.html)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "nintendo-scoped-games-es-empty-title", - "category": "acceptance-contract", - "root_cause": "nintendo_real_card_routes_not_historical_membership", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com" - ], - "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".searchresult_row", - "title_selector": ".page-title-text", - "link_selector": "a[href]:has(.page-title-text)" - } - } - }, - "artifact": "<div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html\"><span class=\"page-title-text\"></span></a></div><div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Juegos-de-dispositivo-inteligente/Another-Game-999992.html\"><span class=\"page-title-text\">Future Game Beta</span></a></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-scoped-games-es-duplicates", - "category": "acceptance-contract", - "root_cause": "nintendo_real_card_routes_not_historical_membership", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com" - ], - "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".searchresult_row", - "title_selector": ".page-title-text", - "link_selector": "a[href]:has(.page-title-text)" - } - } - }, - "artifact": "<div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html\"><span class=\"page-title-text\">Future Game Alpha</span></a></div><div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html\"><span class=\"page-title-text\">Future Game Alpha</span></a></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-scoped-games-es-navigation-only", - "category": "acceptance-contract", - "root_cause": "nintendo_real_card_routes_not_historical_membership", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com" - ], - "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".searchresult_row", - "title_selector": ".page-title-text", - "link_selector": "a[href]:has(.page-title-text)" - } - } - }, - "artifact": "<nav><a href=\"https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html\">Future Game Alpha</a><a href=\"https://www.nintendo.com/es-es/Juegos/Juegos-de-dispositivo-inteligente/Another-Game-999992.html\">Future Game Beta</a></nav>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-scoped-games-es-unrelated-path", - "category": "acceptance-contract", - "root_cause": "nintendo_real_card_routes_not_historical_membership", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com" - ], - "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".searchresult_row", - "title_selector": ".page-title-text", - "link_selector": "a[href]:has(.page-title-text)" - } + { + "id": "import", + "alternatives": [ + "Import" + ] } - }, - "artifact": "<div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/help\"><span class=\"page-title-text\">Help</span></a></div><div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Juegos-de-dispositivo-inteligente/Another-Game-999992.html\"><span class=\"page-title-text\">Future Game Beta</span></a></div>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "forbidden-html-surface-dormant-template", - "category": "answer-surface", - "root_cause": "dormant_html_templates_not_blocking_notices", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/catalog", - "task_kind": "body", - "expected": [ - "Game catalog" - ], - "forbidden_any": [ - "CAPTCHA", - "Access denied" - ], - "min_content_chars": 1 - }, - "artifact": "<html><body><h1>Game catalog</h1><p>Actual game list</p><script type=\"text/x-template\">CAPTCHA</script></body></html>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "forbidden-html-surface-visible-challenge", - "category": "answer-surface", - "root_cause": "dormant_html_templates_not_blocking_notices", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/catalog", - "task_kind": "body", - "expected": [ - "Game catalog" - ], - "forbidden_any": [ - "CAPTCHA", - "Access denied" - ], - "min_content_chars": 1 - }, - "artifact": "<html><body><h1>Game catalog</h1><p>CAPTCHA required</p></body></html>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "forbidden-html-surface-explicitly-hidden", - "category": "answer-surface", - "root_cause": "dormant_html_templates_not_blocking_notices", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/catalog", - "task_kind": "body", - "expected": [ - "Game catalog" - ], - "forbidden_any": [ - "CAPTCHA", - "Access denied" - ], - "min_content_chars": 1 - }, - "artifact": "<main>Game catalog<div hidden>Access denied</div></main>", - "ok": true, - "expected_task_pass": true - }, - { - "id": "forbidden-html-surface-actual-denial", - "category": "answer-surface", - "root_cause": "dormant_html_templates_not_blocking_notices", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/catalog", - "task_kind": "body", - "expected": [ - "Game catalog" - ], - "forbidden_any": [ - "CAPTCHA", - "Access denied" + ] + } + }, + "artifact": "Documentation\nInstall\nImport\nCAPTCHA", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "forbidden_requirement_hits": [ + "CAPTCHA" + ] + } + }, + { + "id": "literal-footer-pipe-remains-matchable", + "category": "format-normalization", + "root_cause": "multi_pipe_prose_misclassified_as_markdown_table", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/pipe-text", + "task_kind": "body", + "required_facts": [ + { + "id": "required", + "alternatives": [ + "上海互联网举报中心 |" + ] + } + ] + }, + "artifact": "上海互联网举报中心 | 儿童色情信息举报专区 |", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "fact_coverage": 1.0 + } + }, + { + "id": "multiple-literal-pipes-preserve-operators", + "category": "format-normalization", + "root_cause": "multi_pipe_prose_misclassified_as_markdown_table", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/pipe-text", + "task_kind": "body", + "required_facts": [ + { + "id": "required", + "alternatives": [ + "alpha beta gamma" + ] + } + ] + }, + "artifact": "alpha | beta | gamma", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "fact_coverage": 0.0 + } + }, + { + "category": "format-normalization", + "root_cause": "multi_pipe_prose_misclassified_as_markdown_table", + "ok": true, + "artifact": "| Expression | Result |\n| --- | --- |\n| alpha \\| beta | gamma |", + "id": "escaped-table-pipe-keeps-literal-expression", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/pipe-table", + "task_kind": "body", + "required_facts": [ + { + "id": "expression", + "alternatives": [ + "alpha | beta" + ] + } + ] + }, + "expected_task_pass": true + }, + { + "category": "format-normalization", + "root_cause": "multi_pipe_prose_misclassified_as_markdown_table", + "ok": true, + "artifact": "| Expression | Result |\n| --- | --- |\n| alpha \\| beta | gamma |", + "id": "escaped-table-pipe-cannot-be-removed", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/pipe-table", + "task_kind": "body", + "required_facts": [ + { + "id": "expression", + "alternatives": [ + "alpha beta" + ] + } + ] + }, + "expected_task_pass": false + }, + { + "id": "dynamic-membership-new-entities-accepted", + "category": "acceptance-contract", + "root_cause": "dynamic_list_membership_not_identity", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "record_count_policy": { + "list_type": "variable", + "source_observed_count": 2, + "required_ratio": 0.8, + "minimum_records": 2, + "source_observation_counts": [ + 2, + 2, + 2 ], - "min_content_chars": 1 - }, - "artifact": "<main>Game catalog<div>Access denied</div></main>", - "ok": true, - "expected_task_pass": false - }, - { - "id": "forbidden-html-surface-plain-challenge", - "category": "answer-surface", - "root_cause": "dormant_html_templates_not_blocking_notices", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/catalog", - "task_kind": "body", - "expected": [ - "Game catalog" + "source_evidence_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "reason": "Independent source count; list membership is not frozen." + } + } + }, + "artifact": "[New first title](/item/91)\n[New second title](/item/92)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-membership-wrong-path-rejected", + "category": "acceptance-contract", + "root_cause": "dynamic_list_membership_not_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "record_count_policy": { + "list_type": "variable", + "source_observed_count": 2, + "required_ratio": 0.8, + "minimum_records": 2, + "source_observation_counts": [ + 2, + 2, + 2 ], - "forbidden_any": [ - "CAPTCHA", - "Access denied" + "source_evidence_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "reason": "Independent source count; list membership is not frozen." + } + } + }, + "artifact": "[New first title](/item/91)\n[New second title](/category/92)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-membership-empty-title-rejected", + "category": "acceptance-contract", + "root_cause": "dynamic_list_membership_not_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "record_count_policy": { + "list_type": "variable", + "source_observed_count": 2, + "required_ratio": 0.8, + "minimum_records": 2, + "source_observation_counts": [ + 2, + 2, + 2 ], - "min_content_chars": 1 - }, - "artifact": "Game catalog\nCAPTCHA required", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-markdown-section-actual-game-section", - "category": "structured-extraction", - "root_cause": "game_card_announcements_not_navigation_news", - "failure_mode": "false-negative", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)", - "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", - "end_pattern": "^[ \\t#*]*ゲームを楽しむ" - } - } - }, - "artifact": "## ピックアップ\n[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)\n### ゲームを楽しむ", - "ok": true, - "expected_task_pass": true - }, - { - "id": "nintendo-markdown-section-navigation-news-only", - "category": "structured-extraction", - "root_cause": "game_card_announcements_not_navigation_news", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)", - "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", - "end_pattern": "^[ \\t#*]*ゲームを楽しむ" - } - } - }, - "artifact": "[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)\n## ピックアップ\nNo game records\n### ゲームを楽しむ", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-markdown-section-links-after-game-section", - "category": "structured-extraction", - "root_cause": "game_card_announcements_not_navigation_news", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)", - "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", - "end_pattern": "^[ \\t#*]*ゲームを楽しむ" - } - } - }, - "artifact": "## ピックアップ\nNo game records\n### ゲームを楽しむ\n[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-markdown-section-bullet-heading", - "category": "structured-extraction", - "root_cause": "game_card_announcements_not_navigation_news", - "failure_mode": "false-negative", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)", - "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", - "end_pattern": "^[ \\t#*]*ゲームを楽しむ" - } - } - }, - "artifact": "* ピックアップ\n[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-026-bilibili-rank-markdown-scope-inside", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual Name 0](https://www.bilibili.com/video/BV1Ab4111111)\n[Actual Name 1](https://www.bilibili.com/video/BV1Ab4222222)\nbilibili", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-026-bilibili-rank-markdown-scope-navigation-only", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "[Actual Name 0](https://www.bilibili.com/video/BV1Ab4111111)\n[Actual Name 1](https://www.bilibili.com/video/BV1Ab4222222)\n排行榜根据稿件内容质量，近期的数据综合展示，动态更新\nNo target records\nbilibili", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-026-bilibili-rank-markdown-scope-footer-only", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\nNo target records\nbilibili\n[Actual Name 0](https://www.bilibili.com/video/BV1Ab4111111)\n[Actual Name 1](https://www.bilibili.com/video/BV1Ab4222222)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-048-en-github-trending-developers-markdown-scope-inside", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://github.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!login$|signup$|features$|topics$|trending$)[A-Za-z0-9_.-]+", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "article.Box-row h1", - "start_pattern": "These are the developers building the hot tools today\\.", - "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "These are the developers building the hot tools today.\n[Actual Name 0](https://github.com/newalpha)\n[Actual Name 1](https://github.com/newbeta)\nFooter", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-048-en-github-trending-developers-markdown-scope-navigation-only", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://github.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!login$|signup$|features$|topics$|trending$)[A-Za-z0-9_.-]+", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "article.Box-row h1", - "start_pattern": "These are the developers building the hot tools today\\.", - "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "[Actual Name 0](https://github.com/newalpha)\n[Actual Name 1](https://github.com/newbeta)\nThese are the developers building the hot tools today.\nNo target records\nFooter", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-048-en-github-trending-developers-markdown-scope-footer-only", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://github.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!login$|signup$|features$|topics$|trending$)[A-Za-z0-9_.-]+", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "article.Box-row h1", - "start_pattern": "These are the developers building the hot tools today\\.", - "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" - }, - "name_validation": "nonempty" + "source_evidence_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "reason": "Independent source count; list membership is not frozen." + } + } + }, + "artifact": "[New first title](/item/91)\n[](/item/92)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-membership-short-list-rejected", + "category": "acceptance-contract", + "root_cause": "dynamic_list_membership_not_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "record_count_policy": { + "list_type": "variable", + "source_observed_count": 2, + "required_ratio": 0.8, + "minimum_records": 2, + "source_observation_counts": [ + 2, + 2, + 2 + ], + "source_evidence_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "reason": "Independent source count; list membership is not frozen." + } + } + }, + "artifact": "[New first title](/item/91)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-membership-duplicate-not-extra-item", + "category": "acceptance-contract", + "root_cause": "dynamic_list_membership_not_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "record_count_policy": { + "list_type": "variable", + "source_observed_count": 2, + "required_ratio": 0.8, + "minimum_records": 2, + "source_observation_counts": [ + 2, + 2, + 2 + ], + "source_evidence_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "reason": "Independent source count; list membership is not frozen." + } + } + }, + "artifact": "[New first title](/item/91)\n[New first title](/item/91)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "quake-complete-visible-cards", + "category": "structured-extraction", + "root_cause": "earthquake_record_field_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)\n10.0 km\n3.1\nNorth Valley\n2026-09-20T05:00:00Z\n7.4 km", + "ok": true, + "expected_task_pass": true + }, + { + "id": "quake-complete-markdown-cards", + "category": "structured-extraction", + "root_cause": "earthquake_record_field_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "[4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)\n10.0 km](/event/1)\n[3.1\nNorth Valley\n2026-09-20T05:00:00Z\n7.4 km](/event/2)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "quake-table-fields", + "category": "structured-extraction", + "root_cause": "earthquake_record_field_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "| magnitude | location | time | depth |\n| --- | --- | --- | --- |\n| 4.2 | Harbor Island | 2026-09-20T04:00:00Z | 10.0 km |\n| 3.1 | North Valley | 2026-09-20T05:00:00Z | 7.4 km |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "quake-dates-alone", + "category": "structured-extraction", + "root_cause": "earthquake_record_field_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "Earthquakes\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n2026-09-20T05:00:00Z\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "quake-missing-depth", + "category": "structured-extraction", + "root_cause": "earthquake_record_field_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)\n3.1\nNorth Valley\n2026-09-20T05:00:00Z\n7.4 km", + "ok": true, + "expected_task_pass": false + }, + { + "id": "quake-split-card-must-not-borrow", + "category": "structured-extraction", + "root_cause": "earthquake_record_field_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "[4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)](/event/1)\n[10.0 km\n3.1\nNorth Valley\n2026-09-20T05:00:00Z\n7.4 km](/event/2)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "quake-duplicate-not-extra-event", + "category": "structured-extraction", + "root_cause": "earthquake_record_field_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)\n10.0 km\n4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)\n10.0 km", + "ok": true, + "expected_task_pass": false + }, + { + "id": "quake-invalid-calendar", + "category": "structured-extraction", + "root_cause": "earthquake_record_field_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "4.2\nHarbor Island\n2026-09-20 12:00:00 (UTC+08:00)\n10.0 km\n3.1\nNorth Valley\n2026-02-30T05:00:00Z\n7.4 km", + "ok": true, + "expected_task_pass": false + }, + { + "id": "body-single-source-occurrence-not-duplicated", + "category": "reference-evidence", + "root_cause": "collector_parent_child_text_duplication", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/api-docs", + "task_kind": "body", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "client name identifies the application requesting authorization redirect addresses determine where an approved response can be delivered", + "source_text": "Client name identifies the application requesting authorization.\n\nRedirect addresses determine where an approved response can be delivered.", + "segmentation_method": "sentences" + }, + { + "id": "body-region-2", + "text": "allowed scopes specify the operations available to this application grant types describe the supported authentication flows the service returns the registered identifier and a secret for subsequent requests", + "source_text": "Allowed scopes specify the operations available to this application.\n\nGrant types describe the supported authentication flows.\n\nThe service returns the registered identifier and a secret for subsequent requests.", + "segmentation_method": "sentences" } - }, - "artifact": "These are the developers building the hot tools today.\nNo target records\nFooter\n[Actual Name 0](https://github.com/newalpha)\n[Actual Name 1](https://github.com/newbeta)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-044-en-real-github-trending-markdown-scope-inside", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://github.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!trending(?:/|$)|apps/|sponsors/|topics/|collections/|settings/|features/|marketplace/|orgs/|users/|login/|signup/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "article.Box-row h2", - "start_pattern": "See what the GitHub community is most excited about today\\.", - "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" - }, - "name_validation": "nonempty" + ] + } + }, + "artifact": "Client name identifies the application requesting authorization. Redirect addresses determine where an approved response can be delivered. Allowed scopes specify the operations available to this application. Grant types describe the supported authentication flows. The service returns the registered identifier and a secret for subsequent requests.", + "ok": true, + "expected_task_pass": true + }, + { + "id": "body-missing-source-paragraph-rejected", + "category": "reference-evidence", + "root_cause": "collector_parent_child_text_duplication", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/api-docs", + "task_kind": "body", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "client name identifies the application requesting authorization redirect addresses determine where an approved response can be delivered", + "source_text": "Client name identifies the application requesting authorization.\n\nRedirect addresses determine where an approved response can be delivered.", + "segmentation_method": "sentences" + }, + { + "id": "body-region-2", + "text": "allowed scopes specify the operations available to this application grant types describe the supported authentication flows the service returns the registered identifier and a secret for subsequent requests", + "source_text": "Allowed scopes specify the operations available to this application.\n\nGrant types describe the supported authentication flows.\n\nThe service returns the registered identifier and a secret for subsequent requests.", + "segmentation_method": "sentences" } - }, - "artifact": "See what the GitHub community is most excited about today.\n[Actual Name 0](https://github.com/newalpha/project)\n[Actual Name 1](https://github.com/newbeta/project)\nFooter", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-044-en-real-github-trending-markdown-scope-navigation-only", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://github.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!trending(?:/|$)|apps/|sponsors/|topics/|collections/|settings/|features/|marketplace/|orgs/|users/|login/|signup/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "article.Box-row h2", - "start_pattern": "See what the GitHub community is most excited about today\\.", - "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" - }, - "name_validation": "nonempty" + ] + } + }, + "artifact": "Client name.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "body-real-repeated-occurrences-preserved", + "category": "reference-evidence", + "root_cause": "collector_parent_child_text_duplication", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/api-docs", + "task_kind": "body", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "paragraphs", + "regions": [ + { + "id": "body-region-1", + "text": "client name identifies the application requesting authorization redirect addresses determine where an approved response can be delivered allowed scopes specify the operations available to this application grant types describe the supported authentication flows the service returns the registered identifier and a secret for subsequent requests", + "source_text": "Client name identifies the application requesting authorization. Redirect addresses determine where an approved response can be delivered. Allowed scopes specify the operations available to this application. Grant types describe the supported authentication flows. The service returns the registered identifier and a secret for subsequent requests.", + "segmentation_method": "paragraphs" + }, + { + "id": "body-region-2", + "text": "client name identifies the application requesting authorization redirect addresses determine where an approved response can be delivered allowed scopes specify the operations available to this application grant types describe the supported authentication flows the service returns the registered identifier and a secret for subsequent requests", + "source_text": "Client name identifies the application requesting authorization. Redirect addresses determine where an approved response can be delivered. Allowed scopes specify the operations available to this application. Grant types describe the supported authentication flows. The service returns the registered identifier and a secret for subsequent requests.", + "segmentation_method": "paragraphs" } - }, - "artifact": "[Actual Name 0](https://github.com/newalpha/project)\n[Actual Name 1](https://github.com/newbeta/project)\nSee what the GitHub community is most excited about today.\nNo target records\nFooter", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-044-en-real-github-trending-markdown-scope-footer-only", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://github.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!trending(?:/|$)|apps/|sponsors/|topics/|collections/|settings/|features/|marketplace/|orgs/|users/|login/|signup/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "article.Box-row h2", - "start_pattern": "See what the GitHub community is most excited about today\\.", - "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" - }, - "name_validation": "nonempty" + ] + } + }, + "artifact": "Client name identifies the application requesting authorization. Redirect addresses determine where an approved response can be delivered. Allowed scopes specify the operations available to this application. Grant types describe the supported authentication flows. The service returns the registered identifier and a secret for subsequent requests.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scoped-list-new-short-names", + "category": "structured-extraction", + "root_cause": "declared_list_scope_and_nonempty_names", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".main-list a", + "start_pattern": "Main chart", + "end_pattern": "Other recommendations" + }, + "stable_text_anchors": [ + { + "id": "footer", + "alternatives": [ + "Company address" + ] } - }, - "artifact": "See what the GitHub community is most excited about today.\nNo target records\nFooter\n[Actual Name 0](https://github.com/newalpha/project)\n[Actual Name 1](https://github.com/newbeta/project)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "auxiliary-video-links-two-main-plus-popup", - "category": "structured-extraction", - "root_cause": "hidden_related_video_popup_not_main_rank", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Main Alpha](https://www.bilibili.com/video/BV1Ab4111111)\n123 views\n[Other One](https://www.bilibili.com/video/BV1Ab4333333)\n[Other Two](https://www.bilibili.com/video/BV1Ab4444444)\n全部上榜视频\n[Main Beta](https://www.bilibili.com/video/BV1Ab4222222)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "auxiliary-video-links-popup-cannot-fill", - "category": "structured-extraction", - "root_cause": "hidden_related_video_popup_not_main_rank", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Main Alpha](https://www.bilibili.com/video/BV1Ab4111111)\n123 views\n[Other One](https://www.bilibili.com/video/BV1Ab4333333)\n[Other Two](https://www.bilibili.com/video/BV1Ab4444444)\n全部上榜视频", - "ok": true, - "expected_task_pass": false - }, - { - "id": "auxiliary-video-links-normal-two-main", - "category": "structured-extraction", - "root_cause": "hidden_related_video_popup_not_main_rank", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Main Alpha](https://www.bilibili.com/video/BV1Ab4111111)\n[Main Beta](https://www.bilibili.com/video/BV1Ab4222222)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "markdown-title-metadata-inner-double", - "category": "format-normalization", - "root_cause": "quoted_title_metadata_not_link_destination", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual \"Name\"](https://www.bilibili.com/video/BV1Ab4111111 \"Actual \"Name\"\")", - "ok": true, - "expected_task_pass": true - }, - { - "id": "markdown-title-metadata-inner-apostrophe", - "category": "format-normalization", - "root_cause": "quoted_title_metadata_not_link_destination", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual Name](https://www.bilibili.com/video/BV1Ab4111111 \"User's title\")", - "ok": true, - "expected_task_pass": true - }, - { - "id": "markdown-title-metadata-wrong-path", - "category": "format-normalization", - "root_cause": "quoted_title_metadata_not_link_destination", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual Name](https://www.bilibili.com/help \"Actual \"Name\"\")", - "ok": true, - "expected_task_pass": false - }, - { - "id": "markdown-title-metadata-bare-url", - "category": "format-normalization", - "root_cause": "quoted_title_metadata_not_link_destination", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\nActual Name https://www.bilibili.com/video/BV1Ab4111111", - "ok": true, - "expected_task_pass": false - }, - { - "id": "adjacent-song-pair-escaped-markdown-complete", - "category": "structured-extraction", - "root_cause": "adjacent_markdown_links_same_song_identity", - "failure_mode": "false-negative", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] - } - }, - "artifact": "[곡정보](/song/detail.htm?songId=101)\n\n[새 노래 Alpha](javascript:melon.play.playSong\\('1000002721',101\\);)\n\n[곡정보](/song/detail.htm?songId=102)\n\n[새 노래 Beta](javascript:melon.play.playSong\\('1000002721',102\\);)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "adjacent-song-pair-mismatched-identity", - "category": "structured-extraction", - "root_cause": "adjacent_markdown_links_same_song_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] - } - }, - "artifact": "[곡정보](/song/detail.htm?songId=101)\n\n[새 노래 Alpha](javascript:melon.play.playSong\\('1000002721',101\\);)\n\n[곡정보](/song/detail.htm?songId=999)\n\n[새 노래 Beta](javascript:melon.play.playSong\\('1000002721',102\\);)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "same-row-song-pair-complete", - "category": "structured-extraction", - "root_cause": "visible_song_name_and_detail_link_same_row_identity", - "failure_mode": "false-negative", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "title_selector": ".rank01 a", - "link_selector": "a.song_info[href]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] - } - }, - "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=102) | [새 노래 Beta](javascript:melon.play.playSong%28'1000002721',102%29;) |", - "ok": true, - "expected_task_pass": true - }, - { - "id": "same-row-song-pair-mismatched-id", - "category": "structured-extraction", - "root_cause": "visible_song_name_and_detail_link_same_row_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "title_selector": ".rank01 a", - "link_selector": "a.song_info[href]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] - } - }, - "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=999) | [새 노래 Beta](javascript:melon.play.playSong%28'1000002721',102%29;) |", - "ok": true, - "expected_task_pass": false - }, - { - "id": "same-row-song-pair-cross-row", - "category": "structured-extraction", - "root_cause": "visible_song_name_and_detail_link_same_row_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "title_selector": ".rank01 a", - "link_selector": "a.song_info[href]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] - } - }, - "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=102) |\n| [새 노래 Beta](javascript:melon.play.playSong%28'1000002721',102%29;) |", - "ok": true, - "expected_task_pass": false - }, - { - "id": "same-row-song-pair-duplicates", - "category": "structured-extraction", - "root_cause": "visible_song_name_and_detail_link_same_row_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "title_selector": ".rank01 a", - "link_selector": "a.song_info[href]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] - } - }, - "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |", - "ok": true, - "expected_task_pass": false - }, - { - "id": "same-row-song-pair-empty-name", - "category": "structured-extraction", - "root_cause": "visible_song_name_and_detail_link_same_row_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "title_selector": ".rank01 a", - "link_selector": "a.song_info[href]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] - } - }, - "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=102) | [](javascript:melon.play.playSong%28'1000002721',102%29;) |", - "ok": true, - "expected_task_pass": false - }, - { - "id": "same-row-song-pair-direct-named-links", - "category": "structured-extraction", - "root_cause": "visible_song_name_and_detail_link_same_row_identity", - "failure_mode": "false-negative", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "title_selector": ".rank01 a", - "link_selector": "a.song_info[href]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] - } - }, - "artifact": "[새 노래 Alpha](/song/detail.htm?songId=101)\n[새 노래 Beta](/song/detail.htm?songId=102)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-052-bilibili-com-v-footer-scope-inside", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/v/popular/all/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".video-card", - "title_selector": "p.video-name", - "link_selector": "a[href*=\"/video/\"]", - "start_pattern": "各个领域中新奇好玩的优质内容都在这里", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "各个领域中新奇好玩的优质内容都在这里~\n[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)\nbilibili", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-052-bilibili-com-v-footer-scope-footer-only", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/v/popular/all/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".video-card", - "title_selector": "p.video-name", - "link_selector": "a[href*=\"/video/\"]", - "start_pattern": "各个领域中新奇好玩的优质内容都在这里", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "各个领域中新奇好玩的优质内容都在这里~\nbilibili\n[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-052-bilibili-com-v-footer-scope-navigation-only", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/v/popular/all/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".video-card", - "title_selector": "p.video-name", - "link_selector": "a[href*=\"/video/\"]", - "start_pattern": "各个领域中新奇好玩的优质内容都在这里", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)\n各个领域中新奇好玩的优质内容都在这里~\nbilibili", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-052-bilibili-com-v-footer-scope-no-footer", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/v/popular/all/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".video-card", - "title_selector": "p.video-name", - "link_selector": "a[href*=\"/video/\"]", - "start_pattern": "各个领域中新奇好玩的优质内容都在这里", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "各个领域中新奇好玩的优质内容都在这里~\n[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-063-s-weibo-com-top-footer-scope-inside", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-negative", - "case": { - "url": "https://s.weibo.com/top/summary", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "s.weibo.com" - ], - "entry_path": "/weibo", - "stable_text_anchors": [], - "identity_query_keys": [ - "q" - ], - "entry_scope": { - "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", - "start_pattern": "序号[^\\n]*关键词", - "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" - }, - "name_validation": "nonempty" - } - }, - "artifact": "序号 关键词\n[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)\n关于微博 微博帮助", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-063-s-weibo-com-top-footer-scope-footer-only", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-positive", - "case": { - "url": "https://s.weibo.com/top/summary", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "s.weibo.com" - ], - "entry_path": "/weibo", - "stable_text_anchors": [], - "identity_query_keys": [ - "q" - ], - "entry_scope": { - "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", - "start_pattern": "序号[^\\n]*关键词", - "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" - }, - "name_validation": "nonempty" - } - }, - "artifact": "序号 关键词\n关于微博 微博帮助\n[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-063-s-weibo-com-top-footer-scope-navigation-only", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-positive", - "case": { - "url": "https://s.weibo.com/top/summary", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "s.weibo.com" - ], - "entry_path": "/weibo", - "stable_text_anchors": [], - "identity_query_keys": [ - "q" - ], - "entry_scope": { - "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", - "start_pattern": "序号[^\\n]*关键词", - "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" - }, - "name_validation": "nonempty" - } - }, - "artifact": "[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)\n序号 关键词\n关于微博 微博帮助", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-063-s-weibo-com-top-footer-scope-no-footer", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-negative", - "case": { - "url": "https://s.weibo.com/top/summary", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "s.weibo.com" - ], - "entry_path": "/weibo", - "stable_text_anchors": [], - "identity_query_keys": [ - "q" - ], - "entry_scope": { - "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", - "start_pattern": "序号[^\\n]*关键词", - "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" - }, - "name_validation": "nonempty" - } - }, - "artifact": "序号 关键词\n[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "arabic-currency-visible-format-prefix", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-negative", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ] + } + }, + "artifact": "Main chart\n[918](/item/1)\n[爱](/item/2)\nOther recommendations\nCompany address", + "ok": true, + "expected_task_pass": true + }, + { + "id": "scoped-list-sidebar-not-main", + "category": "structured-extraction", + "root_cause": "declared_list_scope_and_nonempty_names", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".main-list a", + "start_pattern": "Main chart", + "end_pattern": "Other recommendations" + }, + "stable_text_anchors": [ + { + "id": "footer", + "alternatives": [ + "Company address" + ] } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم 659.99\nدرهم 659.99\nدرهم 659.99\nدرهم 659.99\nدرهم 659.99\n", - "ok": true, - "expected_task_pass": true - }, - { - "id": "arabic-currency-visible-format-suffix", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-negative", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ] + } + }, + "artifact": "Main chart\n[918](/item/1)\nOther recommendations\n[爱](/item/2)\nCompany address", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scoped-list-html-valid", + "category": "structured-extraction", + "root_cause": "declared_list_scope_and_nonempty_names", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".main-list a", + "start_pattern": "Main chart", + "end_pattern": "Other recommendations" + }, + "stable_text_anchors": [ + { + "id": "footer", + "alternatives": [ + "Company address" + ] } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\n659.99 درهم\n659.99 درهم\n659.99 درهم\n659.99 درهم\n659.99 درهم\n", - "ok": true, - "expected_task_pass": true - }, - { - "id": "arabic-currency-visible-format-direction-marks", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-negative", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ] + } + }, + "artifact": "<div class=\"main-list\"><a href=\"/item/1\">918</a><a href=\"/item/2\">爱</a></div><footer>Company address</footer>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "scoped-list-html-sidebar", + "category": "structured-extraction", + "root_cause": "declared_list_scope_and_nonempty_names", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".main-list a", + "start_pattern": "Main chart", + "end_pattern": "Other recommendations" + }, + "stable_text_anchors": [ + { + "id": "footer", + "alternatives": [ + "Company address" + ] } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\n‏659.99 درهم‎\n‏659.99 درهم‎\n‏659.99 درهم‎\n‏659.99 درهم‎\n‏659.99 درهم‎\n", - "ok": true, - "expected_task_pass": true - }, - { - "id": "arabic-currency-visible-format-prefix-direction", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-negative", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ] + } + }, + "artifact": "<div class=\"main-list\"><a href=\"/item/1\">918</a></div><aside><a href=\"/item/2\">爱</a></aside><footer>Company address</footer>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scoped-list-empty-name", + "category": "structured-extraction", + "root_cause": "declared_list_scope_and_nonempty_names", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".main-list a", + "start_pattern": "Main chart", + "end_pattern": "Other recommendations" + }, + "stable_text_anchors": [ + { + "id": "footer", + "alternatives": [ + "Company address" + ] } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم‎659.99‏\nدرهم‎659.99‏\nدرهم‎659.99‏\nدرهم‎659.99‏\nدرهم‎659.99‏\n", - "ok": true, - "expected_task_pass": true - }, - { - "id": "arabic-currency-visible-format-four-only", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-positive", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ] + } + }, + "artifact": "Main chart\n[](/item/1)\n[爱](/item/2)\nCompany address", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scoped-list-absent-heading", + "category": "structured-extraction", + "root_cause": "declared_list_scope_and_nonempty_names", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".main-list a", + "start_pattern": "Main chart", + "end_pattern": "Other recommendations" + }, + "stable_text_anchors": [ + { + "id": "footer", + "alternatives": [ + "Company address" + ] } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\n659.99 درهم\n659.99 درهم\n659.99 درهم\n659.99 درهم\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "arabic-currency-visible-format-unrelated-digits", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-positive", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ] + } + }, + "artifact": "[918](/item/1)\n[爱](/item/2)\nCompany address", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scoped-list-hidden-html", + "category": "structured-extraction", + "root_cause": "declared_list_scope_and_nonempty_names", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".main-list a", + "start_pattern": "Main chart", + "end_pattern": "Other recommendations" + }, + "stable_text_anchors": [ + { + "id": "footer", + "alternatives": [ + "Company address" + ] } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "arabic-currency-visible-format-currency-only", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-positive", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ] + } + }, + "artifact": "<div class=\"main-list\"><a href=\"/item/1\">918</a><a hidden href=\"/item/2\">爱</a></div><footer>Company address</footer>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "owned-sibling-title-detail-link", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/song/detail\\.htm", + "identity_query_keys": [ + "songId" + ], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".card", + "title_selector": ".name", + "link_selector": "a.detail" + } + } + }, + "artifact": "<div class=\"card\"><p class=\"name\">Real song</p><a class=\"detail\" href=\"/song/detail.htm?songId=123\">Song information</a></div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "owned-sibling-does-not-cross-card", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/song/detail\\.htm", + "identity_query_keys": [ + "songId" + ], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".card", + "title_selector": ".name", + "link_selector": "a.detail" + } + } + }, + "artifact": "<div class=\"card\"><p class=\"name\">Real song</p></div><div class=\"card\"><a class=\"detail\" href=\"/song/detail.htm?songId=123\">Song information</a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "owned-sibling-ambiguous-link", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/song/detail\\.htm", + "identity_query_keys": [ + "songId" + ], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".card", + "title_selector": ".name", + "link_selector": "a.detail" + } + } + }, + "artifact": "<div class=\"card\"><p class=\"name\">Real song</p><a class=\"detail\" href=\"/song/detail.htm?songId=123\">Song information</a><a class=\"detail\" href=\"/song/detail.htm?songId=456\">Second song</a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "owned-sibling-hidden-title", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/song/detail\\.htm", + "identity_query_keys": [ + "songId" + ], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".card", + "title_selector": ".name", + "link_selector": "a.detail" + } + } + }, + "artifact": "<div class=\"card\"><p class=\"name\" hidden>Real song</p><a class=\"detail\" href=\"/song/detail.htm?songId=123\">Song information</a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "song-detail-literal-dot-not-wildcard", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/song/detail\\.htm", + "identity_query_keys": [ + "songId" + ], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".card", + "title_selector": ".name", + "link_selector": "a.detail" + } + } + }, + "artifact": "<div class=\"card\"><p class=\"name\">Real song</p><a class=\"detail\" href=\"/song/detailXhtm?songId=123\">Song information</a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "owned-title-equivalent-markdown", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/chart", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/song/detail\\.htm", + "identity_query_keys": [ + "songId" + ], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".card", + "title_selector": ".name", + "link_selector": "a.detail" + } + } + }, + "artifact": "[Real song](/song/detail.htm?songId=123)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "product-owned-price-visible", + "category": "structured-extraction", + "root_cause": "source_declared_price_availability", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/search", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "linked_product", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "price_patterns": [ + "\\$[0-9]+\\.[0-9]{2}" + ], + "price_unavailable_patterns": [ + "Click to see price" + ], + "entry_scope": { + "html_selector": ".product", + "title_selector": ".name", + "link_selector": "a.detail", + "price_selector": ".price" + } + } + }, + "artifact": "<div class=\"product\"><h2 class=\"name\">Headphones</h2><a class=\"detail\" href=\"/dp/B012345678\">Details</a><span class=\"price\">$29.99</span></div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "product-explicit-source-price-unavailable", + "category": "structured-extraction", + "root_cause": "source_declared_price_availability", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/search", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "linked_product", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "price_patterns": [ + "\\$[0-9]+\\.[0-9]{2}" + ], + "price_unavailable_patterns": [ + "Click to see price" + ], + "entry_scope": { + "html_selector": ".product", + "title_selector": ".name", + "link_selector": "a.detail", + "price_selector": ".price" + } + } + }, + "artifact": "<div class=\"product\"><h2 class=\"name\">Headphones</h2><a class=\"detail\" href=\"/dp/B012345678\">Details</a>Click to see price</div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "product-price-simply-missing", + "category": "structured-extraction", + "root_cause": "source_declared_price_availability", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/search", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "linked_product", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "price_patterns": [ + "\\$[0-9]+\\.[0-9]{2}" + ], + "price_unavailable_patterns": [ + "Click to see price" + ], + "entry_scope": { + "html_selector": ".product", + "title_selector": ".name", + "link_selector": "a.detail", + "price_selector": ".price" + } + } + }, + "artifact": "<div class=\"product\"><h2 class=\"name\">Headphones</h2><a class=\"detail\" href=\"/dp/B012345678\">Details</a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "product-price-cannot-borrow-from-neighbor", + "category": "structured-extraction", + "root_cause": "source_declared_price_availability", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/search", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "linked_product", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "price_patterns": [ + "\\$[0-9]+\\.[0-9]{2}" + ], + "price_unavailable_patterns": [ + "Click to see price" + ], + "entry_scope": { + "html_selector": ".product", + "title_selector": ".name", + "link_selector": "a.detail", + "price_selector": ".price" + } + } + }, + "artifact": "<div class=\"product\"><h2 class=\"name\">Headphones</h2><a class=\"detail\" href=\"/dp/B012345678\">Details</a></div><div class=\"product\"><span class=\"price\">$29.99</span></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "product-unavailable-cannot-borrow", + "category": "structured-extraction", + "root_cause": "source_declared_price_availability", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/search", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "linked_product", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "price_patterns": [ + "\\$[0-9]+\\.[0-9]{2}" + ], + "price_unavailable_patterns": [ + "Click to see price" + ], + "entry_scope": { + "html_selector": ".product", + "title_selector": ".name", + "link_selector": "a.detail", + "price_selector": ".price" + } + } + }, + "artifact": "<div class=\"product\"><h2 class=\"name\">Headphones</h2><a class=\"detail\" href=\"/dp/B012345678\">Details</a></div><div class=\"product\">Click to see price</div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "product-price-markdown-equivalent", + "category": "structured-extraction", + "root_cause": "source_declared_price_availability", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/search", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "linked_product", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "price_patterns": [ + "\\$[0-9]+\\.[0-9]{2}" + ], + "price_unavailable_patterns": [ + "Click to see price" + ], + "entry_scope": { + "html_selector": ".product", + "title_selector": ".name", + "link_selector": "a.detail", + "price_selector": ".price" + } + } + }, + "artifact": "[Headphones](/dp/B012345678)\n$29.99", + "ok": true, + "expected_task_pass": true + }, + { + "id": "product-unavailable-markdown-equivalent", + "category": "structured-extraction", + "root_cause": "source_declared_price_availability", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/search", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "linked_product", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/dp/[A-Z0-9]{10}(?:/ref=[^/]+)?", + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "price_patterns": [ + "\\$[0-9]+\\.[0-9]{2}" + ], + "price_unavailable_patterns": [ + "Click to see price" + ], + "entry_scope": { + "html_selector": ".product", + "title_selector": ".name", + "link_selector": "a.detail", + "price_selector": ".price" + } + } + }, + "artifact": "[Headphones](/dp/B012345678)\nClick to see price", + "ok": true, + "expected_task_pass": true + }, + { + "id": "currency-whitespace-accepts-source-format", + "category": "acceptance-contract", + "root_cause": "overescaped_source_field_patterns", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "page_patterns": [ + "₹\\s*[0-9]" + ] + } + }, + "artifact": "[Actual item](/item/1)\n₹ 1499", + "ok": true, + "expected_task_pass": true + }, + { + "id": "currency-whitespace-rejects-non-field", + "category": "acceptance-contract", + "root_cause": "overescaped_source_field_patterns", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "page_patterns": [ + "₹\\s*[0-9]" + ] + } + }, + "artifact": "[Actual item](/item/1)\nRs unknown", + "ok": true, + "expected_task_pass": false + }, + { + "id": "counter-word-boundaries-accepts-source-format", + "category": "acceptance-contract", + "root_cause": "overescaped_source_field_patterns", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "page_patterns": [ + "\\b[0-9][0-9,]* points\\b" + ] + } + }, + "artifact": "[Actual item](/item/1)\n123 points", + "ok": true, + "expected_task_pass": true + }, + { + "id": "counter-word-boundaries-rejects-non-field", + "category": "acceptance-contract", + "root_cause": "overescaped_source_field_patterns", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "page_patterns": [ + "\\b[0-9][0-9,]* points\\b" + ] + } + }, + "artifact": "[Actual item](/item/1)\n123 appointments", + "ok": true, + "expected_task_pass": false + }, + { + "id": "api-summary-reference-complete", + "category": "reference-evidence", + "root_cause": "api_operation_reference_omission", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/api", + "task_kind": "body", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "paragraphs", + "regions": [ + { + "id": "body-region-1", + "text": "get animals list registered animals and their current attributes post animals register a new animal using the submitted data delete animals id remove the animal identified by its identifier", + "source_text": "GET /animals List registered animals and their current attributes.\n\nPOST /animals Register a new animal using the submitted data.\n\nDELETE /animals/{id} Remove the animal identified by its identifier.", + "segmentation_method": "paragraphs" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "api_operation", + "minimum_records": 3 + } + }, + "artifact": "GET /animals List registered animals and their current attributes.\n\nPOST /animals Register a new animal using the submitted data.\n\nDELETE /animals/{id} Remove the animal identified by its identifier.", + "ok": true, + "expected_task_pass": true + }, + { + "id": "api-paths-alone-not-summaries", + "category": "reference-evidence", + "root_cause": "api_operation_reference_omission", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/api", + "task_kind": "body", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "paragraphs", + "regions": [ + { + "id": "body-region-1", + "text": "get animals list registered animals and their current attributes post animals register a new animal using the submitted data delete animals id remove the animal identified by its identifier", + "source_text": "GET /animals List registered animals and their current attributes.\n\nPOST /animals Register a new animal using the submitted data.\n\nDELETE /animals/{id} Remove the animal identified by its identifier.", + "segmentation_method": "paragraphs" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "api_operation", + "minimum_records": 3 + } + }, + "artifact": "/animals\n/animals/{id}\nAPI documentation", + "ok": true, + "expected_task_pass": false + }, + { + "id": "api-missing-operation-summary", + "category": "reference-evidence", + "root_cause": "api_operation_reference_omission", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/api", + "task_kind": "body", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "paragraphs", + "regions": [ + { + "id": "body-region-1", + "text": "get animals list registered animals and their current attributes post animals register a new animal using the submitted data delete animals id remove the animal identified by its identifier", + "source_text": "GET /animals List registered animals and their current attributes.\n\nPOST /animals Register a new animal using the submitted data.\n\nDELETE /animals/{id} Remove the animal identified by its identifier.", + "segmentation_method": "paragraphs" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "api_operation", + "minimum_records": 3 + } + }, + "artifact": "GET /animals List registered animals and their current attributes.\n\nPOST /animals Register a new animal using the submitted data.\n\nDELETE /animals/{id} ", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-real-row-shape", + "category": "structured-extraction", + "root_cause": "table_record_field_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" + } + } + }, + "artifact": "The table below has 1000 rows\n1Ada Example20female4blue05/12/2025\n2Bob Example70male3green22/05/2026", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-date-list-not-records", + "category": "structured-extraction", + "root_cause": "table_record_field_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" + } + } + }, + "artifact": "The table below has 1000 rows\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n05/12/2025\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-missing-cell-not-complete", + "category": "structured-extraction", + "root_cause": "table_record_field_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" + } + } + }, + "artifact": "The table below has 1000 rows\n1Ada Example20female4blue05/12/2025\n2Bob Example70male322/05/2026", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-invalid-calendar-not-complete", + "category": "structured-extraction", + "root_cause": "table_record_field_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" + } + } + }, + "artifact": "The table below has 1000 rows\n1Ada Example20female4blue05/12/2025\n2Bob Example70male3green31/02/2026", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-duplicate-id-not-extra-record", + "category": "structured-extraction", + "root_cause": "table_record_field_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" + } + } + }, + "artifact": "The table below has 1000 rows\n1Ada Example20female4blue05/12/2025\n1Bob Example70male3green22/05/2026", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-other-section-cannot-fill", + "category": "structured-extraction", + "root_cause": "table_record_field_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" + } + } + }, + "artifact": "The table below has 1000 rows\n1Ada Example20female4blue05/12/2025\nView Source\n2Bob Example70male3green22/05/2026", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-markdown-row-ownership", + "category": "structured-extraction", + "root_cause": "table_record_field_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})" + } + } + }, + "artifact": "The table below has 1000 rows\n| ID | Name | Progress | Gender | Rating | Favourite Color | Date Of Birth |\n| --- | --- | --- | --- | --- | --- | --- |\n| 1 | Ada Example | 20 | female | 4 | blue | 05/12/2025 |\n| 2 | Bob Example | 70 | male | 3 | green | 22/05/2026 |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "body-reference-excludes-icon-font-name", + "category": "reference-evidence", + "root_cause": "body_reference_nontext_controls", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/virtual-list", + "task_kind": "body", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "basic virtual list displays the initial mounted options only records present after automatic loading belong to this observation scrolling loads additional records outside this scope visible option names and their positions form the body of this example", + "source_text": "Basic virtual list displays the initial mounted options.\n\nOnly records present after automatic loading belong to this observation; scrolling loads additional records outside this scope.\n\nVisible option names and their positions form the body of this example.", + "segmentation_method": "sentences" + } + ] + } + }, + "artifact": "Basic virtual list displays the initial mounted options. Only records present after automatic loading belong to this observation; scrolling loads additional records outside this scope. Visible option names and their positions form the body of this example.", + "ok": true, + "expected_task_pass": true + }, + { + "id": "body-icon-label-is-not-real-body", + "category": "reference-evidence", + "root_cause": "body_reference_nontext_controls", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/virtual-list", + "task_kind": "body", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "basic virtual list displays the initial mounted options only records present after automatic loading belong to this observation scrolling loads additional records outside this scope visible option names and their positions form the body of this example", + "source_text": "Basic virtual list displays the initial mounted options.\n\nOnly records present after automatic loading belong to this observation; scrolling loads additional records outside this scope.\n\nVisible option names and their positions form the body of this example.", + "segmentation_method": "sentences" + } + ] + } + }, + "artifact": "code expand_more content_copy", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scoped-visible-news-title", + "category": "structured-extraction", + "root_cause": "scoped_visible_record_names", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/\\d+$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a" + } + } + }, + "artifact": "<main><a style=\"display: grid; opacity: 1\" href=\"/article/1\">A new headline</a></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "scoped-hidden-news-ancestor", + "category": "structured-extraction", + "root_cause": "scoped_visible_record_names", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/\\d+$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a" + } + } + }, + "artifact": "<main><div style=\"DISPLAY: none !important\"><a href=\"/article/1\">Hidden headline</a></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scoped-hidden-news-own-style", + "category": "structured-extraction", + "root_cause": "scoped_visible_record_names", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/\\d+$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a" + } + } + }, + "artifact": "<main><a style=\"visibility:hidden\" href=\"/article/1\">Hidden headline</a></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scoped-news-image-alt-is-not-title", + "category": "structured-extraction", + "root_cause": "scoped_visible_record_names", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/\\d+$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a" + } + } + }, + "artifact": "<main><h2>Product recommendations</h2><a href=\"/article/1\"><img alt=\"A news headline\" src=\"x.png\"></a></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scoped-news-hidden-copy-not-extra-record", + "category": "structured-extraction", + "root_cause": "scoped_visible_record_names", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/article/\\d+$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a" + } + } + }, + "artifact": "<main><a href=\"/article/1\">Visible headline</a><div style=\"opacity:0\"><a href=\"/article/2\">Hidden headline</a></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "first-news-before-sidebar", + "category": "structured-extraction", + "root_cause": "declared_list_scope_and_nonempty_names", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/\\d+$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a", + "end_pattern": "^[ \\t#*]*产品推荐[ \\t*]*$" + } + } + }, + "artifact": "[New headline](/article/1)\n产品推荐\n[Another](/article/2)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "sidebar-cannot-fill-first-news", + "category": "structured-extraction", + "root_cause": "declared_list_scope_and_nonempty_names", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/\\d+$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a", + "end_pattern": "^[ \\t#*]*产品推荐[ \\t*]*$" + } + } + }, + "artifact": "产品推荐\n[Another](/article/2)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "news-hotissue-route-is-valid", + "category": "structured-extraction", + "root_cause": "observed_article_route_aliases", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/(?:mnews/(?:hotissue/)?)?article/[0-9]+/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a" + }, + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)$" + } + }, + "artifact": "<main><a href=\"/mnews/hotissue/article/015/0005334180\">New headline</a></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "news-extra-route-component-is-invalid", + "category": "structured-extraction", + "root_cause": "observed_article_route_aliases", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/(?:mnews/(?:hotissue/)?)?article/[0-9]+/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a" + }, + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)$" + } + }, + "artifact": "<main><a href=\"/mnews/unrelated/article/015/0005334180\">New headline</a></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "news-alias-is-one-article", + "category": "structured-extraction", + "root_cause": "observed_article_route_aliases", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/(?:mnews/(?:hotissue/)?)?article/[0-9]+/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a" + }, + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)$" + } + }, + "artifact": "<main><a href=\"/article/015/0005334180\">Headline</a><a href=\"/mnews/hotissue/article/015/0005334180\">Headline</a></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "ordinary-label-not-screenreader-caption", + "category": "acceptance-contract", + "root_cause": "screenreader_only_reference", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/[0-9]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "visible-search-label", + "alternatives": [ + "Recherche" + ] + } + ] + } + }, + "artifact": "Recherche\n[Actual headline](/article/1)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "unrelated-assistive-caption-does-not-replace-visible-label", + "category": "acceptance-contract", + "root_cause": "screenreader_only_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/[0-9]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "visible-search-label", + "alternatives": [ + "Recherche" + ] + } + ] + } + }, + "artifact": "Abonnement Newsletter\n[Actual headline](/article/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "visible-label-does-not-replace-news", + "category": "acceptance-contract", + "root_cause": "screenreader_only_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/[0-9]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "visible-search-label", + "alternatives": [ + "Recherche" + ] + } + ] + } + }, + "artifact": "Recherche", + "ok": true, + "expected_task_pass": false + }, + { + "id": "visible-title-with-aria-hidden-still-counts", + "category": "structured-extraction", + "root_cause": "aria_metadata_is_not_visual_visibility", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/\\d+$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a" + } + } + }, + "artifact": "<main><a href=\"/article/1\"><span aria-hidden=\"true\">Actual headline</span></a></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "aria-hidden-title-with-display-none-does-not-count", + "category": "structured-extraction", + "root_cause": "aria_metadata_is_not_visual_visibility", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/news", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/\\d+$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a" + } + } + }, + "artifact": "<main><a href=\"/article/1\"><span aria-hidden=\"true\" style=\"display:none\">Hidden headline</span></a></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "loaded-page-articles-html", + "category": "structured-extraction", + "root_cause": "initial_loaded_page_article_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/article/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a[href]" + }, + "title_exclusion_patterns": [ + "Politique" + ] + } + }, + "artifact": "<header><a href=\"/article/1\">Independent climate dashboard</a></header><main><a href=\"/article/2\">New headline</a></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "loaded-page-articles-markdown", + "category": "structured-extraction", + "root_cause": "initial_loaded_page_article_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/article/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a[href]" + }, + "title_exclusion_patterns": [ + "Politique" + ] + } + }, + "artifact": "[Independent climate dashboard](/article/1)\n[New headline](/article/2)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "loaded-carousel-records-count", + "category": "structured-extraction", + "root_cause": "initial_loaded_page_article_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/article/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a[href]" + }, + "title_exclusion_patterns": [ + "Politique" + ] + } + }, + "artifact": "<main><div style=\"width:20px;overflow:hidden\"><div style=\"width:1000px\"><a href=\"/article/1\">First story</a><a href=\"/article/2\">Second story</a></div></div></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "category-label-is-not-article-name", + "category": "structured-extraction", + "root_cause": "initial_loaded_page_article_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/article/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a[href]" + }, + "title_exclusion_patterns": [ + "Politique" + ] + } + }, + "artifact": "[Politique](/article/1)\n[New headline](/article/2)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "same-article-repeat-cannot-fill-page-count", + "category": "structured-extraction", + "root_cause": "initial_loaded_page_article_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/article/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a[href]" + }, + "title_exclusion_patterns": [ + "Politique" + ] + } + }, + "artifact": "[First story](/article/1)\n[First story](/article/1?from=sidebar)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "page-article-wrong-path-cannot-fill-count", + "category": "structured-extraction", + "root_cause": "initial_loaded_page_article_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/article/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a[href]" + }, + "title_exclusion_patterns": [ + "Politique" + ] + } + }, + "artifact": "[First story](/article/1)\n[Category](/category/2)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "redirected-relative-article-uses-response-origin", + "category": "structured-extraction", + "root_cause": "redirected_document_link_base", + "failure_mode": "false-negative", + "case": { + "url": "https://old.example.test/start", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "new.example.test" + ], + "entry_path": "/article/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a[href]" + } + } + }, + "artifact": "<main><a href=\"/article/1\">Actual headline</a></main>", + "final_url": "https://new.example.test/news/", + "ok": true, + "expected_task_pass": true + }, + { + "id": "wrong-redirect-cannot-borrow-original-origin", + "category": "structured-extraction", + "root_cause": "redirected_document_link_base", + "failure_mode": "false-positive", + "case": { + "url": "https://old.example.test/start", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "old.example.test" + ], + "entry_path": "/article/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a[href]" + } + } + }, + "artifact": "<main><a href=\"/article/1\">Actual headline</a></main>", + "final_url": "https://other.example.test/news/", + "ok": true, + "expected_task_pass": false + }, + { + "id": "first-scroller-new-items-pass", + "category": "acceptance-contract", + "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/scroller", + "task_kind": "body", + "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "virtualscroller renders large amounts of data efficiently", + "source_text": "virtualscroller renders large amounts of data efficiently", + "segmentation_method": "balanced_tokens_fallback" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "Item #\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".demo", + "root_index": 0, + "row_selector": ".items > div", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?=Item #\\d+)", + "end_pattern": "(?m)^PREVIEW$", + "plain_row_pattern": "(?P<item>Item #\\d+)" + } + } + }, + "artifact": "VirtualScroller renders large amounts of data efficiently.\nItem #40\nItem #41", + "ok": true, + "expected_task_pass": true + }, + { + "id": "scroller-controls-not-required-body", + "category": "acceptance-contract", + "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/scroller", + "task_kind": "body", + "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "virtualscroller renders large amounts of data efficiently", + "source_text": "virtualscroller renders large amounts of data efficiently", + "segmentation_method": "balanced_tokens_fallback" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "Item #\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".demo", + "root_index": 0, + "row_selector": ".items > div", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?=Item #\\d+)", + "end_pattern": "(?m)^PREVIEW$", + "plain_row_pattern": "(?P<item>Item #\\d+)" + } + } + }, + "artifact": "VirtualScroller renders large amounts of data efficiently.<div class=\"demo\"><div class=\"items\"><div>Item #40</div><div>Item #41</div></div></div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "other-scroller-cannot-fill-first", + "category": "acceptance-contract", + "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/scroller", + "task_kind": "body", + "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "virtualscroller renders large amounts of data efficiently", + "source_text": "virtualscroller renders large amounts of data efficiently", + "segmentation_method": "balanced_tokens_fallback" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "Item #\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".demo", + "root_index": 0, + "row_selector": ".items > div", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?=Item #\\d+)", + "end_pattern": "(?m)^PREVIEW$", + "plain_row_pattern": "(?P<item>Item #\\d+)" + } + } + }, + "artifact": "VirtualScroller renders large amounts of data efficiently.<div class=\"demo\"><div class=\"items\"><div>Item #40</div></div></div><div class=\"demo\"><div class=\"items\"><div>Item #41</div></div></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scroller-description-alone-fails", + "category": "acceptance-contract", + "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/scroller", + "task_kind": "body", + "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "virtualscroller renders large amounts of data efficiently", + "source_text": "virtualscroller renders large amounts of data efficiently", + "segmentation_method": "balanced_tokens_fallback" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "Item #\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".demo", + "root_index": 0, + "row_selector": ".items > div", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?=Item #\\d+)", + "end_pattern": "(?m)^PREVIEW$", + "plain_row_pattern": "(?P<item>Item #\\d+)" + } + } + }, + "artifact": "VirtualScroller renders large amounts of data efficiently.\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scroller-duplicate-item-fails", + "category": "acceptance-contract", + "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/scroller", + "task_kind": "body", + "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "virtualscroller renders large amounts of data efficiently", + "source_text": "virtualscroller renders large amounts of data efficiently", + "segmentation_method": "balanced_tokens_fallback" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "Item #\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".demo", + "root_index": 0, + "row_selector": ".items > div", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?=Item #\\d+)", + "end_pattern": "(?m)^PREVIEW$", + "plain_row_pattern": "(?P<item>Item #\\d+)" + } + } + }, + "artifact": "VirtualScroller renders large amounts of data efficiently.\nItem #40\nItem #40", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scroller-missing-description-fails", + "category": "acceptance-contract", + "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/scroller", + "task_kind": "body", + "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "virtualscroller renders large amounts of data efficiently", + "source_text": "virtualscroller renders large amounts of data efficiently", + "segmentation_method": "balanced_tokens_fallback" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "Item #\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".demo", + "root_index": 0, + "row_selector": ".items > div", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?=Item #\\d+)", + "end_pattern": "(?m)^PREVIEW$", + "plain_row_pattern": "(?P<item>Item #\\d+)" + } + } + }, + "artifact": "Item #40\nItem #41", + "ok": true, + "expected_task_pass": false + }, + { + "id": "scroller-malformed-label-fails", + "category": "acceptance-contract", + "root_cause": "virtual_list_rows_not_toolbar_or_other_examples", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/scroller", + "task_kind": "body", + "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "virtualscroller renders large amounts of data efficiently", + "source_text": "virtualscroller renders large amounts of data efficiently", + "segmentation_method": "balanced_tokens_fallback" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "Item #\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".demo", + "root_index": 0, + "row_selector": ".items > div", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?=Item #\\d+)", + "end_pattern": "(?m)^PREVIEW$", + "plain_row_pattern": "(?P<item>Item #\\d+)" + } + } + }, + "artifact": "VirtualScroller renders large amounts of data efficiently.<div class=\"demo\"><div class=\"items\"><div>Item #40</div><div>Broken</div></div></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "html-code-example-does-not-change-plain-table-format", + "category": "structured-extraction", + "root_cause": "embedded_examples_are_not_document_format", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/scroller", + "task_kind": "body", + "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "virtualscroller renders large amounts of data efficiently", + "source_text": "virtualscroller renders large amounts of data efficiently", + "segmentation_method": "balanced_tokens_fallback" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "Item #\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".demo", + "root_index": 0, + "row_selector": ".items > div", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?=Item #\\d+)", + "end_pattern": "(?m)^PREVIEW$", + "plain_row_pattern": "(?P<item>Item #\\d+)" + } + } + }, + "artifact": "VirtualScroller renders large amounts of data efficiently.\nItem #40\nItem #41\nPREVIEW\nUsage\n<div class=\"sample-code\">Example markup</div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fenced-example-cannot-fill-live-rows", + "category": "structured-extraction", + "root_cause": "embedded_examples_are_not_document_format", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/scroller", + "task_kind": "body", + "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "virtualscroller renders large amounts of data efficiently", + "source_text": "virtualscroller renders large amounts of data efficiently", + "segmentation_method": "balanced_tokens_fallback" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "Item #\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".demo", + "root_index": 0, + "row_selector": ".items > div", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?=Item #\\d+)", + "end_pattern": "(?m)^PREVIEW$", + "plain_row_pattern": "(?P<item>Item #\\d+)" + } + } + }, + "artifact": "VirtualScroller renders large amounts of data efficiently.\n```html\n<div class=\"demo\"><div class=\"items\"><div>Item #40</div><div>Item #41</div></div></div>\n```", + "ok": true, + "expected_task_pass": false + }, + { + "id": "whole-fenced-data-answer-remains-valid", + "category": "structured-extraction", + "root_cause": "embedded_examples_are_not_document_format", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/scroller", + "task_kind": "body", + "ground_truth_main_content": "VirtualScroller renders large amounts of data efficiently.", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "virtualscroller renders large amounts of data efficiently", + "source_text": "virtualscroller renders large amounts of data efficiently", + "segmentation_method": "balanced_tokens_fallback" + } + ] + }, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "Item #\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".demo", + "root_index": 0, + "row_selector": ".items > div", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?=Item #\\d+)", + "end_pattern": "(?m)^PREVIEW$", + "plain_row_pattern": "(?P<item>Item #\\d+)" + } + } + }, + "artifact": "```text\nVirtualScroller renders large amounts of data efficiently.\nItem #40\nItem #41\n```", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fixed-demo-seven-of-eight-missing-0", + "category": "acceptance-contract", + "root_cause": "short_repeated_demo_body_uses_record_coverage", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/demo", + "task_kind": "body", + "ground_truth_main_content": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7", + "body_requirements": {}, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*[0-7]" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*" + }, + "stable_text_anchors": [ + { + "id": "preview", + "alternatives": [ + "PREVIEW" + ] + } + ] + } + }, + "artifact": "Item #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7\nPREVIEW", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fixed-demo-seven-of-eight-missing-4", + "category": "acceptance-contract", + "root_cause": "short_repeated_demo_body_uses_record_coverage", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/demo", + "task_kind": "body", + "ground_truth_main_content": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7", + "body_requirements": {}, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*[0-7]" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*" + }, + "stable_text_anchors": [ + { + "id": "preview", + "alternatives": [ + "PREVIEW" + ] + } + ] + } + }, + "artifact": "Item #0\nItem #1\nItem #2\nItem #3\nItem #5\nItem #6\nItem #7\nPREVIEW", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fixed-demo-seven-of-eight-missing-7", + "category": "acceptance-contract", + "root_cause": "short_repeated_demo_body_uses_record_coverage", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/demo", + "task_kind": "body", + "ground_truth_main_content": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7", + "body_requirements": {}, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*[0-7]" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*" + }, + "stable_text_anchors": [ + { + "id": "preview", + "alternatives": [ + "PREVIEW" + ] + } + ] + } + }, + "artifact": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nPREVIEW", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fixed-demo-wrong-initial-indices", + "category": "acceptance-contract", + "root_cause": "short_repeated_demo_body_uses_record_coverage", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/demo", + "task_kind": "body", + "ground_truth_main_content": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7", + "body_requirements": {}, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*[0-7]" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*" + }, + "stable_text_anchors": [ + { + "id": "preview", + "alternatives": [ + "PREVIEW" + ] + } + ] + } + }, + "artifact": "Item #40\nItem #41\nItem #42\nItem #43\nItem #44\nItem #45\nItem #46\nItem #47\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "fixed-demo-toolbar-only", + "category": "acceptance-contract", + "root_cause": "short_repeated_demo_body_uses_record_coverage", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/demo", + "task_kind": "body", + "ground_truth_main_content": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7", + "body_requirements": {}, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*[0-7]" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*" + }, + "stable_text_anchors": [ + { + "id": "preview", + "alternatives": [ + "PREVIEW" + ] + } + ] + } + }, + "artifact": "PREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\nPREVIEW\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "fixed-demo-six-is-below-eighty-percent", + "category": "acceptance-contract", + "root_cause": "short_repeated_demo_body_uses_record_coverage", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/demo", + "task_kind": "body", + "ground_truth_main_content": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nItem #6\nItem #7", + "body_requirements": {}, + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*[0-7]" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*" + }, + "stable_text_anchors": [ + { + "id": "preview", + "alternatives": [ + "PREVIEW" + ] + } + ] + } + }, + "artifact": "Item #0\nItem #1\nItem #2\nItem #3\nItem #4\nItem #5\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-token-and-auxiliary-leading-zero-postal", + "category": "structured-extraction", + "root_cause": "typed_table_values_and_auxiliary_text", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" + } + } + }, + "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr><tr><td>Bob Example</td><td>04362-9499</td><td>bob@example.test</td></tr></tbody></table>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-token-and-auxiliary-auxiliary-label", + "category": "structured-extraction", + "root_cause": "typed_table_values_and_auxiliary_text", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" + } + } + }, + "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test<span class=\"screen-reader-only\">opens a new window</span></td></tr><tr><td>Bob Example</td><td>04362-9499</td><td>bob@example.test</td></tr></tbody></table>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-token-and-auxiliary-invalid-postal", + "category": "structured-extraction", + "root_cause": "typed_table_values_and_auxiliary_text", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" + } + } + }, + "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr><tr><td>Bob Example</td><td>bad</td><td>bob@example.test</td></tr></tbody></table>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-token-and-auxiliary-missing-email", + "category": "structured-extraction", + "root_cause": "typed_table_values_and_auxiliary_text", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" + } + } + }, + "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr><tr><td>Bob Example</td><td>04362-9499</td></tr></tbody></table>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-token-and-auxiliary-duplicate-identity", + "category": "structured-extraction", + "root_cause": "typed_table_values_and_auxiliary_text", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" + } + } + }, + "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr></tbody></table>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-token-and-auxiliary-hidden-email-cannot-fill", + "category": "structured-extraction", + "root_cause": "typed_table_values_and_auxiliary_text", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" + } + } + }, + "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr><tr><td>Bob Example</td><td>04362-9499</td><td><span class=\"screen-reader-only\">bob@example.test</span></td></tr></tbody></table>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-token-and-auxiliary-later-table-cannot-fill", + "category": "structured-extraction", + "root_cause": "typed_table_values_and_auxiliary_text", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" + } + } + }, + "artifact": "<table><tbody><tr><td>Ada Sample</td><td>00123</td><td>ada@example.test</td></tr></tbody></table><table><tbody><tr><td>Bob Example</td><td>04362-9499</td><td>bob@example.test</td></tr></tbody></table>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-token-and-auxiliary-changed-realistic-values", + "category": "structured-extraction", + "root_cause": "typed_table_values_and_auxiliary_text", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)" + } + } + }, + "artifact": "<table><tbody><tr><td>Carol New</td><td>12456</td><td>carol@example.test</td></tr><tr><td>Dana New</td><td>00888</td><td>dana@example.test</td></tr></tbody></table>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-multiline-complete", + "category": "structured-extraction", + "root_cause": "multiline_table_row_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)", + "plain_multiline_row_pattern": "(?m)^(?P<name>[^\\d\\n]+)\\n(?P<postal>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "People\nAda Sample\n00123\nada@example.test\nEdit\nBob Example\n04362-9499\nbob@example.test\nEdit", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-multiline-missing-field", + "category": "structured-extraction", + "root_cause": "multiline_table_row_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)", + "plain_multiline_row_pattern": "(?m)^(?P<name>[^\\d\\n]+)\\n(?P<postal>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "People\nAda Sample\n00123\nada@example.test\nEdit\nBob Example\n04362-9499\nEdit", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-multiline-cannot-cross-row", + "category": "structured-extraction", + "root_cause": "multiline_table_row_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)", + "plain_multiline_row_pattern": "(?m)^(?P<name>[^\\d\\n]+)\\n(?P<postal>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "People\nAda Sample\n00123\nEdit\nBob Example\n04362-9499\nbob@example.test\nEdit", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-multiline-other-example", + "category": "structured-extraction", + "root_cause": "multiline_table_row_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)", + "plain_multiline_row_pattern": "(?m)^(?P<name>[^\\d\\n]+)\\n(?P<postal>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "People\nAda Sample\n00123\nada@example.test\nEdit\nNext example\nBob Example\n04362-9499\nbob@example.test\nEdit", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-multiline-mixed-format", + "category": "structured-extraction", + "root_cause": "multiline_table_row_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "postal", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "email" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td", + "exclude_selector": ".screen-reader-only", + "start_pattern": "People\\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<postal>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)", + "plain_multiline_row_pattern": "(?m)^(?P<name>[^\\d\\n]+)\\n(?P<postal>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "People\nAda Sample\n00123\nada@example.test\nEdit\nBob Example 04362-9499 bob@example.test", + "ok": true, + "expected_task_pass": true + }, + { + "id": "rsuite-complete-record-sixteen-complete", + "category": "acceptance-contract", + "root_cause": "volatile_demo_people_not_frozen_body", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 16, + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n1\nNew\nPerson\nfemale\n35\n00123\nperson1@example.test\nEdit\n2\nNew\nPerson\nfemale\n35\n00123\nperson2@example.test\nEdit\n3\nNew\nPerson\nfemale\n35\n00123\nperson3@example.test\nEdit\n4\nNew\nPerson\nfemale\n35\n00123\nperson4@example.test\nEdit\n5\nNew\nPerson\nfemale\n35\n00123\nperson5@example.test\nEdit\n6\nNew\nPerson\nfemale\n35\n00123\nperson6@example.test\nEdit\n7\nNew\nPerson\nfemale\n35\n00123\nperson7@example.test\nEdit\n8\nNew\nPerson\nfemale\n35\n00123\nperson8@example.test\nEdit\n9\nNew\nPerson\nfemale\n35\n00123\nperson9@example.test\nEdit\n10\nNew\nPerson\nfemale\n35\n00123\nperson10@example.test\nEdit\n11\nNew\nPerson\nfemale\n35\n00123\nperson11@example.test\nEdit\n12\nNew\nPerson\nfemale\n35\n00123\nperson12@example.test\nEdit\n13\nNew\nPerson\nfemale\n35\n00123\nperson13@example.test\nEdit\n14\nNew\nPerson\nfemale\n35\n00123\nperson14@example.test\nEdit\n15\nNew\nPerson\nfemale\n35\n00123\nperson15@example.test\nEdit\n16\nNew\nPerson\nfemale\n35\n00123\nperson16@example.test\nEdit\nLoading...", + "ok": true, + "expected_task_pass": true + }, + { + "id": "rsuite-complete-record-fifteen-incomplete", + "category": "acceptance-contract", + "root_cause": "volatile_demo_people_not_frozen_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 16, + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n1\nNew\nPerson\nfemale\n35\n00123\nperson1@example.test\nEdit\n2\nNew\nPerson\nfemale\n35\n00123\nperson2@example.test\nEdit\n3\nNew\nPerson\nfemale\n35\n00123\nperson3@example.test\nEdit\n4\nNew\nPerson\nfemale\n35\n00123\nperson4@example.test\nEdit\n5\nNew\nPerson\nfemale\n35\n00123\nperson5@example.test\nEdit\n6\nNew\nPerson\nfemale\n35\n00123\nperson6@example.test\nEdit\n7\nNew\nPerson\nfemale\n35\n00123\nperson7@example.test\nEdit\n8\nNew\nPerson\nfemale\n35\n00123\nperson8@example.test\nEdit\n9\nNew\nPerson\nfemale\n35\n00123\nperson9@example.test\nEdit\n10\nNew\nPerson\nfemale\n35\n00123\nperson10@example.test\nEdit\n11\nNew\nPerson\nfemale\n35\n00123\nperson11@example.test\nEdit\n12\nNew\nPerson\nfemale\n35\n00123\nperson12@example.test\nEdit\n13\nNew\nPerson\nfemale\n35\n00123\nperson13@example.test\nEdit\n14\nNew\nPerson\nfemale\n35\n00123\nperson14@example.test\nEdit\n15\nNew\nPerson\nfemale\n35\n00123\nperson15@example.test\nEdit\nLoading...", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rsuite-complete-record-missing-email", + "category": "acceptance-contract", + "root_cause": "volatile_demo_people_not_frozen_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 16, + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n1\nNew\nPerson\nfemale\n35\n00123\nperson1@example.test\nEdit\n2\nNew\nPerson\nfemale\n35\n00123\nperson2@example.test\nEdit\n3\nNew\nPerson\nfemale\n35\n00123\nperson3@example.test\nEdit\n4\nNew\nPerson\nfemale\n35\n00123\nperson4@example.test\nEdit\n5\nNew\nPerson\nfemale\n35\n00123\nperson5@example.test\nEdit\n6\nNew\nPerson\nfemale\n35\n00123\nperson6@example.test\nEdit\n7\nNew\nPerson\nfemale\n35\n00123\nperson7@example.test\nEdit\n8\nNew\nPerson\nfemale\n35\n00123\nperson8@example.test\nEdit\n9\nNew\nPerson\nfemale\n35\n00123\nperson9@example.test\nEdit\n10\nNew\nPerson\nfemale\n35\n00123\nperson10@example.test\nEdit\n11\nNew\nPerson\nfemale\n35\n00123\nperson11@example.test\nEdit\n12\nNew\nPerson\nfemale\n35\n00123\nperson12@example.test\nEdit\n13\nNew\nPerson\nfemale\n35\n00123\nperson13@example.test\nEdit\n14\nNew\nPerson\nfemale\n35\n00123\nperson14@example.test\nEdit\n15\nNew\nPerson\nfemale\n35\n00123\nperson15@example.test\nEdit\n16\nNew\nPerson\nfemale\n35\n00123\nEdit\nLoading...", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rsuite-complete-record-duplicate", + "category": "acceptance-contract", + "root_cause": "volatile_demo_people_not_frozen_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 16, + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n1\nNew\nPerson\nfemale\n35\n00123\nperson1@example.test\nEdit\n2\nNew\nPerson\nfemale\n35\n00123\nperson2@example.test\nEdit\n3\nNew\nPerson\nfemale\n35\n00123\nperson3@example.test\nEdit\n4\nNew\nPerson\nfemale\n35\n00123\nperson4@example.test\nEdit\n5\nNew\nPerson\nfemale\n35\n00123\nperson5@example.test\nEdit\n6\nNew\nPerson\nfemale\n35\n00123\nperson6@example.test\nEdit\n7\nNew\nPerson\nfemale\n35\n00123\nperson7@example.test\nEdit\n8\nNew\nPerson\nfemale\n35\n00123\nperson8@example.test\nEdit\n9\nNew\nPerson\nfemale\n35\n00123\nperson9@example.test\nEdit\n10\nNew\nPerson\nfemale\n35\n00123\nperson10@example.test\nEdit\n11\nNew\nPerson\nfemale\n35\n00123\nperson11@example.test\nEdit\n12\nNew\nPerson\nfemale\n35\n00123\nperson12@example.test\nEdit\n13\nNew\nPerson\nfemale\n35\n00123\nperson13@example.test\nEdit\n14\nNew\nPerson\nfemale\n35\n00123\nperson14@example.test\nEdit\n15\nNew\nPerson\nfemale\n35\n00123\nperson15@example.test\nEdit\n1\nNew\nPerson\nfemale\n35\n00123\nperson1@example.test\nEdit\nLoading...", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rsuite-complete-record-invalid-postcode", + "category": "acceptance-contract", + "root_cause": "volatile_demo_people_not_frozen_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 16, + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n1\nNew\nPerson\nfemale\n35\n00123\nperson1@example.test\nEdit\n2\nNew\nPerson\nfemale\n35\n00123\nperson2@example.test\nEdit\n3\nNew\nPerson\nfemale\n35\n00123\nperson3@example.test\nEdit\n4\nNew\nPerson\nfemale\n35\n00123\nperson4@example.test\nEdit\n5\nNew\nPerson\nfemale\n35\n00123\nperson5@example.test\nEdit\n6\nNew\nPerson\nfemale\n35\n00123\nperson6@example.test\nEdit\n7\nNew\nPerson\nfemale\n35\n00123\nperson7@example.test\nEdit\n8\nNew\nPerson\nfemale\n35\n00123\nperson8@example.test\nEdit\n9\nNew\nPerson\nfemale\n35\n00123\nperson9@example.test\nEdit\n10\nNew\nPerson\nfemale\n35\n00123\nperson10@example.test\nEdit\n11\nNew\nPerson\nfemale\n35\n00123\nperson11@example.test\nEdit\n12\nNew\nPerson\nfemale\n35\n00123\nperson12@example.test\nEdit\n13\nNew\nPerson\nfemale\n35\n00123\nperson13@example.test\nEdit\n14\nNew\nPerson\nfemale\n35\n00123\nperson14@example.test\nEdit\n15\nNew\nPerson\nfemale\n35\n00123\nperson15@example.test\nEdit\n16\nNew\nPerson\nfemale\n35\nbad\nperson16@example.test\nEdit\nLoading...", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rsuite-complete-record-markdown", + "category": "acceptance-contract", + "root_cause": "volatile_demo_people_not_frozen_body", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 16, + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "| Id | First Name | Last Name | Gender | Age | Postcode | Email |\n| --- | --- | --- | --- | --- | --- | --- |\n| 1 | New | Person | female | 35 | 00123 | person1@example.test |\n| 2 | New | Person | female | 35 | 00123 | person2@example.test |\n| 3 | New | Person | female | 35 | 00123 | person3@example.test |\n| 4 | New | Person | female | 35 | 00123 | person4@example.test |\n| 5 | New | Person | female | 35 | 00123 | person5@example.test |\n| 6 | New | Person | female | 35 | 00123 | person6@example.test |\n| 7 | New | Person | female | 35 | 00123 | person7@example.test |\n| 8 | New | Person | female | 35 | 00123 | person8@example.test |\n| 9 | New | Person | female | 35 | 00123 | person9@example.test |\n| 10 | New | Person | female | 35 | 00123 | person10@example.test |\n| 11 | New | Person | female | 35 | 00123 | person11@example.test |\n| 12 | New | Person | female | 35 | 00123 | person12@example.test |\n| 13 | New | Person | female | 35 | 00123 | person13@example.test |\n| 14 | New | Person | female | 35 | 00123 | person14@example.test |\n| 15 | New | Person | female | 35 | 00123 | person15@example.test |\n| 16 | New | Person | female | 35 | 00123 | person16@example.test |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "rsuite-complete-record-only-headers", + "category": "acceptance-contract", + "root_cause": "volatile_demo_people_not_frozen_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 16, + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n(?P<first_name>[^\\d\\n]+)\\n(?P<last_name>[^\\d\\n]+)\\n(?P<gender>(?i:female|male))\\n(?P<age>\\d+)\\n(?P<postcode>\\d{5}(?:-\\d{4})?)\\n(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-ordinal-date-real-date", + "category": "structured-extraction", + "root_cause": "calendar_validity_for_table_dates", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "People\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" + } + } + }, + "artifact": "People\nNew Person 23rd Feb 26", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-ordinal-date-leap-date", + "category": "structured-extraction", + "root_cause": "calendar_validity_for_table_dates", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "People\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" + } + } + }, + "artifact": "People\nNew Person 29th Feb 2024", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-ordinal-date-impossible-day", + "category": "structured-extraction", + "root_cause": "calendar_validity_for_table_dates", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "People\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" + } + } + }, + "artifact": "People\nNew Person 31st Feb 26", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-ordinal-date-non-leap-date", + "category": "structured-extraction", + "root_cause": "calendar_validity_for_table_dates", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "People\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" + } + } + }, + "artifact": "People\nNew Person 29th Feb 2025", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-ordinal-date-wrong-ordinal", + "category": "structured-extraction", + "root_cause": "calendar_validity_for_table_dates", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "People\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P<name>[^\\d\\n]+?)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" + } + } + }, + "artifact": "People\nNew Person 11st Jan 26", + "ok": true, + "expected_task_pass": false + }, + { + "id": "eui-complete-record-eight-complete", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" + } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nHelen\nExample\nHelen_account\n23rd Feb 26\nOnline\nShow source", + "ok": true, + "expected_task_pass": true + }, + { + "id": "eui-complete-record-seven-below80", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" + } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nShow source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "eui-complete-record-impossible-date", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" + } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nHelen\nExample\nHelen_account\n31st Feb 26\nOnline\nShow source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "eui-complete-record-missing-status", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" + } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nHelen\nExample\nHelen_account\n23rd Feb 26\nShow source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "eui-complete-record-duplicates", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" + } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nShow source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "eui-complete-record-cannot-borrow-neighbor", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" + } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\n23rd Feb 26\nOnline\nspare_account\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nHelen\nExample\nHelen_account\n23rd Feb 26\nOnline\nShow source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "eui-complete-record-markdown", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" + } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "| First Name | Last Name | Github | Date of Birth | Online |\n| --- | --- | --- | --- | --- |\n| Ada | Example | Ada_account | 23rd Feb 26 | Online |\n| Bob | Example | Bob_account | 23rd Feb 26 | Online |\n| Carol | Example | Carol_account | 23rd Feb 26 | Online |\n| Dana | Example | Dana_account | 23rd Feb 26 | Online |\n| Emma | Example | Emma_account | 23rd Feb 26 | Online |\n| Frank | Example | Frank_account | 23rd Feb 26 | Online |\n| Grace | Example | Grace_account | 23rd Feb 26 | Online |\n| Helen | Example | Helen_account | 23rd Feb 26 | Online |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "eui-complete-record-html", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" + } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "<table class=\"euiTable\"><tbody><tr><td>Ada</td><td>Example</td><td>Ada_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Bob</td><td>Example</td><td>Bob_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Carol</td><td>Example</td><td>Carol_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Dana</td><td>Example</td><td>Dana_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Emma</td><td>Example</td><td>Emma_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Frank</td><td>Example</td><td>Frank_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Grace</td><td>Example</td><td>Grace_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Helen</td><td>Example</td><td>Helen_account</td><td>23rd Feb 26</td><td>Online</td></tr></tbody></table>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "eui-complete-record-later-table", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" + } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P<github>[^\\s↦↵]+)\\s+(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P<online>(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P<first_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<last_name>(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<github>[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<birth>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P<online>(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } + }, + "body_requirements": {} + }, + "artifact": "<table class=\"euiTable\"><tbody><tr><td>Ada</td><td>Example</td><td>Ada_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Bob</td><td>Example</td><td>Bob_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Carol</td><td>Example</td><td>Carol_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Dana</td><td>Example</td><td>Dana_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Emma</td><td>Example</td><td>Emma_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Frank</td><td>Example</td><td>Frank_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Grace</td><td>Example</td><td>Grace_account</td><td>23rd Feb 26</td><td>Online</td></tr></tbody></table><table class=\"euiTable\"><tbody><tr><td>Helen</td><td>Example</td><td>Helen_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>Iris</td><td>Example</td><td>Iris_account</td><td>23rd Feb 26</td><td>Online</td></tr><tr><td>James</td><td>Example</td><td>James_account</td><td>23rd Feb 26</td><td>Online</td></tr></tbody></table>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "fixed-demo-product-complete", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" + } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 Out of Stock\nChakra Bracelet bib36pfvm Accessories $32 Low Stock\nGalaxy Earrings mbvjkgip5 Accessories $34 In Stock\nPREVIEW", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fixed-demo-product-seven", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" + } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 Out of Stock\nChakra Bracelet bib36pfvm Accessories $32 Low Stock\nPREVIEW", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fixed-demo-product-six", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" + } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 Out of Stock\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "fixed-demo-product-missing-prices", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" + } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories In Stock\nBlack Watch nvklal433 Accessories In Stock\nBlue Band zz21cz3c1 Fitness Low Stock\nBlue T-Shirt 244wgerg2 Clothing In Stock\nBracelet h456wer53 Accessories In Stock\nBrown Purse av2231fwg Accessories Out of Stock\nChakra Bracelet bib36pfvm Accessories Low Stock\nGalaxy Earrings mbvjkgip5 Accessories In Stock\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "fixed-demo-product-wrong-prices", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" + } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $99999 In Stock\nBlack Watch nvklal433 Accessories $99999 In Stock\nBlue Band zz21cz3c1 Fitness $99999 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $99999 In Stock\nBracelet h456wer53 Accessories $99999 In Stock\nBrown Purse av2231fwg Accessories $99999 Out of Stock\nChakra Bracelet bib36pfvm Accessories $99999 Low Stock\nGalaxy Earrings mbvjkgip5 Accessories $99999 In Stock\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "fixed-demo-product-wrong-statuses", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" + } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 In Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 In Stock\nChakra Bracelet bib36pfvm Accessories $32 In Stock\nGalaxy Earrings mbvjkgip5 Accessories $34 In Stock\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "fixed-demo-product-equivalent-currency", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" + } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65.00 In Stock\nBlack Watch nvklal433 Accessories $72.00 In Stock\nBlue Band zz21cz3c1 Fitness $79.00 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29.00 In Stock\nBracelet h456wer53 Accessories $15.00 In Stock\nBrown Purse av2231fwg Accessories $120.00 Out of Stock\nChakra Bracelet bib36pfvm Accessories $32.00 Low Stock\nGalaxy Earrings mbvjkgip5 Accessories $34.00 In Stock\nPREVIEW", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fixed-demo-product-markdown-stars", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" + } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "| Product | Category | Price | Rating | Status |\n| --- | --- | --- | --- | --- |\n| Bamboo Watch f230fh0g3 | Accessories | $65 | ★★★ | In Stock |\n| Black Watch nvklal433 | Accessories | $72 | ★★★ | In Stock |\n| Blue Band zz21cz3c1 | Fitness | $79 | ★★★ | Low Stock |\n| Blue T-Shirt 244wgerg2 | Clothing | $29 | ★★★ | In Stock |\n| Bracelet h456wer53 | Accessories | $15 | ★★★ | In Stock |\n| Brown Purse av2231fwg | Accessories | $120 | ★★★ | Out of Stock |\n| Chakra Bracelet bib36pfvm | Accessories | $32 | ★★★ | Low Stock |\n| Galaxy Earrings mbvjkgip5 | Accessories | $34 | ★★★ | In Stock |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fixed-demo-product-duplicates", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" + } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 Out of Stock\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-jp-new-games", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)" + } + } + }, + "artifact": "<div class=\"nc3-c-softCard\"><a href=\"https://store-jp.nintendo.com/item/software/D70010000999991\"><span class=\"nc3-c-softCard__name\">Future Game Alpha</span></a></div><div class=\"nc3-c-softCard\"><a href=\"https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc\"><span class=\"nc3-c-softCard__name\">Future Game Beta</span></a></div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nintendo-scoped-games-jp-markdown", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)" + } + } + }, + "artifact": "[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nintendo-scoped-games-jp-empty-title", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)" + } + } + }, + "artifact": "<div class=\"nc3-c-softCard\"><a href=\"https://store-jp.nintendo.com/item/software/D70010000999991\"><span class=\"nc3-c-softCard__name\"></span></a></div><div class=\"nc3-c-softCard\"><a href=\"https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc\"><span class=\"nc3-c-softCard__name\">Future Game Beta</span></a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-jp-duplicates", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)" + } + } + }, + "artifact": "<div class=\"nc3-c-softCard\"><a href=\"https://store-jp.nintendo.com/item/software/D70010000999991\"><span class=\"nc3-c-softCard__name\">Future Game Alpha</span></a></div><div class=\"nc3-c-softCard\"><a href=\"https://store-jp.nintendo.com/item/software/D70010000999991\"><span class=\"nc3-c-softCard__name\">Future Game Alpha</span></a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-jp-navigation-only", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)" + } + } + }, + "artifact": "<nav><a href=\"https://store-jp.nintendo.com/item/software/D70010000999991\">Future Game Alpha</a><a href=\"https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc\">Future Game Beta</a></nav>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-jp-unrelated-path", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)" + } + } + }, + "artifact": "<div class=\"nc3-c-softCard\"><a href=\"https://www.nintendo.com/help\"><span class=\"nc3-c-softCard__name\">Help</span></a></div><div class=\"nc3-c-softCard\"><a href=\"https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc\"><span class=\"nc3-c-softCard__name\">Future Game Beta</span></a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-es-new-games", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com" + ], + "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".searchresult_row", + "title_selector": ".page-title-text", + "link_selector": "a[href]:has(.page-title-text)" + } + } + }, + "artifact": "<div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html\"><span class=\"page-title-text\">Future Game Alpha</span></a></div><div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Juegos-de-dispositivo-inteligente/Another-Game-999992.html\"><span class=\"page-title-text\">Future Game Beta</span></a></div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nintendo-scoped-games-es-markdown", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com" + ], + "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".searchresult_row", + "title_selector": ".page-title-text", + "link_selector": "a[href]:has(.page-title-text)" + } + } + }, + "artifact": "[Future Game Alpha](https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html)\n[Future Game Beta](https://www.nintendo.com/es-es/Juegos/Juegos-de-dispositivo-inteligente/Another-Game-999992.html)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nintendo-scoped-games-es-empty-title", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com" + ], + "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".searchresult_row", + "title_selector": ".page-title-text", + "link_selector": "a[href]:has(.page-title-text)" + } + } + }, + "artifact": "<div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html\"><span class=\"page-title-text\"></span></a></div><div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Juegos-de-dispositivo-inteligente/Another-Game-999992.html\"><span class=\"page-title-text\">Future Game Beta</span></a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-es-duplicates", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com" + ], + "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".searchresult_row", + "title_selector": ".page-title-text", + "link_selector": "a[href]:has(.page-title-text)" + } + } + }, + "artifact": "<div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html\"><span class=\"page-title-text\">Future Game Alpha</span></a></div><div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html\"><span class=\"page-title-text\">Future Game Alpha</span></a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-es-navigation-only", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com" + ], + "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".searchresult_row", + "title_selector": ".page-title-text", + "link_selector": "a[href]:has(.page-title-text)" + } + } + }, + "artifact": "<nav><a href=\"https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html\">Future Game Alpha</a><a href=\"https://www.nintendo.com/es-es/Juegos/Juegos-de-dispositivo-inteligente/Another-Game-999992.html\">Future Game Beta</a></nav>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-es-unrelated-path", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com" + ], + "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".searchresult_row", + "title_selector": ".page-title-text", + "link_selector": "a[href]:has(.page-title-text)" + } + } + }, + "artifact": "<div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/help\"><span class=\"page-title-text\">Help</span></a></div><div class=\"searchresult_row\"><a href=\"https://www.nintendo.com/es-es/Juegos/Juegos-de-dispositivo-inteligente/Another-Game-999992.html\"><span class=\"page-title-text\">Future Game Beta</span></a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "forbidden-html-surface-dormant-template", + "category": "answer-surface", + "root_cause": "dormant_html_templates_not_blocking_notices", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected": [ + "Game catalog" + ], + "forbidden_any": [ + "CAPTCHA", + "Access denied" + ], + "min_content_chars": 1 + }, + "artifact": "<html><body><h1>Game catalog</h1><p>Actual game list</p><script type=\"text/x-template\">CAPTCHA</script></body></html>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "forbidden-html-surface-visible-challenge", + "category": "answer-surface", + "root_cause": "dormant_html_templates_not_blocking_notices", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected": [ + "Game catalog" + ], + "forbidden_any": [ + "CAPTCHA", + "Access denied" + ], + "min_content_chars": 1 + }, + "artifact": "<html><body><h1>Game catalog</h1><p>CAPTCHA required</p></body></html>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "forbidden-html-surface-explicitly-hidden", + "category": "answer-surface", + "root_cause": "dormant_html_templates_not_blocking_notices", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected": [ + "Game catalog" + ], + "forbidden_any": [ + "CAPTCHA", + "Access denied" + ], + "min_content_chars": 1 + }, + "artifact": "<main>Game catalog<div hidden>Access denied</div></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "forbidden-html-surface-actual-denial", + "category": "answer-surface", + "root_cause": "dormant_html_templates_not_blocking_notices", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected": [ + "Game catalog" + ], + "forbidden_any": [ + "CAPTCHA", + "Access denied" + ], + "min_content_chars": 1 + }, + "artifact": "<main>Game catalog<div>Access denied</div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "forbidden-html-surface-plain-challenge", + "category": "answer-surface", + "root_cause": "dormant_html_templates_not_blocking_notices", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected": [ + "Game catalog" + ], + "forbidden_any": [ + "CAPTCHA", + "Access denied" + ], + "min_content_chars": 1 + }, + "artifact": "Game catalog\nCAPTCHA required", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-markdown-section-actual-game-section", + "category": "structured-extraction", + "root_cause": "game_card_announcements_not_navigation_news", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)", + "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", + "end_pattern": "^[ \\t#*]*ゲームを楽しむ" + } + } + }, + "artifact": "## ピックアップ\n[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)\n### ゲームを楽しむ", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nintendo-markdown-section-navigation-news-only", + "category": "structured-extraction", + "root_cause": "game_card_announcements_not_navigation_news", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)", + "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", + "end_pattern": "^[ \\t#*]*ゲームを楽しむ" + } + } + }, + "artifact": "[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)\n## ピックアップ\nNo game records\n### ゲームを楽しむ", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-markdown-section-links-after-game-section", + "category": "structured-extraction", + "root_cause": "game_card_announcements_not_navigation_news", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)", + "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", + "end_pattern": "^[ \\t#*]*ゲームを楽しむ" + } + } + }, + "artifact": "## ピックアップ\nNo game records\n### ゲームを楽しむ\n[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-markdown-section-bullet-heading", + "category": "structured-extraction", + "root_cause": "game_card_announcements_not_navigation_news", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)", + "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", + "end_pattern": "^[ \\t#*]*ゲームを楽しむ" + } + } + }, + "artifact": "* ピックアップ\n[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-026-bilibili-rank-markdown-scope-inside", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual Name 0](https://www.bilibili.com/video/BV1Ab4111111)\n[Actual Name 1](https://www.bilibili.com/video/BV1Ab4222222)\nbilibili", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-026-bilibili-rank-markdown-scope-navigation-only", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Actual Name 0](https://www.bilibili.com/video/BV1Ab4111111)\n[Actual Name 1](https://www.bilibili.com/video/BV1Ab4222222)\n排行榜根据稿件内容质量，近期的数据综合展示，动态更新\nNo target records\nbilibili", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-026-bilibili-rank-markdown-scope-footer-only", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\nNo target records\nbilibili\n[Actual Name 0](https://www.bilibili.com/video/BV1Ab4111111)\n[Actual Name 1](https://www.bilibili.com/video/BV1Ab4222222)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-048-en-github-trending-developers-markdown-scope-inside", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://github.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!login$|signup$|features$|topics$|trending$)[A-Za-z0-9_.-]+", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "article.Box-row h1", + "start_pattern": "These are the developers building the hot tools today\\.", + "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "These are the developers building the hot tools today.\n[Actual Name 0](https://github.com/newalpha)\n[Actual Name 1](https://github.com/newbeta)\nFooter", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-048-en-github-trending-developers-markdown-scope-navigation-only", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://github.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!login$|signup$|features$|topics$|trending$)[A-Za-z0-9_.-]+", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "article.Box-row h1", + "start_pattern": "These are the developers building the hot tools today\\.", + "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Actual Name 0](https://github.com/newalpha)\n[Actual Name 1](https://github.com/newbeta)\nThese are the developers building the hot tools today.\nNo target records\nFooter", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-048-en-github-trending-developers-markdown-scope-footer-only", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://github.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!login$|signup$|features$|topics$|trending$)[A-Za-z0-9_.-]+", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "article.Box-row h1", + "start_pattern": "These are the developers building the hot tools today\\.", + "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "These are the developers building the hot tools today.\nNo target records\nFooter\n[Actual Name 0](https://github.com/newalpha)\n[Actual Name 1](https://github.com/newbeta)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-044-en-real-github-trending-markdown-scope-inside", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://github.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!trending(?:/|$)|apps/|sponsors/|topics/|collections/|settings/|features/|marketplace/|orgs/|users/|login/|signup/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "article.Box-row h2", + "start_pattern": "See what the GitHub community is most excited about today\\.", + "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "See what the GitHub community is most excited about today.\n[Actual Name 0](https://github.com/newalpha/project)\n[Actual Name 1](https://github.com/newbeta/project)\nFooter", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-044-en-real-github-trending-markdown-scope-navigation-only", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://github.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!trending(?:/|$)|apps/|sponsors/|topics/|collections/|settings/|features/|marketplace/|orgs/|users/|login/|signup/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "article.Box-row h2", + "start_pattern": "See what the GitHub community is most excited about today\\.", + "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Actual Name 0](https://github.com/newalpha/project)\n[Actual Name 1](https://github.com/newbeta/project)\nSee what the GitHub community is most excited about today.\nNo target records\nFooter", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-044-en-real-github-trending-markdown-scope-footer-only", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://github.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!trending(?:/|$)|apps/|sponsors/|topics/|collections/|settings/|features/|marketplace/|orgs/|users/|login/|signup/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "article.Box-row h2", + "start_pattern": "See what the GitHub community is most excited about today\\.", + "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "See what the GitHub community is most excited about today.\nNo target records\nFooter\n[Actual Name 0](https://github.com/newalpha/project)\n[Actual Name 1](https://github.com/newbeta/project)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "auxiliary-video-links-two-main-plus-popup", + "category": "structured-extraction", + "root_cause": "hidden_related_video_popup_not_main_rank", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Main Alpha](https://www.bilibili.com/video/BV1Ab4111111)\n123 views\n[Other One](https://www.bilibili.com/video/BV1Ab4333333)\n[Other Two](https://www.bilibili.com/video/BV1Ab4444444)\n全部上榜视频\n[Main Beta](https://www.bilibili.com/video/BV1Ab4222222)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "auxiliary-video-links-popup-cannot-fill", + "category": "structured-extraction", + "root_cause": "hidden_related_video_popup_not_main_rank", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Main Alpha](https://www.bilibili.com/video/BV1Ab4111111)\n123 views\n[Other One](https://www.bilibili.com/video/BV1Ab4333333)\n[Other Two](https://www.bilibili.com/video/BV1Ab4444444)\n全部上榜视频", + "ok": true, + "expected_task_pass": false + }, + { + "id": "auxiliary-video-links-normal-two-main", + "category": "structured-extraction", + "root_cause": "hidden_related_video_popup_not_main_rank", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Main Alpha](https://www.bilibili.com/video/BV1Ab4111111)\n[Main Beta](https://www.bilibili.com/video/BV1Ab4222222)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "markdown-title-metadata-inner-double", + "category": "format-normalization", + "root_cause": "quoted_title_metadata_not_link_destination", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual \"Name\"](https://www.bilibili.com/video/BV1Ab4111111 \"Actual \"Name\"\")", + "ok": true, + "expected_task_pass": true + }, + { + "id": "markdown-title-metadata-inner-apostrophe", + "category": "format-normalization", + "root_cause": "quoted_title_metadata_not_link_destination", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual Name](https://www.bilibili.com/video/BV1Ab4111111 \"User's title\")", + "ok": true, + "expected_task_pass": true + }, + { + "id": "markdown-title-metadata-wrong-path", + "category": "format-normalization", + "root_cause": "quoted_title_metadata_not_link_destination", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual Name](https://www.bilibili.com/help \"Actual \"Name\"\")", + "ok": true, + "expected_task_pass": false + }, + { + "id": "markdown-title-metadata-bare-url", + "category": "format-normalization", + "root_cause": "quoted_title_metadata_not_link_destination", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\nActual Name https://www.bilibili.com/video/BV1Ab4111111", + "ok": true, + "expected_task_pass": false + }, + { + "id": "adjacent-song-pair-escaped-markdown-complete", + "category": "structured-extraction", + "root_cause": "adjacent_markdown_links_same_song_identity", + "failure_mode": "false-negative", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "[곡정보](/song/detail.htm?songId=101)\n\n[새 노래 Alpha](javascript:melon.play.playSong\\('1000002721',101\\);)\n\n[곡정보](/song/detail.htm?songId=102)\n\n[새 노래 Beta](javascript:melon.play.playSong\\('1000002721',102\\);)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "adjacent-song-pair-mismatched-identity", + "category": "structured-extraction", + "root_cause": "adjacent_markdown_links_same_song_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "[곡정보](/song/detail.htm?songId=101)\n\n[새 노래 Alpha](javascript:melon.play.playSong\\('1000002721',101\\);)\n\n[곡정보](/song/detail.htm?songId=999)\n\n[새 노래 Beta](javascript:melon.play.playSong\\('1000002721',102\\);)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "same-row-song-pair-complete", + "category": "structured-extraction", + "root_cause": "visible_song_name_and_detail_link_same_row_identity", + "failure_mode": "false-negative", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "title_selector": ".rank01 a", + "link_selector": "a.song_info[href]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=102) | [새 노래 Beta](javascript:melon.play.playSong%28'1000002721',102%29;) |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "same-row-song-pair-mismatched-id", + "category": "structured-extraction", + "root_cause": "visible_song_name_and_detail_link_same_row_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "title_selector": ".rank01 a", + "link_selector": "a.song_info[href]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=999) | [새 노래 Beta](javascript:melon.play.playSong%28'1000002721',102%29;) |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "same-row-song-pair-cross-row", + "category": "structured-extraction", + "root_cause": "visible_song_name_and_detail_link_same_row_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "title_selector": ".rank01 a", + "link_selector": "a.song_info[href]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=102) |\n| [새 노래 Beta](javascript:melon.play.playSong%28'1000002721',102%29;) |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "same-row-song-pair-duplicates", + "category": "structured-extraction", + "root_cause": "visible_song_name_and_detail_link_same_row_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "title_selector": ".rank01 a", + "link_selector": "a.song_info[href]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "same-row-song-pair-empty-name", + "category": "structured-extraction", + "root_cause": "visible_song_name_and_detail_link_same_row_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "title_selector": ".rank01 a", + "link_selector": "a.song_info[href]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=102) | [](javascript:melon.play.playSong%28'1000002721',102%29;) |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "same-row-song-pair-direct-named-links", + "category": "structured-extraction", + "root_cause": "visible_song_name_and_detail_link_same_row_identity", + "failure_mode": "false-negative", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "title_selector": ".rank01 a", + "link_selector": "a.song_info[href]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "[새 노래 Alpha](/song/detail.htm?songId=101)\n[새 노래 Beta](/song/detail.htm?songId=102)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-052-bilibili-com-v-footer-scope-inside", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/v/popular/all/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".video-card", + "title_selector": "p.video-name", + "link_selector": "a[href*=\"/video/\"]", + "start_pattern": "各个领域中新奇好玩的优质内容都在这里", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "各个领域中新奇好玩的优质内容都在这里~\n[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)\nbilibili", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-052-bilibili-com-v-footer-scope-footer-only", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/v/popular/all/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".video-card", + "title_selector": "p.video-name", + "link_selector": "a[href*=\"/video/\"]", + "start_pattern": "各个领域中新奇好玩的优质内容都在这里", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "各个领域中新奇好玩的优质内容都在这里~\nbilibili\n[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-052-bilibili-com-v-footer-scope-navigation-only", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/v/popular/all/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".video-card", + "title_selector": "p.video-name", + "link_selector": "a[href*=\"/video/\"]", + "start_pattern": "各个领域中新奇好玩的优质内容都在这里", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)\n各个领域中新奇好玩的优质内容都在这里~\nbilibili", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-052-bilibili-com-v-footer-scope-no-footer", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/v/popular/all/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".video-card", + "title_selector": "p.video-name", + "link_selector": "a[href*=\"/video/\"]", + "start_pattern": "各个领域中新奇好玩的优质内容都在这里", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "各个领域中新奇好玩的优质内容都在这里~\n[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-063-s-weibo-com-top-footer-scope-inside", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-negative", + "case": { + "url": "https://s.weibo.com/top/summary", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "s.weibo.com" + ], + "entry_path": "/weibo", + "stable_text_anchors": [], + "identity_query_keys": [ + "q" + ], + "entry_scope": { + "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", + "start_pattern": "序号[^\\n]*关键词", + "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" + }, + "name_validation": "nonempty" + } + }, + "artifact": "序号 关键词\n[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)\n关于微博 微博帮助", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-063-s-weibo-com-top-footer-scope-footer-only", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-positive", + "case": { + "url": "https://s.weibo.com/top/summary", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "s.weibo.com" + ], + "entry_path": "/weibo", + "stable_text_anchors": [], + "identity_query_keys": [ + "q" + ], + "entry_scope": { + "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", + "start_pattern": "序号[^\\n]*关键词", + "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" + }, + "name_validation": "nonempty" + } + }, + "artifact": "序号 关键词\n关于微博 微博帮助\n[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-063-s-weibo-com-top-footer-scope-navigation-only", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-positive", + "case": { + "url": "https://s.weibo.com/top/summary", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "s.weibo.com" + ], + "entry_path": "/weibo", + "stable_text_anchors": [], + "identity_query_keys": [ + "q" + ], + "entry_scope": { + "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", + "start_pattern": "序号[^\\n]*关键词", + "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)\n序号 关键词\n关于微博 微博帮助", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-063-s-weibo-com-top-footer-scope-no-footer", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-negative", + "case": { + "url": "https://s.weibo.com/top/summary", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "s.weibo.com" + ], + "entry_path": "/weibo", + "stable_text_anchors": [], + "identity_query_keys": [ + "q" + ], + "entry_scope": { + "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", + "start_pattern": "序号[^\\n]*关键词", + "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" + }, + "name_validation": "nonempty" + } + }, + "artifact": "序号 关键词\n[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "arabic-currency-visible-format-prefix", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-negative", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم 659.99\nدرهم 659.99\nدرهم 659.99\nدرهم 659.99\nدرهم 659.99\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "arabic-currency-visible-format-suffix", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-negative", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\n659.99 درهم\n659.99 درهم\n659.99 درهم\n659.99 درهم\n659.99 درهم\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "arabic-currency-visible-format-direction-marks", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-negative", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\n‏659.99 درهم‎\n‏659.99 درهم‎\n‏659.99 درهم‎\n‏659.99 درهم‎\n‏659.99 درهم‎\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "arabic-currency-visible-format-prefix-direction", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-negative", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم‎659.99‏\nدرهم‎659.99‏\nدرهم‎659.99‏\nدرهم‎659.99‏\nدرهم‎659.99‏\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "arabic-currency-visible-format-four-only", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\n659.99 درهم\n659.99 درهم\n659.99 درهم\n659.99 درهم\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "arabic-currency-visible-format-unrelated-digits", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "arabic-currency-visible-format-currency-only", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم\nدرهم\nدرهم\nدرهم\nدرهم\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-014-en-stable-rsuite-table-blank-lines-complete", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://rsuitejs.com/components/table/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n[ \\t\\n]*)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*(?:\\[.*)?$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n[ \\t\\n]*(?P<first_name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<last_name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<gender>(?i:female|male))\\n[ \\t\\n]*(?P<age>\\d+)\\n[ \\t\\n]*(?P<postcode>\\d{5}(?:-\\d{4})?)\\n[ \\t\\n]*(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\n[ \\t\\n]*Edit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "Id\n\nFirst Name\n\nLast Name\n\nGender\n\nAge\n\nPostcode\n\nEmail\n\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nVan.Tromp@hotmail.com\n\nEdit\n\n2\n\nMarcella\n\nProhaska\n\nfemale\n\n23\n\n81933-1082\n\nGeo18@yahoo.com\n\nEdit\n\nLoading...", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-014-en-stable-rsuite-table-blank-lines-missing-field", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://rsuitejs.com/components/table/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n[ \\t\\n]*)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*(?:\\[.*)?$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n[ \\t\\n]*(?P<first_name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<last_name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<gender>(?i:female|male))\\n[ \\t\\n]*(?P<age>\\d+)\\n[ \\t\\n]*(?P<postcode>\\d{5}(?:-\\d{4})?)\\n[ \\t\\n]*(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\n[ \\t\\n]*Edit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "Id\n\nFirst Name\n\nLast Name\n\nGender\n\nAge\n\nPostcode\n\nEmail\n\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nEdit\n\n2\n\nMarcella\n\nProhaska\n\nfemale\n\n23\n\n81933-1082\n\nGeo18@yahoo.com\n\nEdit\n\nLoading...", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-014-en-stable-rsuite-table-blank-lines-duplicated", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://rsuitejs.com/components/table/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n[ \\t\\n]*)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*(?:\\[.*)?$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n[ \\t\\n]*(?P<first_name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<last_name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<gender>(?i:female|male))\\n[ \\t\\n]*(?P<age>\\d+)\\n[ \\t\\n]*(?P<postcode>\\d{5}(?:-\\d{4})?)\\n[ \\t\\n]*(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\n[ \\t\\n]*Edit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "Id\n\nFirst Name\n\nLast Name\n\nGender\n\nAge\n\nPostcode\n\nEmail\n\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nVan.Tromp@hotmail.com\n\nEdit\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nVan.Tromp@hotmail.com\n\nEdit\n\nLoading...", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-014-en-stable-rsuite-table-blank-lines-one-record", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://rsuitejs.com/components/table/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n[ \\t\\n]*)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*(?:\\[.*)?$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first_name>[^\\d\\n]+?)\\s+(?P<last_name>[^\\d\\n]+?)\\s+(?P<gender>(?i:female|male))\\s+(?P<age>\\d+)\\s+(?P<postcode>\\d{5}(?:-\\d{4})?)\\s+(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n[ \\t\\n]*(?P<first_name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<last_name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<gender>(?i:female|male))\\n[ \\t\\n]*(?P<age>\\d+)\\n[ \\t\\n]*(?P<postcode>\\d{5}(?:-\\d{4})?)\\n[ \\t\\n]*(?P<email>[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\n[ \\t\\n]*Edit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "Id\n\nFirst Name\n\nLast Name\n\nGender\n\nAge\n\nPostcode\n\nEmail\n\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nVan.Tromp@hotmail.com\n\nEdit\n\nLoading...", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-025-025-tabulator-vertical-virtual-dom-blank-lines-complete", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://www.tabulator.info/examples/6.x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n[ \\t\\n]*(?P<name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<progress>\\d{1,3})\\n[ \\t\\n]*(?P<gender>female|male)\\n[ \\t\\n]*(?P<rating>\\d)\\n[ \\t\\n]*(?P<color>[^\\d\\n]+)\\n[ \\t\\n]*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" + } + } + }, + "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\n2\n\nMrs. Monte Pouros\n\n53\n\nfemale\n\n4\n\nlavender\n\n24/03/2026\n\nView Source", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-025-025-tabulator-vertical-virtual-dom-blank-lines-missing-field", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://www.tabulator.info/examples/6.x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n[ \\t\\n]*(?P<name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<progress>\\d{1,3})\\n[ \\t\\n]*(?P<gender>female|male)\\n[ \\t\\n]*(?P<rating>\\d)\\n[ \\t\\n]*(?P<color>[^\\d\\n]+)\\n[ \\t\\n]*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" + } + } + }, + "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n2\n\nMrs. Monte Pouros\n\n53\n\nfemale\n\n4\n\nlavender\n\n24/03/2026\n\nView Source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-025-025-tabulator-vertical-virtual-dom-blank-lines-duplicated", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://www.tabulator.info/examples/6.x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n[ \\t\\n]*(?P<name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<progress>\\d{1,3})\\n[ \\t\\n]*(?P<gender>female|male)\\n[ \\t\\n]*(?P<rating>\\d)\\n[ \\t\\n]*(?P<color>[^\\d\\n]+)\\n[ \\t\\n]*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" + } + } + }, + "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\nView Source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-025-025-tabulator-vertical-virtual-dom-blank-lines-one-record", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://www.tabulator.info/examples/6.x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n[ \\t\\n]*(?P<name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<progress>\\d{1,3})\\n[ \\t\\n]*(?P<gender>female|male)\\n[ \\t\\n]*(?P<rating>\\d)\\n[ \\t\\n]*(?P<color>[^\\d\\n]+)\\n[ \\t\\n]*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" + } + } + }, + "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\nView Source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-loading-label-outside-target-not-global-failure", + "category": "structured-extraction", + "root_cause": "unrelated_example_loading_not_target_failure", + "failure_mode": "false-negative", + "case": { + "url": "https://www.tabulator.info/examples/6.x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n[ \\t\\n]*(?P<name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<progress>\\d{1,3})\\n[ \\t\\n]*(?P<gender>female|male)\\n[ \\t\\n]*(?P<rating>\\d)\\n[ \\t\\n]*(?P<color>[^\\d\\n]+)\\n[ \\t\\n]*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" + } + }, + "forbidden_any": [ + "Access denied", + "403 Forbidden", + "Verify you are human", + "Checking your browser", + "CAPTCHA", + "Just a moment" + ] + }, + "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\n2\n\nMrs. Monte Pouros\n\n53\n\nfemale\n\n4\n\nlavender\n\n24/03/2026\n\nView Source\nOther example\nLoading Example...", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-target-loading-without-records-rejected", + "category": "structured-extraction", + "root_cause": "unrelated_example_loading_not_target_failure", + "failure_mode": "false-positive", + "case": { + "url": "https://www.tabulator.info/examples/6.x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" + } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P<id>\\d+)\\s*(?P<name>[^\\d\\n]+?)\\s*(?P<progress>\\d{1,3})\\s*(?P<gender>female|male)\\s*(?P<rating>\\d)\\s*(?P<color>[^\\d\\n]+?)\\s*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})", + "plain_multiline_row_pattern": "(?m)^(?P<id>\\d+)\\n[ \\t\\n]*(?P<name>[^\\d\\n]+)\\n[ \\t\\n]*(?P<progress>\\d{1,3})\\n[ \\t\\n]*(?P<gender>female|male)\\n[ \\t\\n]*(?P<rating>\\d)\\n[ \\t\\n]*(?P<color>[^\\d\\n]+)\\n[ \\t\\n]*(?P<birth_date>\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" + } + }, + "forbidden_any": [ + "Access denied", + "403 Forbidden", + "Verify you are human", + "Checking your browser", + "CAPTCHA", + "Just a moment" + ] + }, + "artifact": "The table below has 1000 rows\nLoading Example...\nView Source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "heading-only-body-region-image-between-headings", + "category": "body-scope", + "root_cause": "separate_headings_not_contiguous_body_fragment", + "failure_mode": "false-negative", + "case": { + "url": "https://nutui.jd.com/h5/vue/4x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "nutui-independent-heading-0", + "alternatives": [ + "扫码体验" + ], + "source_location": "main heading", + "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" + }, + { + "id": "nutui-independent-heading-1", + "alternatives": [ + "链接" + ], + "source_location": "main heading", + "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" + } + ] + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 4, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "nutui nutui vue 组 件 库 适 用 于 vue 技 术 栈 的 h5 应 用 开 箱 即 用 帮 助 研 发 快 速 开 发 用 户 界 面 提 升 开 发 效 率 改 善 开 发 体 验", + "source_text": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。", + "segmentation_method": "sections" + }, + { + "id": "body-region-2", + "text": "特 性 80 高 质 量 组 件 覆 盖 移 动 端 主 流 场 景 基 于 京 东 app 10 0 视 觉 规 范 支 持 按 需 引 用 详 尽 的 文 档 和 示 例 支 持 typescript 支 持 webstorm vscode 组 件 属 性 高 亮 支 持 动 态 定 制 主 题 支 持 暗 黑 模 式 支 持 国 际 化 单 元 测 试 覆 盖 率 超 过 80 保 障 稳 定 性 提 供 sketch 设 计 资 源", + "source_text": "特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源", + "segmentation_method": "sections" + }, + { + "id": "body-region-4", + "text": "官 方 生 态 由 nutui 官 方 团 队 维 护 的 项 目 如 下 项 目 描 述 nutui nutui nutui 移 动 端 h5 版 本 nutui nutui taro nutui taro 多 端 版 本 nutui icons vue nutui nutui 使 用 的 图 标 库 nutui icons vue taro nutui nutui taro 使 用 的 图 标 库 nutui touch emulator 在 桌 面 端 使 用 nutui 的 辅 助 库 nutui playground nutui 在 线 playground nutui demo nutui 官 方 示 例 合 集", + "source_text": "官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集", + "segmentation_method": "sections" + }, + { + "id": "body-region-5", + "text": "社 区 生 态 由 社 区 维 护 的 项 目 如 下 欢 迎 补 充 项 目 描 述 yang1206 uniapp nutui 基 于 nutui 4 x 版 本 开 发 的 uni app 组 件 库 jwaterwater uni nutui 基 于 nutui 3 x 版 本 开 发 的 uni app 组 件 库 贡 献 核 心 人 员 以 下 是 nutui 的 核 心 贡 献 者 们 richard1015 drjingfubo love forever yangxiaolu1993 snandy eiinu ymm ailululu lkjh3214 yangjinjun3 贡 献 者 们 感 谢 以 下 小 伙 伴 们 为 nutui 发 展 做 出 的 贡 献 共 建 指 南 贡 献 指 南 pr 开 发 者 开 源 协 议 mit", + "source_text": "社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT", + "segmentation_method": "sections" + } + ] + } + }, + "artifact": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。\n\n特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源\n\n官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集\n\n社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT\n\n## 扫码体验\n\n![NutUI](https://example.org/qr.png)\n\n## 链接", + "ok": true, + "expected_task_pass": true + }, + { + "id": "heading-only-body-region-missing-heading", + "category": "body-scope", + "root_cause": "separate_headings_not_contiguous_body_fragment", + "failure_mode": "false-positive", + "case": { + "url": "https://nutui.jd.com/h5/vue/4x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "nutui-independent-heading-0", + "alternatives": [ + "扫码体验" + ], + "source_location": "main heading", + "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" + }, + { + "id": "nutui-independent-heading-1", + "alternatives": [ + "链接" + ], + "source_location": "main heading", + "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" + } + ] + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 4, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "nutui nutui vue 组 件 库 适 用 于 vue 技 术 栈 的 h5 应 用 开 箱 即 用 帮 助 研 发 快 速 开 发 用 户 界 面 提 升 开 发 效 率 改 善 开 发 体 验", + "source_text": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。", + "segmentation_method": "sections" + }, + { + "id": "body-region-2", + "text": "特 性 80 高 质 量 组 件 覆 盖 移 动 端 主 流 场 景 基 于 京 东 app 10 0 视 觉 规 范 支 持 按 需 引 用 详 尽 的 文 档 和 示 例 支 持 typescript 支 持 webstorm vscode 组 件 属 性 高 亮 支 持 动 态 定 制 主 题 支 持 暗 黑 模 式 支 持 国 际 化 单 元 测 试 覆 盖 率 超 过 80 保 障 稳 定 性 提 供 sketch 设 计 资 源", + "source_text": "特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源", + "segmentation_method": "sections" + }, + { + "id": "body-region-4", + "text": "官 方 生 态 由 nutui 官 方 团 队 维 护 的 项 目 如 下 项 目 描 述 nutui nutui nutui 移 动 端 h5 版 本 nutui nutui taro nutui taro 多 端 版 本 nutui icons vue nutui nutui 使 用 的 图 标 库 nutui icons vue taro nutui nutui taro 使 用 的 图 标 库 nutui touch emulator 在 桌 面 端 使 用 nutui 的 辅 助 库 nutui playground nutui 在 线 playground nutui demo nutui 官 方 示 例 合 集", + "source_text": "官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集", + "segmentation_method": "sections" + }, + { + "id": "body-region-5", + "text": "社 区 生 态 由 社 区 维 护 的 项 目 如 下 欢 迎 补 充 项 目 描 述 yang1206 uniapp nutui 基 于 nutui 4 x 版 本 开 发 的 uni app 组 件 库 jwaterwater uni nutui 基 于 nutui 3 x 版 本 开 发 的 uni app 组 件 库 贡 献 核 心 人 员 以 下 是 nutui 的 核 心 贡 献 者 们 richard1015 drjingfubo love forever yangxiaolu1993 snandy eiinu ymm ailululu lkjh3214 yangjinjun3 贡 献 者 们 感 谢 以 下 小 伙 伴 们 为 nutui 发 展 做 出 的 贡 献 共 建 指 南 贡 献 指 南 pr 开 发 者 开 源 协 议 mit", + "source_text": "社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT", + "segmentation_method": "sections" + } + ] + } + }, + "artifact": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。\n\n特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源\n\n官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集\n\n社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT\n\n## 链接", + "ok": true, + "expected_task_pass": false + }, + { + "id": "heading-only-body-region-headings-only", + "category": "body-scope", + "root_cause": "separate_headings_not_contiguous_body_fragment", + "failure_mode": "false-positive", + "case": { + "url": "https://nutui.jd.com/h5/vue/4x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "nutui-independent-heading-0", + "alternatives": [ + "扫码体验" + ], + "source_location": "main heading", + "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" + }, + { + "id": "nutui-independent-heading-1", + "alternatives": [ + "链接" + ], + "source_location": "main heading", + "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" + } + ] + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 4, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "nutui nutui vue 组 件 库 适 用 于 vue 技 术 栈 的 h5 应 用 开 箱 即 用 帮 助 研 发 快 速 开 发 用 户 界 面 提 升 开 发 效 率 改 善 开 发 体 验", + "source_text": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。", + "segmentation_method": "sections" + }, + { + "id": "body-region-2", + "text": "特 性 80 高 质 量 组 件 覆 盖 移 动 端 主 流 场 景 基 于 京 东 app 10 0 视 觉 规 范 支 持 按 需 引 用 详 尽 的 文 档 和 示 例 支 持 typescript 支 持 webstorm vscode 组 件 属 性 高 亮 支 持 动 态 定 制 主 题 支 持 暗 黑 模 式 支 持 国 际 化 单 元 测 试 覆 盖 率 超 过 80 保 障 稳 定 性 提 供 sketch 设 计 资 源", + "source_text": "特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源", + "segmentation_method": "sections" + }, + { + "id": "body-region-4", + "text": "官 方 生 态 由 nutui 官 方 团 队 维 护 的 项 目 如 下 项 目 描 述 nutui nutui nutui 移 动 端 h5 版 本 nutui nutui taro nutui taro 多 端 版 本 nutui icons vue nutui nutui 使 用 的 图 标 库 nutui icons vue taro nutui nutui taro 使 用 的 图 标 库 nutui touch emulator 在 桌 面 端 使 用 nutui 的 辅 助 库 nutui playground nutui 在 线 playground nutui demo nutui 官 方 示 例 合 集", + "source_text": "官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集", + "segmentation_method": "sections" + }, + { + "id": "body-region-5", + "text": "社 区 生 态 由 社 区 维 护 的 项 目 如 下 欢 迎 补 充 项 目 描 述 yang1206 uniapp nutui 基 于 nutui 4 x 版 本 开 发 的 uni app 组 件 库 jwaterwater uni nutui 基 于 nutui 3 x 版 本 开 发 的 uni app 组 件 库 贡 献 核 心 人 员 以 下 是 nutui 的 核 心 贡 献 者 们 richard1015 drjingfubo love forever yangxiaolu1993 snandy eiinu ymm ailululu lkjh3214 yangjinjun3 贡 献 者 们 感 谢 以 下 小 伙 伴 们 为 nutui 发 展 做 出 的 贡 献 共 建 指 南 贡 献 指 南 pr 开 发 者 开 源 协 议 mit", + "source_text": "社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT", + "segmentation_method": "sections" + } + ] + } + }, + "artifact": "## 扫码体验\n\n![NutUI](https://example.org/qr.png)\n\n## 链接", + "ok": true, + "expected_task_pass": false + }, + { + "id": "detail-url-unreserved-encoding-unreserved-encoded", + "category": "format-normalization", + "root_cause": "unreserved_url_encoding_preserves_detail_identity", + "failure_mode": "false-negative", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[Film Beta](/film/fichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "detail-url-unreserved-encoding-encoded-duplicate", + "category": "format-normalization", + "root_cause": "unreserved_url_encoding_preserves_detail_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[Film Alpha](/film/fichefilm_gen_cfilm=123.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "detail-url-unreserved-encoding-reserved-separator", + "category": "format-normalization", + "root_cause": "unreserved_url_encoding_preserves_detail_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[Film Beta](/film%2Ffichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "detail-url-unreserved-encoding-empty-name", + "category": "format-normalization", + "root_cause": "unreserved_url_encoding_preserves_detail_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[](/film/fichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "detail-url-unreserved-encoding-wrong-detail-path", + "category": "format-normalization", + "root_cause": "unreserved_url_encoding_preserves_detail_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[Film Beta](/film/fichefilm_gen_cfilm=456.json)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-link-separated-list-comma-list", + "category": "format-normalization", + "root_cause": "punctuation_separated_explicit_named_links", + "failure_mode": "false-negative", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm_gen_cfilm=123.html), [Film Beta](/film/fichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "named-link-separated-list-semicolon-list", + "category": "format-normalization", + "root_cause": "punctuation_separated_explicit_named_links", + "failure_mode": "false-negative", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm_gen_cfilm=123.html); [Film Beta](/film/fichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "named-link-separated-list-prose-recommendation", + "category": "format-normalization", + "root_cause": "punctuation_separated_explicit_named_links", + "failure_mode": "false-positive", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "Watch [Film Alpha](/film/fichefilm_gen_cfilm=123.html) or [Film Beta](/film/fichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-link-separated-list-prose-between-links", + "category": "format-normalization", + "root_cause": "punctuation_separated_explicit_named_links", + "failure_mode": "false-positive", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm_gen_cfilm=123.html) is better than [Film Beta](/film/fichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "joyn-movie-ownership-fresh-movies", + "category": "structured-extraction", + "root_cause": "movie_records_not_genre_navigation_or_placeholder", + "failure_mode": "false-negative", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "[New Movie](/filme/new-movie)\n[Another Movie](/filme/another-movie)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "joyn-movie-ownership-genres-only", + "category": "structured-extraction", + "root_cause": "movie_records_not_genre_navigation_or_placeholder", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "[Drama](/filme/genre/drama)\n[Comedy](/filme/genre/comedy)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "joyn-movie-ownership-duplicated-movie", + "category": "structured-extraction", + "root_cause": "movie_records_not_genre_navigation_or_placeholder", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "[New Movie](/filme/new-movie)\n[New Movie](/filme/new-movie)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "joyn-movie-ownership-empty-name", + "category": "structured-extraction", + "root_cause": "movie_records_not_genre_navigation_or_placeholder", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "[](/filme/new-movie)\n[Another Movie](/filme/another-movie)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "joyn-movie-ownership-html-placeholders", + "category": "structured-extraction", + "root_cause": "movie_records_not_genre_navigation_or_placeholder", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "<main><a data-testid=\"CSP\" href=\"/filme/new-movie\">Hidden title</a><a data-testid=\"CSP\" href=\"/filme/another-movie\">Hidden title2</a></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "prosieben-owned-content-fresh-content", + "category": "structured-extraction", + "root_cause": "content_cards_not_placeholder_or_editorial_links", + "failure_mode": "false-negative", + "case": { + "url": "https://www.joyn.de/mediatheken/prosieben", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", + "start_pattern": "Meistgesehene Sendungen", + "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" + } + } + }, + "artifact": "Meistgesehene Sendungen\n[New Show](/serien/new-show)\n[New Movie](/filme/new-movie)\nProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen", + "ok": true, + "expected_task_pass": true + }, + { + "id": "prosieben-owned-content-editorial-only", + "category": "structured-extraction", + "root_cause": "content_cards_not_placeholder_or_editorial_links", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/mediatheken/prosieben", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", + "start_pattern": "Meistgesehene Sendungen", + "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" + } + } + }, + "artifact": "Meistgesehene Sendungen\nProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen\n[New Show](/serien/new-show)\n[New Movie](/filme/new-movie)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "prosieben-owned-content-missing-list-region", + "category": "structured-extraction", + "root_cause": "content_cards_not_placeholder_or_editorial_links", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/mediatheken/prosieben", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", + "start_pattern": "Meistgesehene Sendungen", + "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" + } + } + }, + "artifact": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen\n[New Show](/serien/new-show)\n[New Movie](/filme/new-movie)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "prosieben-owned-content-genres", + "category": "structured-extraction", + "root_cause": "content_cards_not_placeholder_or_editorial_links", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/mediatheken/prosieben", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", + "start_pattern": "Meistgesehene Sendungen", + "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" + } + } + }, + "artifact": "Meistgesehene Sendungen\n[Drama](/filme/genre/drama)\n[Comedy](/serien/genre/comedy)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "prosieben-owned-content-placeholder-html", + "category": "structured-extraction", + "root_cause": "content_cards_not_placeholder_or_editorial_links", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/mediatheken/prosieben", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", + "start_pattern": "Meistgesehene Sendungen", + "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" + } + } + }, + "artifact": "<main><a data-testid=\"CSP\" href=\"/serien/new-show\">New Show</a><a data-testid=\"CSP\" href=\"/filme/new-movie\">New Movie</a></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "francetv-content-cards-real-content-paths", + "category": "structured-extraction", + "root_cause": "content_card_routes_not_partial_channel_allowlist", + "failure_mode": "false-negative", + "case": { + "url": "https://www.france.tv/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.france.tv" + ], + "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main li:has(> a[data-card-link=\"true\"])", + "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", + "link_selector": ":scope > a[data-card-link=\"true\"]" + } + } + }, + "artifact": "[Programme Nouveau](/arte/programme-nouveau/)\n[Collection Nouvelle](/collection/collection-nouvelle/)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "francetv-content-cards-root-navigation", + "category": "structured-extraction", + "root_cause": "content_card_routes_not_partial_channel_allowlist", + "failure_mode": "false-positive", + "case": { + "url": "https://www.france.tv/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.france.tv" + ], + "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main li:has(> a[data-card-link=\"true\"])", + "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", + "link_selector": ":scope > a[data-card-link=\"true\"]" + } + } + }, + "artifact": "[Films](/films/)\n[Arte](/arte/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "francetv-content-cards-empty-name", + "category": "structured-extraction", + "root_cause": "content_card_routes_not_partial_channel_allowlist", + "failure_mode": "false-positive", + "case": { + "url": "https://www.france.tv/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.france.tv" + ], + "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main li:has(> a[data-card-link=\"true\"])", + "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", + "link_selector": ":scope > a[data-card-link=\"true\"]" + } + } + }, + "artifact": "[](/arte/programme-nouveau/)\n[Collection Nouvelle](/collection/collection-nouvelle/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "francetv-content-cards-foreign-host", + "category": "structured-extraction", + "root_cause": "content_card_routes_not_partial_channel_allowlist", + "failure_mode": "false-positive", + "case": { + "url": "https://www.france.tv/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.france.tv" + ], + "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main li:has(> a[data-card-link=\"true\"])", + "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", + "link_selector": ":scope > a[data-card-link=\"true\"]" + } + } + }, + "artifact": "[Programme](https://example.org/arte/programme/)\n[Collection](https://example.org/collection/nouvelle/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "francetv-content-cards-same-id-twice", + "category": "structured-extraction", + "root_cause": "content_card_routes_not_partial_channel_allowlist", + "failure_mode": "false-positive", + "case": { + "url": "https://www.france.tv/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.france.tv" + ], + "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main li:has(> a[data-card-link=\"true\"])", + "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", + "link_selector": ":scope > a[data-card-link=\"true\"]" + } + } + }, + "artifact": "[Programme](/arte/programme/)\n[Programme](/arte/programme/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rtve-title-scope-new-programme-and-episode", + "category": "structured-extraction", + "root_cause": "programme_title_links_not_action_buttons", + "failure_mode": "false-negative", + "case": { + "url": "https://www.rtve.es/play/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.rtve.es" + ], + "entry_path": "/play/videos/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" + }, + "title_exclusion_patterns": [ + "(?:\\\\?\\+\\s*info|En directo|Directos)" + ] + } + }, + "artifact": "[Programa Nuevo](/play/videos/programa-nuevo/)\n[Episodio Nuevo](/play/videos/programa-nuevo/episodio/123/)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "rtve-title-scope-buttons-only", + "category": "structured-extraction", + "root_cause": "programme_title_links_not_action_buttons", + "failure_mode": "false-positive", + "case": { + "url": "https://www.rtve.es/play/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.rtve.es" + ], + "entry_path": "/play/videos/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" + }, + "title_exclusion_patterns": [ + "(?:\\\\?\\+\\s*info|En directo|Directos)" + ] + } + }, + "artifact": "[+ info](/play/videos/a/)\n[\\+ info](/play/videos/b/)\n[Directos](/play/videos/directo/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rtve-title-scope-duplicate-id", + "category": "structured-extraction", + "root_cause": "programme_title_links_not_action_buttons", + "failure_mode": "false-positive", + "case": { + "url": "https://www.rtve.es/play/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.rtve.es" + ], + "entry_path": "/play/videos/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" + }, + "title_exclusion_patterns": [ + "(?:\\\\?\\+\\s*info|En directo|Directos)" + ] + } + }, + "artifact": "[Nuevo](/play/videos/nuevo/)\n[Nuevo otro texto](/play/videos/nuevo/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rtve-title-scope-empty-title", + "category": "structured-extraction", + "root_cause": "programme_title_links_not_action_buttons", + "failure_mode": "false-positive", + "case": { + "url": "https://www.rtve.es/play/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.rtve.es" + ], + "entry_path": "/play/videos/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" + }, + "title_exclusion_patterns": [ + "(?:\\\\?\\+\\s*info|En directo|Directos)" + ] + } + }, + "artifact": "[](/play/videos/a/)\n[Real](/play/videos/b/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rtve-title-scope-audio-not-video", + "category": "structured-extraction", + "root_cause": "programme_title_links_not_action_buttons", + "failure_mode": "false-positive", + "case": { + "url": "https://www.rtve.es/play/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.rtve.es" + ], + "entry_path": "/play/videos/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" + }, + "title_exclusion_patterns": [ + "(?:\\\\?\\+\\s*info|En directo|Directos)" + ] + } + }, + "artifact": "[Audio](/play/audios/programa/)\n[Real](/play/videos/b/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rtve-title-scope-html-owned-programme-and-episode", + "category": "structured-extraction", + "root_cause": "programme_title_links_not_action_buttons", + "failure_mode": "false-negative", + "case": { + "url": "https://www.rtve.es/play/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.rtve.es" + ], + "entry_path": "/play/videos/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" + }, + "title_exclusion_patterns": [ + "(?:\\\\?\\+\\s*info|En directo|Directos)" + ] + } + }, + "artifact": "<div class=\"content\"><strong><a href=\"/play/videos/nuevo/\"><span class=\"pretitle\">Nuevo</span></a><a href=\"/play/videos/nuevo/episodio/123/\"><span class=\"maintitle\">Episodio</span></a></strong><a href=\"/play/videos/fake/\">+ info</a></div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "anchor-owned-links-md-independent-nav", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" + } + ] + } + }, + "artifact": "[Library](/library/)\n[Library](/items/new)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "anchor-owned-links-md-record-only", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" + } + ] + } + }, + "artifact": "[Library](/items/new)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "anchor-owned-links-md-repeated-record", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" + } + ] + } + }, + "artifact": "[Library](/items/new)\n[Library](/items/new)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "anchor-owned-links-html-independent-nav", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" + } + ] + } + }, + "artifact": "<nav><a href=\"/library/\">Library</a></nav><main><a href=\"/items/new\">Library</a></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "anchor-owned-links-html-record-only", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" + } + ] + } + }, + "artifact": "<main><a href=\"/items/new\">Library</a></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "anchor-owned-links-html-hidden-nav", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" + } + ] + } + }, + "artifact": "<nav hidden>Library</nav><main><a href=\"/items/new\">Library</a></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "anchor-owned-links-md-same-line-nav", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" + } + ] + } + }, + "artifact": "[Library](/library/) [Library](/items/new)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "anchor-owned-links-non-http-navigation", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" + } + ] + } + }, + "artifact": "[Library](mailto:help@example.org)\n[Library](/items/new)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nullable-table-faithful-empty", + "category": "structured-extraction", + "root_cause": "declared_empty_table_cells_are_real_values", + "failure_mode": "false-negative", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)(?:[ \\t]+(?P<age>\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + } + }, + "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nullable-table-wrong-empty", + "category": "structured-extraction", + "root_cause": "declared_empty_table_cells_are_real_values", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)(?:[ \\t]+(?P<age>\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + } + }, + "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow||\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nullable-table-invented-age", + "category": "structured-extraction", + "root_cause": "declared_empty_table_cells_are_real_values", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)(?:[ \\t]+(?P<age>\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + } + }, + "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen|25|\nPro version", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nullable-table-missing-cell", + "category": "structured-extraction", + "root_cause": "declared_empty_table_cells_are_real_values", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)(?:[ \\t]+(?P<age>\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + } + }, + "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen|\nPro version", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nullable-table-mixed-row", + "category": "structured-extraction", + "root_cause": "declared_empty_table_cells_are_real_values", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)(?:[ \\t]+(?P<age>\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + } + }, + "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Lannister|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": false + }, + { + "id": "documentation-prose-and-grid", + "category": "structured-extraction", + "root_cause": "documentation_prose_and_grid_are_separate_requirements", + "failure_mode": "false-negative", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)(?:[ \\t]+(?P<age>\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "the grid displays structured rows and columns each row preserves the fields of one record", + "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", + "segmentation_method": "sentences" + } + ] + }, + "forbidden_any": [ + "Access denied", + "CAPTCHA" + ] + }, + "artifact": "The grid displays structured rows and columns. Each row preserves the fields of one record.\nCommunity version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": true + }, + { + "id": "documentation-prose-without-grid", + "category": "structured-extraction", + "root_cause": "documentation_prose_and_grid_are_separate_requirements", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)(?:[ \\t]+(?P<age>\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "the grid displays structured rows and columns each row preserves the fields of one record", + "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", + "segmentation_method": "sentences" + } + ] + }, + "forbidden_any": [ + "Access denied", + "CAPTCHA" + ] + }, + "artifact": "The grid displays structured rows and columns. Each row preserves the fields of one record.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "documentation-grid-without-prose", + "category": "structured-extraction", + "root_cause": "documentation_prose_and_grid_are_separate_requirements", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)(?:[ \\t]+(?P<age>\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "the grid displays structured rows and columns each row preserves the fields of one record", + "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", + "segmentation_method": "sentences" + } + ] + }, + "forbidden_any": [ + "Access denied", + "CAPTCHA" + ] + }, + "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": false + }, + { + "id": "consent-alongside-real-documentation", + "category": "structured-extraction", + "root_cause": "cookie_preferences_not_blocked_documentation", + "failure_mode": "false-negative", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)(?:[ \\t]+(?P<age>\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "the grid displays structured rows and columns each row preserves the fields of one record", + "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", + "segmentation_method": "sentences" + } + ] + }, + "forbidden_any": [ + "Access denied", + "CAPTCHA" + ] + }, + "artifact": "Cookie preferences\nThe grid displays structured rows and columns. Each row preserves the fields of one record.\nCommunity version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": true + }, + { + "id": "consent-only-is-not-documentation", + "category": "structured-extraction", + "root_cause": "cookie_preferences_not_blocked_documentation", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)(?:[ \\t]+(?P<age>\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "the grid displays structured rows and columns each row preserves the fields of one record", + "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", + "segmentation_method": "sentences" + } + ] + }, + "forbidden_any": [ + "Access denied", + "CAPTCHA" + ] + }, + "artifact": "Cookie preferences", + "ok": true, + "expected_task_pass": false + }, + { + "id": "consent-does-not-hide-access-denied", + "category": "structured-extraction", + "root_cause": "cookie_preferences_not_blocked_documentation", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)(?:[ \\t]+(?P<age>\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "the grid displays structured rows and columns each row preserves the fields of one record", + "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", + "segmentation_method": "sentences" + } + ] + }, + "forbidden_any": [ + "Access denied", + "CAPTCHA" + ] + }, + "artifact": "Access denied\nThe grid displays structured rows and columns. Each row preserves the fields of one record.\nCommunity version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": false + }, + { + "id": "mui-rowheader-full-name-counts", + "category": "structured-extraction", + "root_cause": "documentation_prose_and_grid_are_separate_requirements", + "failure_mode": "false-negative", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + }, + { + "name": "full", + "label": "Full name", + "type": "text" + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"]),[role=rowheader]", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)[ \\t]+(?P<age>\\d*)[ \\t]+(?P<full>[A-Za-z]+ [A-Za-z]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)\\n[ \\t\\n]*(?P<full>[A-Za-z]+ [A-Za-z]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14", + "full": "Jon Snow" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31", + "full": "Cersei Lannister" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31", + "full": "Jaime Lannister" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11", + "full": "Arya Stark" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "", + "full": "Daenerys Targaryen" + } + ] + } + } + }, + "artifact": "<div role=\"grid\"><div role=\"row\" data-id=\"1\"><div role=\"gridcell\" data-field=\"id\">1</div><div role=\"gridcell\" data-field=\"first\">Jon</div><div role=\"gridcell\" data-field=\"last\">Snow</div><div role=\"gridcell\" data-field=\"age\">14</div><div role=\"rowheader\" data-field=\"full\">Jon Snow</div></div><div role=\"row\" data-id=\"2\"><div role=\"gridcell\" data-field=\"id\">2</div><div role=\"gridcell\" data-field=\"first\">Cersei</div><div role=\"gridcell\" data-field=\"last\">Lannister</div><div role=\"gridcell\" data-field=\"age\">31</div><div role=\"rowheader\" data-field=\"full\">Cersei Lannister</div></div><div role=\"row\" data-id=\"3\"><div role=\"gridcell\" data-field=\"id\">3</div><div role=\"gridcell\" data-field=\"first\">Jaime</div><div role=\"gridcell\" data-field=\"last\">Lannister</div><div role=\"gridcell\" data-field=\"age\">31</div><div role=\"rowheader\" data-field=\"full\">Jaime Lannister</div></div><div role=\"row\" data-id=\"4\"><div role=\"gridcell\" data-field=\"id\">4</div><div role=\"gridcell\" data-field=\"first\">Arya</div><div role=\"gridcell\" data-field=\"last\">Stark</div><div role=\"gridcell\" data-field=\"age\">11</div><div role=\"rowheader\" data-field=\"full\">Arya Stark</div></div><div role=\"row\" data-id=\"5\"><div role=\"gridcell\" data-field=\"id\">5</div><div role=\"gridcell\" data-field=\"first\">Daenerys</div><div role=\"gridcell\" data-field=\"last\">Targaryen</div><div role=\"gridcell\" data-field=\"age\"></div><div role=\"rowheader\" data-field=\"full\">Daenerys Targaryen</div></div></div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "mui-missing-rowheader-full-name", + "category": "structured-extraction", + "root_cause": "documentation_prose_and_grid_are_separate_requirements", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + }, + { + "name": "full", + "label": "Full name", + "type": "text" + } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"]),[role=rowheader]", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P<id>\\d+)\\s+(?P<first>[A-Za-z]+)\\s+(?P<last>[A-Za-z]+)[ \\t]+(?P<age>\\d*)[ \\t]+(?P<full>[A-Za-z]+ [A-Za-z]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<id>\\d+)\\n[ \\t\\n]*(?P<first>[A-Za-z]+)\\n[ \\t\\n]*(?P<last>[A-Za-z]+)\\n[ \\t\\n]*(?P<age>\\d+)\\n[ \\t\\n]*(?P<full>[A-Za-z]+ [A-Za-z]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14", + "full": "Jon Snow" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31", + "full": "Cersei Lannister" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31", + "full": "Jaime Lannister" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11", + "full": "Arya Stark" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "", + "full": "Daenerys Targaryen" + } + ] + } + } + }, + "artifact": "<div role=\"grid\"><div role=\"row\" data-id=\"1\"><div role=\"gridcell\" data-field=\"id\">1</div><div role=\"gridcell\" data-field=\"first\">Jon</div><div role=\"gridcell\" data-field=\"last\">Snow</div><div role=\"gridcell\" data-field=\"age\">14</div></div><div role=\"row\" data-id=\"2\"><div role=\"gridcell\" data-field=\"id\">2</div><div role=\"gridcell\" data-field=\"first\">Cersei</div><div role=\"gridcell\" data-field=\"last\">Lannister</div><div role=\"gridcell\" data-field=\"age\">31</div><div role=\"rowheader\" data-field=\"full\">Cersei Lannister</div></div><div role=\"row\" data-id=\"3\"><div role=\"gridcell\" data-field=\"id\">3</div><div role=\"gridcell\" data-field=\"first\">Jaime</div><div role=\"gridcell\" data-field=\"last\">Lannister</div><div role=\"gridcell\" data-field=\"age\">31</div><div role=\"rowheader\" data-field=\"full\">Jaime Lannister</div></div><div role=\"row\" data-id=\"4\"><div role=\"gridcell\" data-field=\"id\">4</div><div role=\"gridcell\" data-field=\"first\">Arya</div><div role=\"gridcell\" data-field=\"last\">Stark</div><div role=\"gridcell\" data-field=\"age\">11</div><div role=\"rowheader\" data-field=\"full\">Arya Stark</div></div><div role=\"row\" data-id=\"5\"><div role=\"gridcell\" data-field=\"id\">5</div><div role=\"gridcell\" data-field=\"first\">Daenerys</div><div role=\"gridcell\" data-field=\"last\">Targaryen</div><div role=\"gridcell\" data-field=\"age\"></div><div role=\"rowheader\" data-field=\"full\">Daenerys Targaryen</div></div></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "html-anchor-entity-named-space", + "category": "format-normalization", + "root_cause": "html_entity_space_is_visible_whitespace", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "MUI X Data Grid" + ] + } + ] + } + }, + "artifact": "<main><h1>MUI X Data Grid</h1></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "html-anchor-entity-numeric-space", + "category": "format-normalization", + "root_cause": "html_entity_space_is_visible_whitespace", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "MUI X Data Grid" + ] + } + ] + } + }, + "artifact": "<main><h1>MUI X Data Grid</h1></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "html-anchor-entity-wrong-heading", + "category": "format-normalization", + "root_cause": "html_entity_space_is_visible_whitespace", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "MUI X Data Grid" + ] + } + ] + } + }, + "artifact": "<main><h1>MUI X Date Picker</h1></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "html-anchor-entity-literal-entity", + "category": "format-normalization", + "root_cause": "html_entity_space_is_visible_whitespace", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "MUI X Data Grid" + ] + } + ] + } + }, + "artifact": "<main><h1>MUI&nbsp;X Data Grid</h1></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-source-duplicate-faithful", + "category": "structured-extraction", + "root_cause": "source_identical_table_duplicates_count_once", + "failure_mode": "false-negative", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "species", + "label": "Species", + "type": "text" + }, + { + "name": "diet", + "label": "Diet", + "type": "text" + }, + { + "name": "habitat", + "label": "Habitat", + "type": "text" + } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P<species>[A-Z][a-z]+ [a-z]+)\\s+(?P<diet>Herbivore|Carnivore|Omnivore)\\s+(?P<habitat>[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P<species>[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?P<diet>Herbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P<habitat>[A-Za-z, ]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "name": "African Elephant", + "species": "Loxodonta africana", + "diet": "Herbivore", + "habitat": "Savanna, Forests" + }, + { + "name": "Lion", + "species": "Panthera leo", + "diet": "Carnivore", + "habitat": "Savanna, Grassland" + } + ] + } + } + }, + "artifact": "The standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-source-duplicate-not-extra-record", + "category": "structured-extraction", + "root_cause": "source_identical_table_duplicates_count_once", + "failure_mode": "false-positive", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "species", + "label": "Species", + "type": "text" + }, + { + "name": "diet", + "label": "Diet", + "type": "text" + }, + { + "name": "habitat", + "label": "Habitat", + "type": "text" + } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P<species>[A-Z][a-z]+ [a-z]+)\\s+(?P<diet>Herbivore|Carnivore|Omnivore)\\s+(?P<habitat>[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P<species>[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?P<diet>Herbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P<habitat>[A-Za-z, ]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "name": "African Elephant", + "species": "Loxodonta africana", + "diet": "Herbivore", + "habitat": "Savanna, Forests" + }, + { + "name": "Lion", + "species": "Panthera leo", + "diet": "Carnivore", + "habitat": "Savanna, Grassland" + } + ] + } + } + }, + "artifact": "The standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-conflicting-identity-not-deduplicated", + "category": "structured-extraction", + "root_cause": "source_identical_table_duplicates_count_once", + "failure_mode": "false-positive", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "species", + "label": "Species", + "type": "text" + }, + { + "name": "diet", + "label": "Diet", + "type": "text" + }, + { + "name": "habitat", + "label": "Habitat", + "type": "text" + } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P<species>[A-Z][a-z]+ [a-z]+)\\s+(?P<diet>Herbivore|Carnivore|Omnivore)\\s+(?P<habitat>[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P<species>[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?P<diet>Herbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P<habitat>[A-Za-z, ]+)[ \\t]*$" + } + } + }, + "artifact": "The standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n|Lion|Panthera leo|Herbivore|Savanna, Grassland|\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "usage-demo-body-and-owned-rows", + "category": "structured-extraction", + "root_cause": "usage_demo_rows_not_api_component_table", + "failure_mode": "false-negative", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "species", + "label": "Species", + "type": "text" + }, + { + "name": "diet", + "label": "Diet", + "type": "text" + }, + { + "name": "habitat", + "label": "Habitat", + "type": "text" + } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P<species>[A-Z][a-z]+ [a-z]+)\\s+(?P<diet>Herbivore|Carnivore|Omnivore)\\s+(?P<habitat>[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P<species>[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?P<diet>Herbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P<habitat>[A-Za-z, ]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "name": "African Elephant", + "species": "Loxodonta africana", + "diet": "Herbivore", + "habitat": "Savanna, Forests" + }, + { + "name": "Lion", + "species": "Panthera leo", + "diet": "Carnivore", + "habitat": "Savanna, Grassland" + } + ] + } + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "sorting pagination and filtering is supported and done internally by the component itself", + "source_text": "sorting pagination and filtering is supported and done internally by the component itself", + "segmentation_method": "balanced_tokens_fallback" + } + ] + } + }, + "artifact": "Sorting, pagination, and filtering is supported and done internally by the component itself.\nThe standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "usage-prose-alone-not-enough", + "category": "structured-extraction", + "root_cause": "usage_demo_rows_not_api_component_table", + "failure_mode": "false-positive", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "species", + "label": "Species", + "type": "text" + }, + { + "name": "diet", + "label": "Diet", + "type": "text" + }, + { + "name": "habitat", + "label": "Habitat", + "type": "text" + } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P<species>[A-Z][a-z]+ [a-z]+)\\s+(?P<diet>Herbivore|Carnivore|Omnivore)\\s+(?P<habitat>[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P<species>[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?P<diet>Herbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P<habitat>[A-Za-z, ]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "name": "African Elephant", + "species": "Loxodonta africana", + "diet": "Herbivore", + "habitat": "Savanna, Forests" + }, + { + "name": "Lion", + "species": "Panthera leo", + "diet": "Carnivore", + "habitat": "Savanna, Grassland" + } + ] + } + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "sorting pagination and filtering is supported and done internally by the component itself", + "source_text": "sorting pagination and filtering is supported and done internally by the component itself", + "segmentation_method": "balanced_tokens_fallback" + } + ] + } + }, + "artifact": "Sorting, pagination, and filtering is supported and done internally by the component itself.\nThe standard data table presumes that the entire data set is available locally.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "usage-api-table-not-demo", + "category": "structured-extraction", + "root_cause": "usage_demo_rows_not_api_component_table", + "failure_mode": "false-positive", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "species", + "label": "Species", + "type": "text" + }, + { + "name": "diet", + "label": "Diet", + "type": "text" + }, + { + "name": "habitat", + "label": "Habitat", + "type": "text" + } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P<species>[A-Z][a-z]+ [a-z]+)\\s+(?P<diet>Herbivore|Carnivore|Omnivore)\\s+(?P<habitat>[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P<species>[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?P<diet>Herbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P<habitat>[A-Za-z, ]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "name": "African Elephant", + "species": "Loxodonta africana", + "diet": "Herbivore", + "habitat": "Savanna, Forests" + }, + { + "name": "Lion", + "species": "Panthera leo", + "diet": "Carnivore", + "habitat": "Savanna, Grassland" + } + ] + } + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "sorting pagination and filtering is supported and done internally by the component itself", + "source_text": "sorting pagination and filtering is supported and done internally by the component itself", + "segmentation_method": "balanced_tokens_fallback" + } + ] + } + }, + "artifact": "Sorting, pagination, and filtering is supported and done internally by the component itself.\nThe standard data table presumes that the entire data set is available locally.\nAPI\n\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "usage-mixed-fields-rejected", + "category": "structured-extraction", + "root_cause": "usage_demo_rows_not_api_component_table", + "failure_mode": "false-positive", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "species", + "label": "Species", + "type": "text" + }, + { + "name": "diet", + "label": "Diet", + "type": "text" + }, + { + "name": "habitat", + "label": "Habitat", + "type": "text" + } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P<species>[A-Z][a-z]+ [a-z]+)\\s+(?P<diet>Herbivore|Carnivore|Omnivore)\\s+(?P<habitat>[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P<name>[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P<species>[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?P<diet>Herbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P<habitat>[A-Za-z, ]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "name": "African Elephant", + "species": "Loxodonta africana", + "diet": "Herbivore", + "habitat": "Savanna, Forests" + }, + { + "name": "Lion", + "species": "Panthera leo", + "diet": "Carnivore", + "habitat": "Savanna, Grassland" + } + ] + } + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "sorting pagination and filtering is supported and done internally by the component itself", + "source_text": "sorting pagination and filtering is supported and done internally by the component itself", + "segmentation_method": "balanced_tokens_fallback" + } + ] + } + }, + "artifact": "Sorting, pagination, and filtering is supported and done internally by the component itself.\nThe standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Loxodonta africana|Carnivore|Savanna, Grassland|\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "amazonmx-owned-card-fresh-products", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-negative", + "case": { + "url": "https://www.amazon.com.mx/gp/bestsellers/electronics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.amazon.com.mx" + ], + "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/[^?#]*)?", + "stable_text_anchors": [], + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "#gridItemRoot", + "title_selector": "a[role=link] [class*=\"line-clamp\"]", + "link_selector": "a[role=link]:has([class*=\"line-clamp\"])" + } + } + }, + "artifact": "<div id=\"gridItemRoot\"><a role=\"link\" href=\"/dp/B000000001\"><div class=\"line-clamp\">Producto Nuevo Uno</div></a></div><div id=\"gridItemRoot\"><a role=\"link\" href=\"/dp/B000000002\"><div class=\"line-clamp\">Producto Nuevo Dos</div></a></div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "amazonmx-owned-card-image-only", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.com.mx/gp/bestsellers/electronics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.amazon.com.mx" + ], + "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/[^?#]*)?", + "stable_text_anchors": [], + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "#gridItemRoot", + "title_selector": "a[role=link] [class*=\"line-clamp\"]", + "link_selector": "a[role=link]:has([class*=\"line-clamp\"])" + } + } + }, + "artifact": "<div id=\"gridItemRoot\"><a role=\"link\" href=\"/dp/B000000001\"><div class=\"line-clamp\">Producto Nuevo Uno</div></a></div><div id=\"gridItemRoot\"><a role=\"link\" href=\"/dp/B000000002\"><div class=\"line-clamp\"><img alt=\"Producto Nuevo Dos\"></div></a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "amazonmx-owned-card-duplicate-asin", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.com.mx/gp/bestsellers/electronics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.amazon.com.mx" + ], + "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/[^?#]*)?", + "stable_text_anchors": [], + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "#gridItemRoot", + "title_selector": "a[role=link] [class*=\"line-clamp\"]", + "link_selector": "a[role=link]:has([class*=\"line-clamp\"])" + } + } + }, + "artifact": "<div id=\"gridItemRoot\"><a role=\"link\" href=\"/dp/B000000001\"><div class=\"line-clamp\">Producto Nuevo Uno</div></a></div><div id=\"gridItemRoot\"><a role=\"link\" href=\"/dp/B000000001\"><div class=\"line-clamp\">Producto Nuevo Dos</div></a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "amazonmx-owned-card-wrong-host", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.com.mx/gp/bestsellers/electronics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.amazon.com.mx" + ], + "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/[^?#]*)?", + "stable_text_anchors": [], + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "#gridItemRoot", + "title_selector": "a[role=link] [class*=\"line-clamp\"]", + "link_selector": "a[role=link]:has([class*=\"line-clamp\"])" + } + } + }, + "artifact": "<div id=\"gridItemRoot\"><a role=\"link\" href=\"/dp/B000000001\"><div class=\"line-clamp\">Producto Nuevo Uno</div></a></div><div id=\"gridItemRoot\"><a role=\"link\" href=\"https://other.example/dp/B000000002\"><div class=\"line-clamp\">Producto Nuevo Dos</div></a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "falabella-product-scope-owned-title-detail", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-negative", + "case": { + "url": "https://www.falabella.com/falabella-cl/category/cat3091/Equipos-de-musica-y-karaokes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.falabella.com" + ], + "entry_path": "/falabella-cl/product/[0-9]+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/product/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#testId-searchResults-products .grid-pod", + "title_selector": ".pod-subTitle", + "link_selector": "a[data-pod]" + } + } + }, + "artifact": "<div id=\"testId-searchResults-products\"><div class=\"grid-pod\"><a data-pod=\"catalyst-pod\" role=\"button\" href=\"/falabella-cl/product/123/Nuevo/123\"><b class=\"pod-subTitle\">Parlante Nuevo</b></a></div></div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "falabella-product-scope-no-valid-product-destination", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-positive", + "case": { + "url": "https://www.falabella.com/falabella-cl/category/cat3091/Equipos-de-musica-y-karaokes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.falabella.com" + ], + "entry_path": "/falabella-cl/product/[0-9]+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/product/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#testId-searchResults-products .grid-pod", + "title_selector": ".pod-subTitle", + "link_selector": "a[data-pod]" + } + } + }, + "artifact": "<div id=\"testId-searchResults-products\"><div class=\"grid-pod\"><a data-pod=\"catalyst-pod\" role=\"button\" href=\"\" data-key=\"123\"><b class=\"pod-subTitle\">Parlante Nuevo</b></a></div></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "falabella-product-scope-outside-product-card", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-positive", + "case": { + "url": "https://www.falabella.com/falabella-cl/category/cat3091/Equipos-de-musica-y-karaokes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.falabella.com" + ], + "entry_path": "/falabella-cl/product/[0-9]+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/product/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#testId-searchResults-products .grid-pod", + "title_selector": ".pod-subTitle", + "link_selector": "a[data-pod]" + } + } + }, + "artifact": "<div id=\"unrelated-footer\"><div class=\"grid-pod\"><a data-pod=\"catalyst-pod\" role=\"button\" href=\"/falabella-cl/product/123/Nuevo/123\"><b class=\"pod-subTitle\">Parlante Nuevo</b></a></div></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "falabella-product-scope-missing-title", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-positive", + "case": { + "url": "https://www.falabella.com/falabella-cl/category/cat3091/Equipos-de-musica-y-karaokes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.falabella.com" + ], + "entry_path": "/falabella-cl/product/[0-9]+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/product/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#testId-searchResults-products .grid-pod", + "title_selector": ".pod-subTitle", + "link_selector": "a[data-pod]" + } + } + }, + "artifact": "<div id=\"testId-searchResults-products\"><div class=\"grid-pod\"><a data-pod=\"catalyst-pod\" role=\"button\" href=\"/falabella-cl/product/123/Nuevo/123\"><b class=\"pod-subTitle\"></b></a></div></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "lotteon-product-scope-owned-title-detail", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-negative", + "case": { + "url": "https://www.lotteon.com/p/display/main/lotteon", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.lotteon.com" + ], + "entry_path": "/p/product/[A-Za-z0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/p/product/([A-Za-z0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": ".c-product-card", + "title_selector": ".c-product-title__title", + "link_selector": "a.c-product-card__url" + } + } + }, + "artifact": "<div class=\"c-product-card\"><a class=\"c-product-card__url\" href=\"/p/product/LO123\"></a><p class=\"c-product-title__title\">새 상품 이름</p></div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "lotteon-product-scope-no-valid-product-destination", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-positive", + "case": { + "url": "https://www.lotteon.com/p/display/main/lotteon", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.lotteon.com" + ], + "entry_path": "/p/product/[A-Za-z0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/p/product/([A-Za-z0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": ".c-product-card", + "title_selector": ".c-product-title__title", + "link_selector": "a.c-product-card__url" + } + } + }, + "artifact": "<div class=\"c-product-card\"><a class=\"c-product-card__url\" href=\"/p/display/shop/seltDpShop/123\"></a><p class=\"c-product-title__title\">새 상품 이름</p></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "lotteon-product-scope-outside-product-card", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-positive", + "case": { + "url": "https://www.lotteon.com/p/display/main/lotteon", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.lotteon.com" + ], + "entry_path": "/p/product/[A-Za-z0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/p/product/([A-Za-z0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": ".c-product-card", + "title_selector": ".c-product-title__title", + "link_selector": "a.c-product-card__url" + } + } + }, + "artifact": "<div class=\"navigation\"><a class=\"c-product-card__url\" href=\"/p/product/LO123\"></a><p class=\"c-product-title__title\">새 상품 이름</p></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "lotteon-product-scope-missing-title", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-positive", + "case": { + "url": "https://www.lotteon.com/p/display/main/lotteon", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.lotteon.com" + ], + "entry_path": "/p/product/[A-Za-z0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/p/product/([A-Za-z0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": ".c-product-card", + "title_selector": ".c-product-title__title", + "link_selector": "a.c-product-card__url" + } + } + }, + "artifact": "<div class=\"c-product-card\"><a class=\"c-product-card__url\" href=\"/p/product/LO123\"></a><p class=\"c-product-title__title\"></p></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-088-terra-home-new-stories", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.terra.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.terra.com.br" + ], + "entry_path": "/.+,[A-Za-z0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" + }, + "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" + } + }, + "artifact": "[Notícia nova A](https://www.terra.com.br/noticias/mundo/noticia-a,abc123.html)\n[Notícia nova B](https://www.terra.com.br/esportes/noticia-b,def456.html)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-088-terra-home-navigation-only", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.terra.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.terra.com.br" + ], + "entry_path": "/.+,[A-Za-z0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" + }, + "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" + } + }, + "artifact": "[Notícias](https://www.terra.com.br/noticias/)\n[Mundo](https://www.terra.com.br/noticias/mundo/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-088-terra-home-duplicate-story", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.terra.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.terra.com.br" + ], + "entry_path": "/.+,[A-Za-z0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" + }, + "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" + } + }, + "artifact": "[Notícia nova A](https://www.terra.com.br/noticias/mundo/noticia-a,abc123.html)\n[Notícia nova B](https://www.terra.com.br/esportes/outro-titulo,abc123.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-093-pt-real-terra-noticias-new-stories", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.terra.com.br/noticias/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.terra.com.br" + ], + "entry_path": "/.+,[A-Za-z0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" + }, + "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" + } + }, + "artifact": "[Notícia nova A](https://www.terra.com.br/noticias/mundo/noticia-a,abc123.html)\n[Notícia nova B](https://www.terra.com.br/esportes/noticia-b,def456.html)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-093-pt-real-terra-noticias-navigation-only", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.terra.com.br/noticias/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.terra.com.br" + ], + "entry_path": "/.+,[A-Za-z0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" + }, + "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" + } + }, + "artifact": "[Notícias](https://www.terra.com.br/noticias/)\n[Mundo](https://www.terra.com.br/noticias/mundo/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-093-pt-real-terra-noticias-duplicate-story", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.terra.com.br/noticias/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.terra.com.br" + ], + "entry_path": "/.+,[A-Za-z0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" + }, + "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" + } + }, + "artifact": "[Notícia nova A](https://www.terra.com.br/noticias/mundo/noticia-a,abc123.html)\n[Notícia nova B](https://www.terra.com.br/esportes/outro-titulo,abc123.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-096-ar-real-cnn-world-new-stories", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://arabic.cnn.com/world", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "arabic.cnn.com" + ], + "entry_path": "/[^/]+/(?:article|video)/[0-9]{4}/[0-9]{2}/[0-9]{2}/[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".container__item:has(.container__headline-text)", + "title_selector": ".container__headline-text", + "link_selector": "a:has(.container__headline-text)" + } + } + }, + "artifact": "[عنوان جديد ألف](/world/article/2026/09/20/new-story-a)\n[عنوان جديد باء](/middle-east/video/2026/09/20/new-story-b)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-096-ar-real-cnn-world-navigation-only", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://arabic.cnn.com/world", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "arabic.cnn.com" + ], + "entry_path": "/[^/]+/(?:article|video)/[0-9]{4}/[0-9]{2}/[0-9]{2}/[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".container__item:has(.container__headline-text)", + "title_selector": ".container__headline-text", + "link_selector": "a:has(.container__headline-text)" + } + } + }, + "artifact": "[أمريكا](/world/us)\n[روسيا](/world/russia)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-096-ar-real-cnn-world-duplicate-story", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://arabic.cnn.com/world", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "arabic.cnn.com" + ], + "entry_path": "/[^/]+/(?:article|video)/[0-9]{4}/[0-9]{2}/[0-9]{2}/[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".container__item:has(.container__headline-text)", + "title_selector": ".container__headline-text", + "link_selector": "a:has(.container__headline-text)" + } + } + }, + "artifact": "[عنوان جديد ألف](/world/article/2026/09/20/new-story-a)\n[عنوان جديد باء](/world/article/2026/09/20/new-story-a)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "cnnbrasil-qualified-routes-new-election-and-blog", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Nova notícia](/eleicoes/nova-noticia/)\n[Nova análise](/blogs/novo-autor/politica/nova-analise/)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "cnnbrasil-qualified-routes-new-science-and-education", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Ciência nova](/ciencia/descoberta-nova/)\n[Educação nova](/educacao/estudo-novo/)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "cnnbrasil-qualified-routes-category-navigation", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Cinema](/pop/cinema/)\n[Loterias](/economia/loterias/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "cnnbrasil-qualified-routes-blog-author-index", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Autora](/blogs/nova-autora/)\n[Autor](/blogs/novo-autor/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "cnnbrasil-qualified-routes-scoreboard-only", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Partida A](/esportes/lances-ao-vivo/liga/2026/a-b/)\n[Partida B](/esportes/lances-ao-vivo/liga/2026/c-d/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "cnnbrasil-qualified-routes-election-tools", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Ver estatísticas](/eleicoes/2026/candidatos/estatisticas/)\n[Índice CNN](/eleicoes/agregador-de-pesquisas-2025/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "cnnbrasil-qualified-routes-duplicate-article", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Título A](/politica/noticia-a/)\n[Outro título](/politica/noticia-a/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "naver-politics-owned-visible-visible-owned-cards", + "category": "structured-extraction", + "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", + "failure_mode": "false-negative", + "case": { + "url": "https://news.naver.com/section/100", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "n.news.naver.com" + ], + "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#newsct .sa_item:not(.is_blind)", + "title_selector": ".sa_text_strong", + "link_selector": "a.sa_text_title", + "start_pattern": "헤드라인 뉴스", + "end_pattern": "언론사별 가장 많이 본 뉴스" + } + } + }, + "artifact": "<div id=newsct><div class=\"sa_item \"><a class=\"sa_text_title\" href=\"https://n.news.naver.com/mnews/article/001/123\"><strong class=\"sa_text_strong\">New news</strong></a></div><div class=\"sa_item \"><a class=\"sa_text_title\" href=\"https://n.news.naver.com/mnews/article/001/456\"><strong class=\"sa_text_strong\">New news</strong></a></div></div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "naver-politics-owned-visible-hidden-expansion-not-denominator", + "category": "structured-extraction", + "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", + "failure_mode": "false-positive", + "case": { + "url": "https://news.naver.com/section/100", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "n.news.naver.com" + ], + "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#newsct .sa_item:not(.is_blind)", + "title_selector": ".sa_text_strong", + "link_selector": "a.sa_text_title", + "start_pattern": "헤드라인 뉴스", + "end_pattern": "언론사별 가장 많이 본 뉴스" + } + } + }, + "artifact": "<div id=newsct><div class=\"sa_item \"><a class=\"sa_text_title\" href=\"https://n.news.naver.com/mnews/article/001/123\"><strong class=\"sa_text_strong\">New news</strong></a></div><div class=\"sa_item is_blind\"><a class=\"sa_text_title\" href=\"https://n.news.naver.com/mnews/article/001/456\"><strong class=\"sa_text_strong\">New news</strong></a></div></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "naver-politics-owned-visible-empty-title", + "category": "structured-extraction", + "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", + "failure_mode": "false-positive", + "case": { + "url": "https://news.naver.com/section/100", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "n.news.naver.com" + ], + "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#newsct .sa_item:not(.is_blind)", + "title_selector": ".sa_text_strong", + "link_selector": "a.sa_text_title", + "start_pattern": "헤드라인 뉴스", + "end_pattern": "언론사별 가장 많이 본 뉴스" + } + } + }, + "artifact": "<div id=newsct><div class=\"sa_item \"><a class=\"sa_text_title\" href=\"https://n.news.naver.com/mnews/article/001/123\"><strong class=\"sa_text_strong\">New news</strong></a></div><div class=\"sa_item \"><a class=\"sa_text_title\" href=\"https://n.news.naver.com/mnews/article/001/456\"><strong class=\"sa_text_strong\"></strong></a></div></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "naver-politics-owned-visible-sidebar-not-main", + "category": "structured-extraction", + "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", + "failure_mode": "false-positive", + "case": { + "url": "https://news.naver.com/section/100", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "n.news.naver.com" + ], + "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#newsct .sa_item:not(.is_blind)", + "title_selector": ".sa_text_strong", + "link_selector": "a.sa_text_title", + "start_pattern": "헤드라인 뉴스", + "end_pattern": "언론사별 가장 많이 본 뉴스" + } + } + }, + "artifact": "<div id=sidebar><div class=\"sa_item \"><a class=\"sa_text_title\" href=\"https://n.news.naver.com/mnews/article/001/123\"><strong class=\"sa_text_strong\">New news</strong></a></div><div class=\"sa_item \"><a class=\"sa_text_title\" href=\"https://n.news.naver.com/mnews/article/001/456\"><strong class=\"sa_text_strong\">New news</strong></a></div></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "naver-politics-owned-visible-fresh-markdown", + "category": "structured-extraction", + "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", + "failure_mode": "false-negative", + "case": { + "url": "https://news.naver.com/section/100", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "n.news.naver.com" + ], + "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#newsct .sa_item:not(.is_blind)", + "title_selector": ".sa_text_strong", + "link_selector": "a.sa_text_title", + "start_pattern": "헤드라인 뉴스", + "end_pattern": "언론사별 가장 많이 본 뉴스" + } + } + }, + "artifact": "헤드라인 뉴스\n[Fresh A](https://n.news.naver.com/mnews/article/001/789)\n[Fresh B](https://n.news.naver.com/mnews/article/002/987)\n언론사별 가장 많이 본 뉴스", + "ok": true, + "expected_task_pass": true + }, + { + "id": "naver-politics-owned-visible-repeated-id", + "category": "structured-extraction", + "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", + "failure_mode": "false-positive", + "case": { + "url": "https://news.naver.com/section/100", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "n.news.naver.com" + ], + "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#newsct .sa_item:not(.is_blind)", + "title_selector": ".sa_text_strong", + "link_selector": "a.sa_text_title", + "start_pattern": "헤드라인 뉴스", + "end_pattern": "언론사별 가장 많이 본 뉴스" + } + } + }, + "artifact": "<div id=newsct><div class=\"sa_item \"><a class=\"sa_text_title\" href=\"https://n.news.naver.com/mnews/article/001/123\"><strong class=\"sa_text_strong\">New news</strong></a></div><div class=\"sa_item \"><a class=\"sa_text_title\" href=\"https://n.news.naver.com/mnews/article/001/123\"><strong class=\"sa_text_strong\">Another title</strong></a></div></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "owned-anchor-card-anchor-is-card", + "category": "structured-extraction", + "root_cause": "card_anchor_owns_visible_title_and_destination", + "failure_mode": "false-negative", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "<a href=\"/produto/123/new-item\"><span class=\"line-clamp-2\">Fresh product</span></a><a href=\"/produto/456/new-item\"><span class=\"line-clamp-2\">Fresh product</span></a>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "owned-anchor-card-empty-title-with-price", + "category": "structured-extraction", + "root_cause": "card_anchor_owns_visible_title_and_destination", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "<a href=\"/produto/123/new-item\"><span class=\"line-clamp-2\">Fresh product</span></a><a href=\"/produto/456/new-item\"><span class=\"line-clamp-2\"></span><span>R$ 100</span></a>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "owned-anchor-card-price-only", + "category": "structured-extraction", + "root_cause": "card_anchor_owns_visible_title_and_destination", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "<a href=\"/produto/123/a\">R$ 100</a><a href=\"/produto/456/b\">R$ 200</a>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "owned-anchor-card-duplicate-id", + "category": "structured-extraction", + "root_cause": "card_anchor_owns_visible_title_and_destination", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "<a href=\"/produto/123/new-item\"><span class=\"line-clamp-2\">Fresh product</span></a><a href=\"/produto/123/new-item\"><span class=\"line-clamp-2\">Different text</span></a>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "owned-anchor-card-fresh-markdown", + "category": "structured-extraction", + "root_cause": "card_anchor_owns_visible_title_and_destination", + "failure_mode": "false-negative", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "[New product A](/produto/123/a)\n[New product B](/produto/456/b)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "kabum-privacy-footer-without-consent", + "category": "structured-extraction", + "root_cause": "optional_consent_label_not_required_footer_text", + "failure_mode": "false-negative", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [ + { + "id": "privacy-footer", + "alternatives": [ + "Políticas de Privacidade" + ], + "match_scope": "non_record_lines" + } + ], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "[Políticas de Privacidade](/privacidade)\n[Fresh product](/produto/123/a)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "kabum-privacy-privacy-only", + "category": "structured-extraction", + "root_cause": "optional_consent_label_not_required_footer_text", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [ + { + "id": "privacy-footer", + "alternatives": [ + "Políticas de Privacidade" + ], + "match_scope": "non_record_lines" + } + ], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "[Políticas de Privacidade](/privacidade)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "kabum-privacy-product-cannot-impersonate-footer", + "category": "structured-extraction", + "root_cause": "optional_consent_label_not_required_footer_text", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [ + { + "id": "privacy-footer", + "alternatives": [ + "Políticas de Privacidade" + ], + "match_scope": "non_record_lines" + } + ], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "[Políticas de Privacidade](/produto/123/a)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "kooora-news-scope-new-news-cards", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.kooora.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kooora.com" + ], + "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/(blt[0-9a-f]+)$", + "entry_scope": { + "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", + "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", + "link_selector": "a[href]" + } + } + }, + "artifact": "<a href=\"/كرة-قدم/أخبار/new-title/bltabc123\"><span class=\"fco-card__headline-text\">خبر جديد</span></a><a href=\"/كرة-قدم/أخبار/new-title/bltdef456\"><span class=\"fco-card__headline-text\">خبر جديد</span></a>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "kooora-news-scope-teams-and-players", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kooora.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kooora.com" + ], + "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/(blt[0-9a-f]+)$", + "entry_scope": { + "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", + "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", + "link_selector": "a[href]" + } + } + }, + "artifact": "[Team](/كرة-القدم/فريق/name/123)\n[Player](/كرة-القدم/لاعب/name/أخبار/456)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "kooora-news-scope-category-index", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kooora.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kooora.com" + ], + "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/(blt[0-9a-f]+)$", + "entry_scope": { + "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", + "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", + "link_selector": "a[href]" + } + } + }, + "artifact": "[Tennis](/تنس/bltabc123)\n[News](/أخبار)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "kooora-news-scope-collapsed-card", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kooora.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kooora.com" + ], + "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/(blt[0-9a-f]+)$", + "entry_scope": { + "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", + "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", + "link_selector": "a[href]" + } + } + }, + "artifact": "<a href=\"/كرة-قدم/أخبار/new-title/bltabc123\"><span class=\"fco-card__headline-text\">خبر جديد</span></a><div class=\"fco-selective-scroll-section__item--hidden\"><a href=\"/كرة-قدم/أخبار/new-title/bltdef456\"><span class=\"fco-card__headline-text\">خبر جديد</span></a></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "kooora-news-scope-same-id-new-slug", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kooora.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kooora.com" + ], + "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/(blt[0-9a-f]+)$", + "entry_scope": { + "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", + "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", + "link_selector": "a[href]" + } + } + }, + "artifact": "[First](/كرة-قدم/أخبار/title/bltabc123)\n[Second](/كرة-قدم/أخبار/changed-title/bltabc123)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "kooora-news-scope-fresh-listicle", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.kooora.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kooora.com" + ], + "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/(blt[0-9a-f]+)$", + "entry_scope": { + "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", + "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", + "link_selector": "a[href]" + } + } + }, + "artifact": "[News](/كرة-قدم/أخبار/new/bltabc123)\n[Listicle](/كرة-قدم/القوائم/new/bltdef456)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "quake-joined-location-time", + "category": "structured-extraction", + "root_cause": "earthquake_inline_location_timestamp", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "4.2\nHarbor Island2026-09-20 12:00:00 (UTC+08:00)\n10.0 km\n3.1\nNorth Valley2026-09-20T05:00:00Z\n7.4 km", + "ok": true, + "expected_task_pass": true + }, + { + "id": "quake-joined-missing-depth", + "category": "structured-extraction", + "root_cause": "earthquake_inline_location_timestamp", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "4.2\nHarbor Island2026-09-20 12:00:00 (UTC+08:00)\n3.1\nNorth Valley2026-09-20T05:00:00Z\n7.4 km", + "ok": true, + "expected_task_pass": false + }, + { + "id": "quake-joined-invalid-time", + "category": "structured-extraction", + "root_cause": "earthquake_inline_location_timestamp", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "4.2\nHarbor Island2026-99-20 12:00:00 (UTC+08:00)\n10.0 km\n3.1\nNorth Valley2026-09-20T05:00:00Z\n7.4 km", + "ok": true, + "expected_task_pass": false + }, + { + "id": "quake-joined-duplicate-record", + "category": "structured-extraction", + "root_cause": "earthquake_inline_location_timestamp", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "4.2\nHarbor Island2026-09-20 12:00:00 (UTC+08:00)\n10.0 km\n4.2\nHarbor Island2026-09-20 12:00:00 (UTC+08:00)\n10.0 km", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-037-complete", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-negative", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P<item>#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "<main><div class=\"q-virtual-scroll\"><div class=\"q-virtual-scroll__content\"><div class=\"q-item\">#0 - Option 1</div><div class=\"q-item\">#1 - Option 2</div><div class=\"q-item\">#2 - Option 3</div><div class=\"q-item\">#3 - Option 4</div><div class=\"q-item\">#4 - Option 5</div><div class=\"q-item\">#5 - Option 6</div><div class=\"q-item\">#6 - Option 7</div><div class=\"q-item\">#7 - Option 8</div></div></div></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic22-037-below80", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P<item>#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "<main><div class=\"q-virtual-scroll\"><div class=\"q-virtual-scroll__content\"><div class=\"q-item\">#0 - Option 1</div><div class=\"q-item\">#1 - Option 2</div><div class=\"q-item\">#2 - Option 3</div><div class=\"q-item\">#3 - Option 4</div><div class=\"q-item\">#4 - Option 5</div><div class=\"q-item\">#5 - Option 6</div><div class=\"q-item\">#6 - Option 7</div></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-037-duplicate", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P<item>#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "<main><div class=\"q-virtual-scroll\"><div class=\"q-virtual-scroll__content\"><div class=\"q-item\">#0 - Option 1</div><div class=\"q-item\">#0 - Option 1</div><div class=\"q-item\">#0 - Option 1</div><div class=\"q-item\">#0 - Option 1</div><div class=\"q-item\">#0 - Option 1</div><div class=\"q-item\">#0 - Option 1</div><div class=\"q-item\">#0 - Option 1</div><div class=\"q-item\">#0 - Option 1</div></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-037-wrong-scope", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P<item>#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "<main><div class=\"other-list\"><div class=\"q-virtual-scroll__content\"><div class=\"q-item\">#0 - Option 1</div><div class=\"q-item\">#1 - Option 2</div><div class=\"q-item\">#2 - Option 3</div><div class=\"q-item\">#3 - Option 4</div><div class=\"q-item\">#4 - Option 5</div><div class=\"q-item\">#5 - Option 6</div><div class=\"q-item\">#6 - Option 7</div><div class=\"q-item\">#7 - Option 8</div></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-030-complete", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-negative", + "case": { + "url": "https://primevue.dev/virtualscroller/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "<main><div class=\"doc-section-demo\"><div class=\"p-virtualscroller-content\"><div>Item #0</div><div>Item #1</div><div>Item #2</div><div>Item #3</div></div></div></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic22-030-below80", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primevue.dev/virtualscroller/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "<main><div class=\"doc-section-demo\"><div class=\"p-virtualscroller-content\"><div>Item #0</div><div>Item #1</div><div>Item #2</div></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-030-duplicate", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primevue.dev/virtualscroller/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "<main><div class=\"doc-section-demo\"><div class=\"p-virtualscroller-content\"><div>Item #0</div><div>Item #0</div><div>Item #0</div><div>Item #0</div></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-030-wrong-scope", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primevue.dev/virtualscroller/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "<main><div class=\"other-demo\"><div class=\"p-virtualscroller-content\"><div>Item #0</div><div>Item #1</div><div>Item #2</div><div>Item #3</div></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-032-complete", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-negative", + "case": { + "url": "https://primeng.dev/virtualscroller", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "<main><div class=\"doc-section-demo\"><div class=\"p-virtualscroller-content\"><div>Item #0</div><div>Item #1</div><div>Item #2</div><div>Item #3</div></div></div></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic22-032-below80", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primeng.dev/virtualscroller", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "<main><div class=\"doc-section-demo\"><div class=\"p-virtualscroller-content\"><div>Item #0</div><div>Item #1</div><div>Item #2</div></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-032-duplicate", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primeng.dev/virtualscroller", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "<main><div class=\"doc-section-demo\"><div class=\"p-virtualscroller-content\"><div>Item #0</div><div>Item #0</div><div>Item #0</div><div>Item #0</div></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-032-wrong-scope", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primeng.dev/virtualscroller", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "<main><div class=\"other-demo\"><div class=\"p-virtualscroller-content\"><div>Item #0</div><div>Item #1</div><div>Item #2</div><div>Item #3</div></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-098-complete", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-negative", + "case": { + "url": "https://shahid.mbc.net/ar", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 5, + "hosts": [ + "shahid.mbc.net" + ], + "entry_path": "/ar/player/episodes/[^/]+/id-[0-9]+", + "entry_identity_pattern": "/id-([0-9]+)$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".carousel-item", + "title_selector": "h3", + "link_selector": "a[href*=\"/ar/player/episodes/\"]", + "start_pattern": "الأكثر مشاهدة الأن", + "end_pattern": "اتصل بنا" + } + } + }, + "artifact": "<main><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-100\"></a><h3>الموسم 2، الحلقة 1</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-101\"></a><h3>الموسم 2، الحلقة 2</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-102\"></a><h3>الموسم 2، الحلقة 3</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-103\"></a><h3>الموسم 2، الحلقة 4</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-104\"></a><h3>الموسم 2، الحلقة 5</h3></div></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic22-098-below80", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://shahid.mbc.net/ar", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 5, + "hosts": [ + "shahid.mbc.net" + ], + "entry_path": "/ar/player/episodes/[^/]+/id-[0-9]+", + "entry_identity_pattern": "/id-([0-9]+)$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".carousel-item", + "title_selector": "h3", + "link_selector": "a[href*=\"/ar/player/episodes/\"]", + "start_pattern": "الأكثر مشاهدة الأن", + "end_pattern": "اتصل بنا" + } + } + }, + "artifact": "<main><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-100\"></a><h3>الموسم 2، الحلقة 1</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-101\"></a><h3>الموسم 2، الحلقة 2</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-102\"></a><h3>الموسم 2، الحلقة 3</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-103\"></a><h3>الموسم 2، الحلقة 4</h3></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-098-duplicate", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://shahid.mbc.net/ar", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 5, + "hosts": [ + "shahid.mbc.net" + ], + "entry_path": "/ar/player/episodes/[^/]+/id-[0-9]+", + "entry_identity_pattern": "/id-([0-9]+)$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".carousel-item", + "title_selector": "h3", + "link_selector": "a[href*=\"/ar/player/episodes/\"]", + "start_pattern": "الأكثر مشاهدة الأن", + "end_pattern": "اتصل بنا" + } + } + }, + "artifact": "<main><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-100\"></a><h3>الموسم 2، الحلقة 1</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-100\"></a><h3>الموسم 2، الحلقة 2</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-100\"></a><h3>الموسم 2، الحلقة 3</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-100\"></a><h3>الموسم 2، الحلقة 4</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/episodes/sample/id-100\"></a><h3>الموسم 2، الحلقة 5</h3></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-098-wrong-scope", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://shahid.mbc.net/ar", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 5, + "hosts": [ + "shahid.mbc.net" + ], + "entry_path": "/ar/player/episodes/[^/]+/id-[0-9]+", + "entry_identity_pattern": "/id-([0-9]+)$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".carousel-item", + "title_selector": "h3", + "link_selector": "a[href*=\"/ar/player/episodes/\"]", + "start_pattern": "الأكثر مشاهدة الأن", + "end_pattern": "اتصل بنا" + } + } + }, + "artifact": "<main><div class=\"carousel-item\"><a href=\"/ar/player/movies/sample/id-100\"></a><h3>الموسم 2، الحلقة 1</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/movies/sample/id-101\"></a><h3>الموسم 2، الحلقة 2</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/movies/sample/id-102\"></a><h3>الموسم 2، الحلقة 3</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/movies/sample/id-103\"></a><h3>الموسم 2، الحلقة 4</h3></div><div class=\"carousel-item\"><a href=\"/ar/player/movies/sample/id-104\"></a><h3>الموسم 2، الحلقة 5</h3></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-061-complete", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-negative", + "case": { + "url": "https://juejin.cn/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 16, + "hosts": [ + "juejin.cn" + ], + "entry_path": "/post/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".entry-list .entry a.title" + } + } + }, + "artifact": "<main><div class=\"entry-list\"><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章1</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/101\">新文章2</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/102\">新文章3</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/103\">新文章4</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/104\">新文章5</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/105\">新文章6</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/106\">新文章7</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/107\">新文章8</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/108\">新文章9</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/109\">新文章10</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/110\">新文章11</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/111\">新文章12</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/112\">新文章13</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/113\">新文章14</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/114\">新文章15</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/115\">新文章16</a></div></div></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic22-061-below80", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://juejin.cn/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 16, + "hosts": [ + "juejin.cn" + ], + "entry_path": "/post/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".entry-list .entry a.title" + } + } + }, + "artifact": "<main><div class=\"entry-list\"><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章1</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/101\">新文章2</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/102\">新文章3</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/103\">新文章4</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/104\">新文章5</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/105\">新文章6</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/106\">新文章7</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/107\">新文章8</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/108\">新文章9</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/109\">新文章10</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/110\">新文章11</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/111\">新文章12</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/112\">新文章13</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/113\">新文章14</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/114\">新文章15</a></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-061-duplicate", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://juejin.cn/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 16, + "hosts": [ + "juejin.cn" + ], + "entry_path": "/post/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".entry-list .entry a.title" + } + } + }, + "artifact": "<main><div class=\"entry-list\"><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章1</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章2</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章3</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章4</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章5</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章6</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章7</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章8</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章9</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章10</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章11</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章12</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章13</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章14</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章15</a></div><div class=\"entry\"><a class=\"title\" href=\"/post/100\">新文章16</a></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-061-wrong-scope", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://juejin.cn/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 16, + "hosts": [ + "juejin.cn" + ], + "entry_path": "/post/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".entry-list .entry a.title" + } + } + }, + "artifact": "<main><div class=\"entry-list\"><div class=\"entry\"><a class=\"title\" href=\"/tag/100\">新文章1</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/101\">新文章2</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/102\">新文章3</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/103\">新文章4</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/104\">新文章5</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/105\">新文章6</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/106\">新文章7</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/107\">新文章8</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/108\">新文章9</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/109\">新文章10</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/110\">新文章11</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/111\">新文章12</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/112\">新文章13</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/113\">新文章14</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/114\">新文章15</a></div><div class=\"entry\"><a class=\"title\" href=\"/tag/115\">新文章16</a></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-037-prefetch-only", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P<item>#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "<main><div class=\"q-virtual-scroll\"><div class=\"q-virtual-scroll__content\"><div class=\"q-item\">#20 - Option 21</div><div class=\"q-item\">#21 - Option 22</div><div class=\"q-item\">#22 - Option 23</div><div class=\"q-item\">#23 - Option 24</div><div class=\"q-item\">#24 - Option 25</div><div class=\"q-item\">#25 - Option 26</div><div class=\"q-item\">#26 - Option 27</div><div class=\"q-item\">#27 - Option 28</div></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-030-prefetch-only", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primevue.dev/virtualscroller/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "<main><div class=\"doc-section-demo\"><div class=\"p-virtualscroller-content\"><div>Item #20</div><div>Item #21</div><div>Item #22</div><div>Item #23</div></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-032-prefetch-only", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primeng.dev/virtualscroller", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P<item>(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "<main><div class=\"doc-section-demo\"><div class=\"p-virtualscroller-content\"><div>Item #20</div><div>Item #21</div><div>Item #22</div><div>Item #23</div></div></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-copy-marker-complete", + "category": "format-normalization", + "root_cause": "table_auxiliary_copy_markers", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "date", + "label": "Date", + "type": "date_ordinal_en" + }, + { + "name": "state", + "label": "State", + "type": "text", + "pattern": "Online|Offline" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "(?m)^(?=\\| Name \\|)", + "end_pattern": "END", + "plain_row_pattern": "(?P<name>[A-Za-z]+) (?P<date>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2}) (?P<state>Online|Offline)", + "plain_text_ignored_patterns": [ + "[↦↵](?=[ \\t]*(?:\\||$))", + "\\(external, opens in a new tab or window\\)" + ] + } + } + }, + "artifact": "| Name↦ | Date↦ | State↵ |\n| --- | --- | --- |\n| Ada(external, opens in a new tab or window)↦ | 29th Sep 25↦ | Online↵ |\nEND", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-copy-marker-bad-date", + "category": "format-normalization", + "root_cause": "table_auxiliary_copy_markers", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "date", + "label": "Date", + "type": "date_ordinal_en" + }, + { + "name": "state", + "label": "State", + "type": "text", + "pattern": "Online|Offline" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "(?m)^(?=\\| Name \\|)", + "end_pattern": "END", + "plain_row_pattern": "(?P<name>[A-Za-z]+) (?P<date>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2}) (?P<state>Online|Offline)", + "plain_text_ignored_patterns": [ + "[↦↵](?=[ \\t]*(?:\\||$))", + "\\(external, opens in a new tab or window\\)" + ] + } + } + }, + "artifact": "| Name↦ | Date↦ | State↵ |\n| --- | --- | --- |\n| Ada↦ | 32nd Sep 25↦ | Online↵ |\nEND", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-copy-marker-missing-field", + "category": "format-normalization", + "root_cause": "table_auxiliary_copy_markers", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "date", + "label": "Date", + "type": "date_ordinal_en" + }, + { + "name": "state", + "label": "State", + "type": "text", + "pattern": "Online|Offline" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "(?m)^(?=\\| Name \\|)", + "end_pattern": "END", + "plain_row_pattern": "(?P<name>[A-Za-z]+) (?P<date>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2}) (?P<state>Online|Offline)", + "plain_text_ignored_patterns": [ + "[↦↵](?=[ \\t]*(?:\\||$))", + "\\(external, opens in a new tab or window\\)" + ] + } + } + }, + "artifact": "| Name↦ | Date↦ | State↵ |\n| --- | --- | --- |\n| Ada↦ | ↦ | Online↵ |\nEND", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-copy-marker-literal-arrow", + "category": "format-normalization", + "root_cause": "table_auxiliary_copy_markers", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "date", + "label": "Date", + "type": "date_ordinal_en" + }, + { + "name": "state", + "label": "State", + "type": "text", + "pattern": "Online|Offline" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "(?m)^(?=\\| Name \\|)", + "end_pattern": "END", + "plain_row_pattern": "(?P<name>[A-Za-z]+) (?P<date>\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2}) (?P<state>Online|Offline)", + "plain_text_ignored_patterns": [ + "[↦↵](?=[ \\t]*(?:\\||$))", + "\\(external, opens in a new tab or window\\)" + ] + } + } + }, + "artifact": "| Name↦ | Date↦ | State↵ |\n| --- | --- | --- |\n| Ada↦ | 29th Sep 25↦ | On↦line↵ |\nEND", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-compact-cell-joined", + "category": "format-normalization", + "root_cause": "adjacent_cell_elements_and_empty_column_preservation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text", + "compact_whitespace": true + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" + } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category\n| Product | Category | Price | Rating | Status |\n| --- | --- | --- | --- | --- |\n| Bamboo Watchf230fh0g3 | Accessories | $65 | | In Stock |\nPREVIEW", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-compact-cell-wrong-id", + "category": "format-normalization", + "root_cause": "adjacent_cell_elements_and_empty_column_preservation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text", + "compact_whitespace": true + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" + } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category\n| Product | Category | Price | Rating | Status |\n| --- | --- | --- | --- | --- |\n| Bamboo Watchwrongid | Accessories | $65 | | In Stock |\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-compact-cell-wrong-price", + "category": "format-normalization", + "root_cause": "adjacent_cell_elements_and_empty_column_preservation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text", + "compact_whitespace": true + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" + } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P<product>[^\\n]+?\\s+[a-z0-9]{9})\\s+(?P<category>Accessories|Fitness|Clothing)\\s+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?P<status>In Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P<product>[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?P<category>Accessories|Fitness|Clothing)[\\s]+(?P<price>\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?P<status>In Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category\n| Product | Category | Price | Rating | Status |\n| --- | --- | --- | --- | --- |\n| Bamboo Watchf230fh0g3 | Accessories | $99 | | In Stock |\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-037-next-section-excluded", + "category": "structured-extraction", + "root_cause": "adjacent_virtual_demo_scope_boundary", + "failure_mode": "false-negative", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P<item>#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "### Basic\n\n#0 - Option 1\n#1 - Option 2\n#2 - Option 3\n#3 - Option 4\n#4 - Option 5\n#5 - Option 6\n#6 - Option 7\n#7 - Option 8\n\n### Horizontal\n\n#0 - Option 1\n#1 - Option 2\n#2 - Option 3\n#3 - Option 4\n#4 - Option 5\n#5 - Option 6\n#6 - Option 7\n#7 - Option 8", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic22-037-next-section-cannot-fill", + "category": "structured-extraction", + "root_cause": "adjacent_virtual_demo_scope_boundary", + "failure_mode": "false-positive", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" + } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P<item>#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "### Basic\n\n#0 - Option 1\n#1 - Option 2\n\n### Horizontal\n\n#0 - Option 1\n#1 - Option 2\n#2 - Option 3\n#3 - Option 4\n#4 - Option 5\n#5 - Option 6\n#6 - Option 7\n#7 - Option 8", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-visible-text-visible-title", + "category": "structured-extraction", + "root_cause": "image_alt_is_not_visible_record_name", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty" + } + }, + "artifact": "[![Decorative logo](/logo.png)Actual title](/item/1)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "named-visible-text-image-only", + "category": "structured-extraction", + "root_cause": "image_alt_is_not_visible_record_name", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty" + } + }, + "artifact": "[![Actual title](/logo.png)](/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-visible-text-image-plus-action", + "category": "structured-extraction", + "root_cause": "image_alt_is_not_visible_record_name", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty" + } + }, + "artifact": "[![Actual title](/logo.png)Read more](/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "joyn-completed-window-67", + "category": "structured-extraction", + "root_cause": "premature_partial_dynamic_collection_denominator", + "failure_mode": "false-negative", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 67, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "<main><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-0\"><h3>Movie 0</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-1\"><h3>Movie 1</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-2\"><h3>Movie 2</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-3\"><h3>Movie 3</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-4\"><h3>Movie 4</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-5\"><h3>Movie 5</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-6\"><h3>Movie 6</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-7\"><h3>Movie 7</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-8\"><h3>Movie 8</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-9\"><h3>Movie 9</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-10\"><h3>Movie 10</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-11\"><h3>Movie 11</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-12\"><h3>Movie 12</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-13\"><h3>Movie 13</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-14\"><h3>Movie 14</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-15\"><h3>Movie 15</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-16\"><h3>Movie 16</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-17\"><h3>Movie 17</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-18\"><h3>Movie 18</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-19\"><h3>Movie 19</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-20\"><h3>Movie 20</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-21\"><h3>Movie 21</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-22\"><h3>Movie 22</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-23\"><h3>Movie 23</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-24\"><h3>Movie 24</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-25\"><h3>Movie 25</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-26\"><h3>Movie 26</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-27\"><h3>Movie 27</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-28\"><h3>Movie 28</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-29\"><h3>Movie 29</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-30\"><h3>Movie 30</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-31\"><h3>Movie 31</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-32\"><h3>Movie 32</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-33\"><h3>Movie 33</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-34\"><h3>Movie 34</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-35\"><h3>Movie 35</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-36\"><h3>Movie 36</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-37\"><h3>Movie 37</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-38\"><h3>Movie 38</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-39\"><h3>Movie 39</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-40\"><h3>Movie 40</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-41\"><h3>Movie 41</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-42\"><h3>Movie 42</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-43\"><h3>Movie 43</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-44\"><h3>Movie 44</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-45\"><h3>Movie 45</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-46\"><h3>Movie 46</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-47\"><h3>Movie 47</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-48\"><h3>Movie 48</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-49\"><h3>Movie 49</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-50\"><h3>Movie 50</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-51\"><h3>Movie 51</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-52\"><h3>Movie 52</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-53\"><h3>Movie 53</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-54\"><h3>Movie 54</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-55\"><h3>Movie 55</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-56\"><h3>Movie 56</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-57\"><h3>Movie 57</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-58\"><h3>Movie 58</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-59\"><h3>Movie 59</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-60\"><h3>Movie 60</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-61\"><h3>Movie 61</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-62\"><h3>Movie 62</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-63\"><h3>Movie 63</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-64\"><h3>Movie 64</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-65\"><h3>Movie 65</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-66\"><h3>Movie 66</h3></a></div></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "joyn-completed-window-27", + "category": "structured-extraction", + "root_cause": "premature_partial_dynamic_collection_denominator", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 67, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "<main><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-0\"><h3>Movie 0</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-1\"><h3>Movie 1</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-2\"><h3>Movie 2</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-3\"><h3>Movie 3</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-4\"><h3>Movie 4</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-5\"><h3>Movie 5</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-6\"><h3>Movie 6</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-7\"><h3>Movie 7</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-8\"><h3>Movie 8</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-9\"><h3>Movie 9</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-10\"><h3>Movie 10</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-11\"><h3>Movie 11</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-12\"><h3>Movie 12</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-13\"><h3>Movie 13</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-14\"><h3>Movie 14</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-15\"><h3>Movie 15</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-16\"><h3>Movie 16</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-17\"><h3>Movie 17</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-18\"><h3>Movie 18</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-19\"><h3>Movie 19</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-20\"><h3>Movie 20</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-21\"><h3>Movie 21</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-22\"><h3>Movie 22</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-23\"><h3>Movie 23</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-24\"><h3>Movie 24</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-25\"><h3>Movie 25</h3></a></div><div class=\"CardStandard_StandardCardWrapper\"><a data-testid=\"LL--movie\" href=\"/filme/example-26\"><h3>Movie 26</h3></a></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-owned-software-20", + "category": "structured-extraction", + "root_cause": "adjacent_search_result_regions", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/jp/search/index.html?q=Nintendo", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 20, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/jp/games/(?:switch2|switch)/[A-Za-z0-9-]+/index\\.html|/item/software/[A-Za-z0-9]+)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)", + "start_pattern": "(?m)^#{1,6}[ \\t]+ゲームソフトをさがす[ \\t]*$", + "end_pattern": "表示価格はすべて" + } + } + }, + "artifact": "### ページをさがす\n[Official website 0](https://www.nintendo.com/jp/games/switch2/game0/index.html)\n[Official website 1](https://www.nintendo.com/jp/games/switch2/game1/index.html)\n[Official website 2](https://www.nintendo.com/jp/games/switch2/game2/index.html)\n[Official website 3](https://www.nintendo.com/jp/games/switch2/game3/index.html)\n[Official website 4](https://www.nintendo.com/jp/games/switch2/game4/index.html)\n[Official website 5](https://www.nintendo.com/jp/games/switch2/game5/index.html)\n[Official website 6](https://www.nintendo.com/jp/games/switch2/game6/index.html)\n[Official website 7](https://www.nintendo.com/jp/games/switch2/game7/index.html)\n[Official website 8](https://www.nintendo.com/jp/games/switch2/game8/index.html)\n[Official website 9](https://www.nintendo.com/jp/games/switch2/game9/index.html)\n[Official website 10](https://www.nintendo.com/jp/games/switch2/game10/index.html)\n[Official website 11](https://www.nintendo.com/jp/games/switch2/game11/index.html)\n[Official website 12](https://www.nintendo.com/jp/games/switch2/game12/index.html)\n[Official website 13](https://www.nintendo.com/jp/games/switch2/game13/index.html)\n[Official website 14](https://www.nintendo.com/jp/games/switch2/game14/index.html)\n[Official website 15](https://www.nintendo.com/jp/games/switch2/game15/index.html)\n[Official website 16](https://www.nintendo.com/jp/games/switch2/game16/index.html)\n[Official website 17](https://www.nintendo.com/jp/games/switch2/game17/index.html)\n[Official website 18](https://www.nintendo.com/jp/games/switch2/game18/index.html)\n[Official website 19](https://www.nintendo.com/jp/games/switch2/game19/index.html)\n### ゲームソフトをさがす\n[Game 0](https://store-jp.nintendo.com/item/software/D70010000000000)\n[Game 1](https://store-jp.nintendo.com/item/software/D70010000000001)\n[Game 2](https://store-jp.nintendo.com/item/software/D70010000000002)\n[Game 3](https://store-jp.nintendo.com/item/software/D70010000000003)\n[Game 4](https://store-jp.nintendo.com/item/software/D70010000000004)\n[Game 5](https://store-jp.nintendo.com/item/software/D70010000000005)\n[Game 6](https://store-jp.nintendo.com/item/software/D70010000000006)\n[Game 7](https://store-jp.nintendo.com/item/software/D70010000000007)\n[Game 8](https://store-jp.nintendo.com/item/software/D70010000000008)\n[Game 9](https://store-jp.nintendo.com/item/software/D70010000000009)\n[Game 10](https://store-jp.nintendo.com/item/software/D70010000000010)\n[Game 11](https://store-jp.nintendo.com/item/software/D70010000000011)\n[Game 12](https://store-jp.nintendo.com/item/software/D70010000000012)\n[Game 13](https://store-jp.nintendo.com/item/software/D70010000000013)\n[Game 14](https://store-jp.nintendo.com/item/software/D70010000000014)\n[Game 15](https://store-jp.nintendo.com/item/software/D70010000000015)\n[Game 16](https://store-jp.nintendo.com/item/software/D70010000000016)\n[Game 17](https://store-jp.nintendo.com/item/software/D70010000000017)\n[Game 18](https://store-jp.nintendo.com/item/software/D70010000000018)\n[Game 19](https://store-jp.nintendo.com/item/software/D70010000000019)\n表示価格はすべて", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nintendo-owned-software-2", + "category": "structured-extraction", + "root_cause": "adjacent_search_result_regions", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/jp/search/index.html?q=Nintendo", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 20, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/jp/games/(?:switch2|switch)/[A-Za-z0-9-]+/index\\.html|/item/software/[A-Za-z0-9]+)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)", + "start_pattern": "(?m)^#{1,6}[ \\t]+ゲームソフトをさがす[ \\t]*$", + "end_pattern": "表示価格はすべて" + } + } + }, + "artifact": "### ページをさがす\n[Official website 0](https://www.nintendo.com/jp/games/switch2/game0/index.html)\n[Official website 1](https://www.nintendo.com/jp/games/switch2/game1/index.html)\n[Official website 2](https://www.nintendo.com/jp/games/switch2/game2/index.html)\n[Official website 3](https://www.nintendo.com/jp/games/switch2/game3/index.html)\n[Official website 4](https://www.nintendo.com/jp/games/switch2/game4/index.html)\n[Official website 5](https://www.nintendo.com/jp/games/switch2/game5/index.html)\n[Official website 6](https://www.nintendo.com/jp/games/switch2/game6/index.html)\n[Official website 7](https://www.nintendo.com/jp/games/switch2/game7/index.html)\n[Official website 8](https://www.nintendo.com/jp/games/switch2/game8/index.html)\n[Official website 9](https://www.nintendo.com/jp/games/switch2/game9/index.html)\n[Official website 10](https://www.nintendo.com/jp/games/switch2/game10/index.html)\n[Official website 11](https://www.nintendo.com/jp/games/switch2/game11/index.html)\n[Official website 12](https://www.nintendo.com/jp/games/switch2/game12/index.html)\n[Official website 13](https://www.nintendo.com/jp/games/switch2/game13/index.html)\n[Official website 14](https://www.nintendo.com/jp/games/switch2/game14/index.html)\n[Official website 15](https://www.nintendo.com/jp/games/switch2/game15/index.html)\n[Official website 16](https://www.nintendo.com/jp/games/switch2/game16/index.html)\n[Official website 17](https://www.nintendo.com/jp/games/switch2/game17/index.html)\n[Official website 18](https://www.nintendo.com/jp/games/switch2/game18/index.html)\n[Official website 19](https://www.nintendo.com/jp/games/switch2/game19/index.html)\n### ゲームソフトをさがす\n[Game 0](https://store-jp.nintendo.com/item/software/D70010000000000)\n[Game 1](https://store-jp.nintendo.com/item/software/D70010000000001)\n表示価格はすべて", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-062-1", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title B](/item/92)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-062-2", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title A](/item/91)\n[Invalid](/about)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-067-3", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title B](/item/92)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-067-4", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title A](/item/91)\n[Invalid](/about)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-045-5", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title B](/item/92)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-045-6", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title A](/item/91)\n[Invalid](/about)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-095-7", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title B](/item/92)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-095-8", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title A](/item/91)\n[Invalid](/about)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-011-9", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Product", + "type": "text" + }, + { + "name": "f1", + "label": "Category", + "type": "text" + }, + { + "name": "f2", + "label": "Price", + "type": "number" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Product(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | 1.5 |\n| Beta | Beta | 1.5 |\n| Gamma | Gamma | 1.5 |\n| Delta | Delta | 1.5 |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-011-10", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Product", + "type": "text" + }, + { + "name": "f1", + "label": "Category", + "type": "text" + }, + { + "name": "f2", + "label": "Price", + "type": "number" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Product(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | |\n| Beta | Beta | |\n| Gamma | Gamma | |\n| Delta | Delta | |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-011-11", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Product", + "type": "text" + }, + { + "name": "f1", + "label": "Category", + "type": "text" + }, + { + "name": "f2", + "label": "Price", + "type": "number" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Product(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | 1.5 |\n| Alpha | Alpha | 1.5 |\n| Alpha | Alpha | 1.5 |\n| Alpha | Alpha | 1.5 |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-011-12", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Product", + "type": "text" + }, + { + "name": "f1", + "label": "Category", + "type": "text" + }, + { + "name": "f2", + "label": "Price", + "type": "number" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Product(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | 1.5 |\n\n| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | 1.5 |\n| Beta | Beta | 1.5 |\n| Gamma | Gamma | 1.5 |\n| Delta | Delta | 1.5 |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-024-13", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + }, + { + "name": "f2", + "label": "Phone Number", + "type": "token", + "pattern": "[+()\\d][\\d ()+.-]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Email | Phone Number |\n| --- | --- | --- |\n| Alpha | person@example.org | (123) 456 789 |\n| Beta | person@example.org | (123) 456 789 |\n| Gamma | person@example.org | (123) 456 789 |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-024-14", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + }, + { + "name": "f2", + "label": "Phone Number", + "type": "token", + "pattern": "[+()\\d][\\d ()+.-]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Email | Phone Number |\n| --- | --- | --- |\n| Alpha | person@example.org | |\n| Beta | person@example.org | |\n| Gamma | person@example.org | |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-024-15", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + }, + { + "name": "f2", + "label": "Phone Number", + "type": "token", + "pattern": "[+()\\d][\\d ()+.-]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Email | Phone Number |\n| --- | --- | --- |\n| Alpha | person@example.org | (123) 456 789 |\n| Alpha | person@example.org | (123) 456 789 |\n| Alpha | person@example.org | (123) 456 789 |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-024-16", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + }, + { + "name": "f2", + "label": "Phone Number", + "type": "token", + "pattern": "[+()\\d][\\d ()+.-]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Email | Phone Number |\n| --- | --- | --- |\n| Alpha | person@example.org | (123) 456 789 |\n\n| Name | Email | Phone Number |\n| --- | --- | --- |\n| Alpha | person@example.org | (123) 456 789 |\n| Beta | person@example.org | (123) 456 789 |\n| Gamma | person@example.org | (123) 456 789 |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-029-17", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Size", + "type": "token", + "pattern": "\\d+(?:\\.\\d+)?\\s*[kKmMgG]?[bB]" + }, + { + "name": "f2", + "label": "Type", + "type": "text" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Size | Type |\n| --- | --- | --- |\n| Alpha | 25kb | Alpha |\n| Beta | 25kb | Beta |\n| Gamma | 25kb | Gamma |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-029-18", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Size", + "type": "token", + "pattern": "\\d+(?:\\.\\d+)?\\s*[kKmMgG]?[bB]" + }, + { + "name": "f2", + "label": "Type", + "type": "text" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Size | Type |\n| --- | --- | --- |\n| Alpha | 25kb | |\n| Beta | 25kb | |\n| Gamma | 25kb | |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-029-19", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Size", + "type": "token", + "pattern": "\\d+(?:\\.\\d+)?\\s*[kKmMgG]?[bB]" + }, + { + "name": "f2", + "label": "Type", + "type": "text" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Size | Type |\n| --- | --- | --- |\n| Alpha | 25kb | Alpha |\n| Alpha | 25kb | Alpha |\n| Alpha | 25kb | Alpha |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-029-20", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Size", + "type": "token", + "pattern": "\\d+(?:\\.\\d+)?\\s*[kKmMgG]?[bB]" + }, + { + "name": "f2", + "label": "Type", + "type": "text" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Size | Type |\n| --- | --- | --- |\n| Alpha | 25kb | Alpha |\n\n| Name | Size | Type |\n| --- | --- | --- |\n| Alpha | 25kb | Alpha |\n| Beta | 25kb | Beta |\n| Gamma | 25kb | Gamma |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-038-21", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "No.", + "type": "integer" + }, + { + "name": "f1", + "label": "Name", + "type": "text" + }, + { + "name": "f2", + "label": "Weight", + "type": "number" + }, + { + "name": "f3", + "label": "Symbol", + "type": "token", + "pattern": "[A-Z][a-z]?" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*No\\.(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| No. | Name | Weight | Symbol |\n| --- | --- | --- | --- |\n| 1 | Alpha | 1.5 | He |\n| 2 | Beta | 1.5 | He |\n| 3 | Gamma | 1.5 | He |\n| 4 | Delta | 1.5 | He |\n| 5 | Epsilon | 1.5 | He |\n| 6 | Zeta | 1.5 | He |\n| 7 | Eta | 1.5 | He |\n| 8 | Theta | 1.5 | He |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-038-22", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "No.", + "type": "integer" + }, + { + "name": "f1", + "label": "Name", + "type": "text" + }, + { + "name": "f2", + "label": "Weight", + "type": "number" + }, + { + "name": "f3", + "label": "Symbol", + "type": "token", + "pattern": "[A-Z][a-z]?" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*No\\.(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| No. | Name | Weight | Symbol |\n| --- | --- | --- | --- |\n| 1 | Alpha | 1.5 | |\n| 2 | Beta | 1.5 | |\n| 3 | Gamma | 1.5 | |\n| 4 | Delta | 1.5 | |\n| 5 | Epsilon | 1.5 | |\n| 6 | Zeta | 1.5 | |\n| 7 | Eta | 1.5 | |\n| 8 | Theta | 1.5 | |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-038-23", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "No.", + "type": "integer" + }, + { + "name": "f1", + "label": "Name", + "type": "text" + }, + { + "name": "f2", + "label": "Weight", + "type": "number" + }, + { + "name": "f3", + "label": "Symbol", + "type": "token", + "pattern": "[A-Z][a-z]?" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*No\\.(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| No. | Name | Weight | Symbol |\n| --- | --- | --- | --- |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-038-24", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "No.", + "type": "integer" + }, + { + "name": "f1", + "label": "Name", + "type": "text" + }, + { + "name": "f2", + "label": "Weight", + "type": "number" + }, + { + "name": "f3", + "label": "Symbol", + "type": "token", + "pattern": "[A-Z][a-z]?" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*No\\.(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| No. | Name | Weight | Symbol |\n| --- | --- | --- | --- |\n| 1 | Alpha | 1.5 | He |\n\n| No. | Name | Weight | Symbol |\n| --- | --- | --- | --- |\n| 1 | Alpha | 1.5 | He |\n| 2 | Beta | 1.5 | He |\n| 3 | Gamma | 1.5 | He |\n| 4 | Delta | 1.5 | He |\n| 5 | Epsilon | 1.5 | He |\n| 6 | Zeta | 1.5 | He |\n| 7 | Eta | 1.5 | He |\n| 8 | Theta | 1.5 | He |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-041-25", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Role", + "type": "text" + }, + { + "name": "f2", + "label": "Status", + "type": "text" + }, + { + "name": "f3", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Role | Status | Email |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Beta | Beta | Beta | person@example.org |\n| Gamma | Gamma | Gamma | person@example.org |\n| Delta | Delta | Delta | person@example.org |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-041-26", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Role", + "type": "text" + }, + { + "name": "f2", + "label": "Status", + "type": "text" + }, + { + "name": "f3", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Role | Status | Email |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | |\n| Beta | Beta | Beta | |\n| Gamma | Gamma | Gamma | |\n| Delta | Delta | Delta | |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-041-27", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Role", + "type": "text" + }, + { + "name": "f2", + "label": "Status", + "type": "text" + }, + { + "name": "f3", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Role | Status | Email |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Alpha | Alpha | Alpha | person@example.org |\n| Alpha | Alpha | Alpha | person@example.org |\n| Alpha | Alpha | Alpha | person@example.org |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-041-28", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Role", + "type": "text" + }, + { + "name": "f2", + "label": "Status", + "type": "text" + }, + { + "name": "f3", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Role | Status | Email |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n\n| Name | Role | Status | Email |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Beta | Beta | Beta | person@example.org |\n| Gamma | Gamma | Gamma | person@example.org |\n| Delta | Delta | Delta | person@example.org |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-058-29", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "申请人", + "type": "text" + }, + { + "name": "f1", + "label": "审批状态", + "type": "text" + }, + { + "name": "f2", + "label": "签署方式", + "type": "text" + }, + { + "name": "f3", + "label": "电子邮件", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*申请人(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| 申请人 | 审批状态 | 签署方式 | 电子邮件 |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Beta | Beta | Beta | person@example.org |\n| Gamma | Gamma | Gamma | person@example.org |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-058-30", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "申请人", + "type": "text" + }, + { + "name": "f1", + "label": "审批状态", + "type": "text" + }, + { + "name": "f2", + "label": "签署方式", + "type": "text" + }, + { + "name": "f3", + "label": "电子邮件", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*申请人(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| 申请人 | 审批状态 | 签署方式 | 电子邮件 |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | |\n| Beta | Beta | Beta | |\n| Gamma | Gamma | Gamma | |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-058-31", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "申请人", + "type": "text" + }, + { + "name": "f1", + "label": "审批状态", + "type": "text" + }, + { + "name": "f2", + "label": "签署方式", + "type": "text" + }, + { + "name": "f3", + "label": "电子邮件", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*申请人(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| 申请人 | 审批状态 | 签署方式 | 电子邮件 |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Alpha | Alpha | Alpha | person@example.org |\n| Alpha | Alpha | Alpha | person@example.org |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-058-32", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "申请人", + "type": "text" + }, + { + "name": "f1", + "label": "审批状态", + "type": "text" + }, + { + "name": "f2", + "label": "签署方式", + "type": "text" + }, + { + "name": "f3", + "label": "电子邮件", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*申请人(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| 申请人 | 审批状态 | 签署方式 | 电子邮件 |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n\n| 申请人 | 审批状态 | 签署方式 | 电子邮件 |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Beta | Beta | Beta | person@example.org |\n| Gamma | Gamma | Gamma | person@example.org |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-016-33", + "category": "structured-extraction", + "root_cause": "independent_body_scope_omission_or_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/body", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Example tree" + ] + } + ] + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "example tree apples flowers", + "source_text": "Example tree\n\nApples\n\nFlowers", + "segmentation_method": "sections" + } + ] + } + }, + "artifact": "Example tree\nApples\nFlowers", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-016-34", + "category": "structured-extraction", + "root_cause": "independent_body_scope_omission_or_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/body", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Example tree" + ] + } + ] + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "example tree apples flowers", + "source_text": "Example tree\n\nApples\n\nFlowers", + "segmentation_method": "sections" + } + ] + } + }, + "artifact": "Navigation\nExample tree", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-039-35", + "category": "structured-extraction", + "root_cause": "independent_body_scope_omission_or_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/body", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Example tree" + ] + } + ] + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "example tree apples flowers", + "source_text": "Example tree\n\nApples\n\nFlowers", + "segmentation_method": "sections" + } + ] + } + }, + "artifact": "Example tree\nApples\nFlowers", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-039-36", + "category": "structured-extraction", + "root_cause": "independent_body_scope_omission_or_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/body", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Example tree" + ] + } + ] + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "example tree apples flowers", + "source_text": "Example tree\n\nApples\n\nFlowers", + "segmentation_method": "sections" + } + ] + } + }, + "artifact": "Navigation\nExample tree", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-visible-html-image-only", + "category": "structured-extraction", + "root_cause": "image_metadata_is_not_visible_record_name", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "<main><a href=\"/item/1\"><img alt=\"Real title\"></a></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-visible-html-visible-name", + "category": "structured-extraction", + "root_cause": "image_metadata_is_not_visible_record_name", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "<main><a href=\"/item/1\"><img alt=\"Poster\">Real title</a></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-nested-anchor-keep-navigation", + "category": "structured-extraction", + "root_cause": "decomposed_descendant_anchor_in_nonrecord_text", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "title_selector": "h3", + "link_selector": "a[href]" + }, + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Navigation" + ], + "match_scope": "non_record_lines" + } + ] + } + }, + "artifact": "<main><nav>Navigation</nav><article><a href=\"/item/1\"><h3>Navigation</h3><a href=\"/item/1\">More</a></a></article></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-nested-anchor-record-is-not-navigation", + "category": "structured-extraction", + "root_cause": "decomposed_descendant_anchor_in_nonrecord_text", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "title_selector": "h3", + "link_selector": "a[href]" + }, + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Navigation" + ], + "match_scope": "non_record_lines" + } + ] + } + }, + "artifact": "<main><nav></nav><article><a href=\"/item/1\"><h3>Navigation</h3><a href=\"/item/1\">More</a></a></article></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-tree-separated", + "category": "body-scope", + "root_cause": "separate_visible_nodes_not_cross_node_ngrams", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/tree", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Tree with dynamic data" + ] + } + ] + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 3, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "visible-block-1", + "text": "tree with dynamic data", + "source_text": "tree with dynamic data", + "segmentation_method": "balanced_tokens_fallback" + }, + { + "id": "visible-block-2", + "text": "fruits", + "source_text": "fruits", + "segmentation_method": "balanced_tokens_fallback" + }, + { + "id": "visible-block-3", + "text": "vegetables", + "source_text": "vegetables", + "segmentation_method": "balanced_tokens_fallback" + } + ] + } + }, + "artifact": "Tree with dynamic data\nDecoration\nFruits\nDecoration\nVegetables", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-tree-missing-node", + "category": "body-scope", + "root_cause": "separate_visible_nodes_not_cross_node_ngrams", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/tree", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Tree with dynamic data" + ] + } + ] + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 3, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "visible-block-1", + "text": "tree with dynamic data", + "source_text": "tree with dynamic data", + "segmentation_method": "balanced_tokens_fallback" + }, + { + "id": "visible-block-2", + "text": "fruits", + "source_text": "fruits", + "segmentation_method": "balanced_tokens_fallback" + }, + { + "id": "visible-block-3", + "text": "vegetables", + "source_text": "vegetables", + "segmentation_method": "balanced_tokens_fallback" + } + ] + } + }, + "artifact": "Tree with dynamic data\nDecoration\nFruits", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-inline-html-real-markdown", + "category": "structured-extraction", + "root_cause": "inline_html_documentation_is_not_a_rendered_table", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Role", + "type": "text" + }, + { + "name": "f2", + "label": "Status", + "type": "text" + }, + { + "name": "f3", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "Documentation\n| Name | Role | Status | Email |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Beta | Beta | Beta | person@example.org |\n| Gamma | Gamma | Gamma | person@example.org |\n| Delta | Delta | Delta | person@example.org |\n\nThe `<table>` contains a `<tr>` and a `<td>`.", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-inline-html-example-only", + "category": "structured-extraction", + "root_cause": "inline_html_documentation_is_not_a_rendered_table", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Role", + "type": "text" + }, + { + "name": "f2", + "label": "Status", + "type": "text" + }, + { + "name": "f3", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P<f0>[^\t\n]+)\\t(?P<f1>[^\t\n]+)\\t(?P<f2>[^\t\n]+)\\t(?P<f3>[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "Documentation\nThe `<table>` contains a `<tr>` and a `<td>`.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-list-boundary-045-inside", + "category": "structured-extraction", + "root_cause": "markdown_section_ownership_matches_source_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "start_pattern": "(?m)^#{1,6}[ \\t]+NBA Standings[ \\t]*$", + "end_pattern": "(?m)^#{1,6}[ \\t]+Legend[ \\t]*$" + } + } + }, + "artifact": "# NBA Standings\n[New A](/item/1)\n[New B](/item/2)\n## Legend", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-list-boundary-045-outside", + "category": "structured-extraction", + "root_cause": "markdown_section_ownership_matches_source_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "start_pattern": "(?m)^#{1,6}[ \\t]+NBA Standings[ \\t]*$", + "end_pattern": "(?m)^#{1,6}[ \\t]+Legend[ \\t]*$" + } + } + }, + "artifact": "[Outside A](/item/1)\n[Outside B](/item/2)\n# NBA Standings\n[One only](/item/3)\n## Legend\n[Outside C](/item/4)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-list-boundary-067-inside", + "category": "structured-extraction", + "root_cause": "markdown_section_ownership_matches_source_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "start_pattern": "(?m)^#{1,6}[ \\t]+トピックス[ \\t]*$", + "end_pattern": "(?m)^#{1,6}[ \\t]+ヤフコメ トレンド[ \\t]*$" + } + } + }, + "artifact": "## トピックス\n[New A](/item/1)\n[New B](/item/2)\n## ヤフコメ トレンド", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-list-boundary-067-outside", + "category": "structured-extraction", + "root_cause": "markdown_section_ownership_matches_source_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "start_pattern": "(?m)^#{1,6}[ \\t]+トピックス[ \\t]*$", + "end_pattern": "(?m)^#{1,6}[ \\t]+ヤフコメ トレンド[ \\t]*$" + } + } + }, + "artifact": "[Outside A](/item/1)\n[Outside B](/item/2)\n## トピックス\n[One only](/item/3)\n## ヤフコメ トレンド\n[Outside C](/item/4)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-list-boundary-062-inside", + "category": "structured-extraction", + "root_cause": "markdown_section_ownership_matches_source_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "start_pattern": "(?m)^(?:推薦|推荐|\\[推荐\\]\\([^\\n]+\\))[ \\t]*$", + "end_pattern": "(?m)^(?:#{1,6}[ \\t]+)?派友在看[ \\t]*$" + } + } + }, + "artifact": "推荐\n[New A](/item/1)\n[New B](/item/2)\n派友在看", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-list-boundary-062-outside", + "category": "structured-extraction", + "root_cause": "markdown_section_ownership_matches_source_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "start_pattern": "(?m)^(?:推薦|推荐|\\[推荐\\]\\([^\\n]+\\))[ \\t]*$", + "end_pattern": "(?m)^(?:#{1,6}[ \\t]+)?派友在看[ \\t]*$" + } + } + }, + "artifact": "[Outside A](/item/1)\n[Outside B](/item/2)\n推荐\n[One only](/item/3)\n派友在看\n[Outside C](/item/4)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-caption-md", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$" + } + } + }, + "artifact": "START\n[Artwork title<br>7 Staffeln<br>](/serien/new-a)\n[Artwork title<br>Documentary<br>](/serien/new-b)\nEND", + "ok": true, + "expected_task_pass": true + }, + { + "id": "remaining7-caption-only-not-title", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$" + } + } + }, + "artifact": "START\n[<br>7 Staffeln<br>](/serien/new-a)\n[<br>Documentary<br>](/filme/new-b)\nEND", + "ok": true, + "expected_task_pass": true + }, + { + "id": "remaining7-bare-title", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$" + } + } + }, + "artifact": "START\n[Alpha](/serien/a)\n[Beta](/serien/b)\nEND", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-missing-caption", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$" + } + } + }, + "artifact": "START\n[Artwork title<br>7 Staffeln<br>](/serien/new-a)\n[Artwork title](/serien/new-b)\nEND", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-duplicate-caption-card", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$" + } + } + }, + "artifact": "START\n[Artwork title<br>7 Staffeln<br>](/serien/new-a)\n[Artwork title<br>7 Staffeln<br>](/serien/new-a)\nEND", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-wrong-caption-link", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$" + } + } + }, + "artifact": "START\n[Artwork title<br>7 Staffeln<br>](/serien/new-a)\n[Artwork title<br>Documentary<br>](/navigation/new-b)\nEND", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-outside-card-scope", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$" + } + } + }, + "artifact": "START\n[Artwork title<br>7 Staffeln<br>](/serien/new-a)\nEND\n[Artwork title<br>Documentary<br>](/serien/new-b)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-image-caption-only", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$" + } + } + }, + "artifact": "START\n[![7 Staffeln](/a.png)](/serien/a)\n[![8 Staffeln](/b.png)](/serien/b)\nEND", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-caption-html", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$" + } + } + }, + "artifact": "<main><div data-testid=\"CS\"><a data-testid=\"LL--series\" href=\"/serien/1\"><h3>Hidden artwork title</h3><p data-testid=\"CMRKCAPT\">1 Staffeln</p></a></div><div data-testid=\"CS\"><a data-testid=\"LL--series\" href=\"/serien/2\"><h3>Hidden artwork title</h3><p data-testid=\"CMRKCAPT\">2 Staffeln</p></a></div></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "remaining7-caption-html-missing", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$" + } + } + }, + "artifact": "<main><div data-testid=\"CS\"><a data-testid=\"LL--series\" href=\"/serien/1\"><h3>Hidden artwork title</h3><p data-testid=\"CMRKCAPT\">1 Staffeln</p></a></div><div data-testid=\"CS\"><a data-testid=\"LL--series\" href=\"/serien/2\"><h3>Hidden artwork title</h3></a></div></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-plan-md", + "category": "structured-extraction", + "root_cause": "same_card_name_price_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "name", + "label": "Plan", + "type": "text", + "selector": ":scope", + "own_text": true + }, + { + "name": "price", + "label": "Monthly starting price", + "type": "token", + "pattern": "\\d+,\\d{2}", + "selector": ":scope > div > p" + } + ], + "record_identity_fields": [ + "name" + ], + "duplicate_policy": "deduplicate_identical", + "table_scope": { + "root_selector": "#plans-tabs", + "root_index": 0, + "row_selector": "[id^=\"uolplay_\"]:has(> div > p)", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?:#{1,6}\\s+)?Planos UOL Play disponíveis\\s*$", + "end_pattern": "(?m)^(?:#{1,6}\\s+)?Assinaturas individuais\\s*$", + "plain_row_pattern": "(?P<name>UOL Play [^\\n]+?) A partir de R\\$ (?P<price>\\d+,\\d{2}) /mês", + "plain_multiline_row_pattern": "(?m)^(?P<name>UOL Play [^\\n]+)\\n+(?:[ \\t]*\\n)*A partir de R\\$[ \\t]*\\n+(?:[ \\t]*\\n)*(?P<price>\\d+,\\d{2})[ \\t]*\\n+(?:[ \\t]*\\n)*[ \\t]*/mês[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "## Planos UOL Play disponíveis\n\nUOL Play Fresh\n\nA partir de R$\n\n18,50\n\n/mês\n\nUOL Play Other\n\nA partir de R$\n\n78,20\n\n/mês\n#### Assinaturas individuais", + "ok": true, + "expected_task_pass": true + }, + { + "id": "remaining7-plan-html", + "category": "structured-extraction", + "root_cause": "same_card_name_price_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "name", + "label": "Plan", + "type": "text", + "selector": ":scope", + "own_text": true + }, + { + "name": "price", + "label": "Monthly starting price", + "type": "token", + "pattern": "\\d+,\\d{2}", + "selector": ":scope > div > p" + } + ], + "record_identity_fields": [ + "name" + ], + "duplicate_policy": "deduplicate_identical", + "table_scope": { + "root_selector": "#plans-tabs", + "root_index": 0, + "row_selector": "[id^=\"uolplay_\"]:has(> div > p)", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?:#{1,6}\\s+)?Planos UOL Play disponíveis\\s*$", + "end_pattern": "(?m)^(?:#{1,6}\\s+)?Assinaturas individuais\\s*$", + "plain_row_pattern": "(?P<name>UOL Play [^\\n]+?) A partir de R\\$ (?P<price>\\d+,\\d{2}) /mês", + "plain_multiline_row_pattern": "(?m)^(?P<name>UOL Play [^\\n]+)\\n+(?:[ \\t]*\\n)*A partir de R\\$[ \\t]*\\n+(?:[ \\t]*\\n)*(?P<price>\\d+,\\d{2})[ \\t]*\\n+(?:[ \\t]*\\n)*[ \\t]*/mês[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "<div id=\"plans-tabs\"><div id=\"uolplay_1\">UOL Play 1<div>A partir de R$<p>10,99</p>/mês</div></div><div id=\"uolplay_2\">UOL Play 2<div>A partir de R$<p>20,99</p>/mês</div></div></div>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "remaining7-plan-missing-price", + "category": "structured-extraction", + "root_cause": "same_card_name_price_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "name", + "label": "Plan", + "type": "text", + "selector": ":scope", + "own_text": true + }, + { + "name": "price", + "label": "Monthly starting price", + "type": "token", + "pattern": "\\d+,\\d{2}", + "selector": ":scope > div > p" + } + ], + "record_identity_fields": [ + "name" + ], + "duplicate_policy": "deduplicate_identical", + "table_scope": { + "root_selector": "#plans-tabs", + "root_index": 0, + "row_selector": "[id^=\"uolplay_\"]:has(> div > p)", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?:#{1,6}\\s+)?Planos UOL Play disponíveis\\s*$", + "end_pattern": "(?m)^(?:#{1,6}\\s+)?Assinaturas individuais\\s*$", + "plain_row_pattern": "(?P<name>UOL Play [^\\n]+?) A partir de R\\$ (?P<price>\\d+,\\d{2}) /mês", + "plain_multiline_row_pattern": "(?m)^(?P<name>UOL Play [^\\n]+)\\n+(?:[ \\t]*\\n)*A partir de R\\$[ \\t]*\\n+(?:[ \\t]*\\n)*(?P<price>\\d+,\\d{2})[ \\t]*\\n+(?:[ \\t]*\\n)*[ \\t]*/mês[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "Planos UOL Play disponíveis\n\nUOL Play Fresh\n\nA partir de R$\n\n18,50\n\n/mês\n\nUOL Play Other\n\nA partir de R$\n\n\n\n/mês\nAssinaturas individuais", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-plan-duplicate", + "category": "structured-extraction", + "root_cause": "same_card_name_price_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "name", + "label": "Plan", + "type": "text", + "selector": ":scope", + "own_text": true + }, + { + "name": "price", + "label": "Monthly starting price", + "type": "token", + "pattern": "\\d+,\\d{2}", + "selector": ":scope > div > p" + } + ], + "record_identity_fields": [ + "name" + ], + "duplicate_policy": "deduplicate_identical", + "table_scope": { + "root_selector": "#plans-tabs", + "root_index": 0, + "row_selector": "[id^=\"uolplay_\"]:has(> div > p)", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?:#{1,6}\\s+)?Planos UOL Play disponíveis\\s*$", + "end_pattern": "(?m)^(?:#{1,6}\\s+)?Assinaturas individuais\\s*$", + "plain_row_pattern": "(?P<name>UOL Play [^\\n]+?) A partir de R\\$ (?P<price>\\d+,\\d{2}) /mês", + "plain_multiline_row_pattern": "(?m)^(?P<name>UOL Play [^\\n]+)\\n+(?:[ \\t]*\\n)*A partir de R\\$[ \\t]*\\n+(?:[ \\t]*\\n)*(?P<price>\\d+,\\d{2})[ \\t]*\\n+(?:[ \\t]*\\n)*[ \\t]*/mês[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "Planos UOL Play disponíveis\n\nUOL Play Fresh\n\nA partir de R$\n\n18,50\n\n/mês\n\nUOL Play Fresh\n\nA partir de R$\n\n18,50\n\n/mês\nAssinaturas individuais", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-plan-cross-card", + "category": "structured-extraction", + "root_cause": "same_card_name_price_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "name", + "label": "Plan", + "type": "text", + "selector": ":scope", + "own_text": true + }, + { + "name": "price", + "label": "Monthly starting price", + "type": "token", + "pattern": "\\d+,\\d{2}", + "selector": ":scope > div > p" + } + ], + "record_identity_fields": [ + "name" + ], + "duplicate_policy": "deduplicate_identical", + "table_scope": { + "root_selector": "#plans-tabs", + "root_index": 0, + "row_selector": "[id^=\"uolplay_\"]:has(> div > p)", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?:#{1,6}\\s+)?Planos UOL Play disponíveis\\s*$", + "end_pattern": "(?m)^(?:#{1,6}\\s+)?Assinaturas individuais\\s*$", + "plain_row_pattern": "(?P<name>UOL Play [^\\n]+?) A partir de R\\$ (?P<price>\\d+,\\d{2}) /mês", + "plain_multiline_row_pattern": "(?m)^(?P<name>UOL Play [^\\n]+)\\n+(?:[ \\t]*\\n)*A partir de R\\$[ \\t]*\\n+(?:[ \\t]*\\n)*(?P<price>\\d+,\\d{2})[ \\t]*\\n+(?:[ \\t]*\\n)*[ \\t]*/mês[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "Planos UOL Play disponíveis\n\nUOL Play Fresh\n\nUOL Play Empty\n\nA partir de R$\n\n18,50\n\n/mês\n\nUOL Play Other\n\nA partir de R$\n\n\n\n/mês\nAssinaturas individuais", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-plan-html-missing", + "category": "structured-extraction", + "root_cause": "same_card_name_price_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "name", + "label": "Plan", + "type": "text", + "selector": ":scope", + "own_text": true + }, + { + "name": "price", + "label": "Monthly starting price", + "type": "token", + "pattern": "\\d+,\\d{2}", + "selector": ":scope > div > p" + } + ], + "record_identity_fields": [ + "name" + ], + "duplicate_policy": "deduplicate_identical", + "table_scope": { + "root_selector": "#plans-tabs", + "root_index": 0, + "row_selector": "[id^=\"uolplay_\"]:has(> div > p)", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?:#{1,6}\\s+)?Planos UOL Play disponíveis\\s*$", + "end_pattern": "(?m)^(?:#{1,6}\\s+)?Assinaturas individuais\\s*$", + "plain_row_pattern": "(?P<name>UOL Play [^\\n]+?) A partir de R\\$ (?P<price>\\d+,\\d{2}) /mês", + "plain_multiline_row_pattern": "(?m)^(?P<name>UOL Play [^\\n]+)\\n+(?:[ \\t]*\\n)*A partir de R\\$[ \\t]*\\n+(?:[ \\t]*\\n)*(?P<price>\\d+,\\d{2})[ \\t]*\\n+(?:[ \\t]*\\n)*[ \\t]*/mês[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "<div id=\"plans-tabs\"><div id=\"uolplay_1\">UOL Play 1<div>A partir de R$<p>10,99</p>/mês</div></div><div id=\"uolplay_2\">UOL Play 2<div>A partir de R$/mês</div></div></div>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-plan-no-names", + "category": "structured-extraction", + "root_cause": "same_card_name_price_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "name", + "label": "Plan", + "type": "text", + "selector": ":scope", + "own_text": true + }, + { + "name": "price", + "label": "Monthly starting price", + "type": "token", + "pattern": "\\d+,\\d{2}", + "selector": ":scope > div > p" + } + ], + "record_identity_fields": [ + "name" + ], + "duplicate_policy": "deduplicate_identical", + "table_scope": { + "root_selector": "#plans-tabs", + "root_index": 0, + "row_selector": "[id^=\"uolplay_\"]:has(> div > p)", + "cell_selector": ":scope", + "start_pattern": "(?m)^(?:#{1,6}\\s+)?Planos UOL Play disponíveis\\s*$", + "end_pattern": "(?m)^(?:#{1,6}\\s+)?Assinaturas individuais\\s*$", + "plain_row_pattern": "(?P<name>UOL Play [^\\n]+?) A partir de R\\$ (?P<price>\\d+,\\d{2}) /mês", + "plain_multiline_row_pattern": "(?m)^(?P<name>UOL Play [^\\n]+)\\n+(?:[ \\t]*\\n)*A partir de R\\$[ \\t]*\\n+(?:[ \\t]*\\n)*(?P<price>\\d+,\\d{2})[ \\t]*\\n+(?:[ \\t]*\\n)*[ \\t]*/mês[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "Planos UOL Play disponíveis\n\n\n\nA partir de R$\n\n18,50\n\n/mês\n\n\n\nA partir de R$\n\n78,20\n\n/mês\nAssinaturas individuais", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-bound-body-complete", + "category": "structured-extraction", + "root_cause": "volatile_body_requires_bound_source", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "intro", + "alternatives": [ + "Assinaturas individuais" + ] + } + ] + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 4, + "segmentation_method": "independent_source_sections", + "regions": [ + { + "id": "source-section-1", + "text": "Assine qualquer COMBO ANUAL do UOL Play e receba o valor de volta para usar na ingresso.com. Todo mês você recebe seu voucher por e-mail.", + "source_text": "Assine qualquer COMBO ANUAL do UOL Play e receba o valor de volta para usar na ingresso.com. Todo mês você recebe seu voucher por e-mail.", + "segmentation_method": "independent_source_sections" + }, + { + "id": "source-section-2", + "text": "Acompanhe todas as emoções do Brasileirão Sub-20, Brasileirão Feminino, WGP e K1 Kickboxing, além de acessar todos os conteúdos do Canal UOL e os originais do UOL, que você só encontra aqui.", + "source_text": "Acompanhe todas as emoções do Brasileirão Sub-20, Brasileirão Feminino, WGP e K1 Kickboxing, além de acessar todos os conteúdos do Canal UOL e os originais do UOL, que você só encontra aqui.", + "segmentation_method": "independent_source_sections" + }, + { + "id": "source-section-3", + "text": "Você também pode contratar seus streamings favoritos pelo UOL Play, aproveitar vários conteúdos originais e ainda ter acesso a todos os benefícios do Clube UOL.", + "source_text": "Você também pode contratar seus streamings favoritos pelo UOL Play, aproveitar vários conteúdos originais e ainda ter acesso a todos os benefícios do Clube UOL.", + "segmentation_method": "independent_source_sections" + }, + { + "id": "source-section-4", + "text": "Os conteúdos mais incríveis do Universal+, com filmes como Top Gun, Kill Bill e Sonic. Séries de tirar o fôlego, como Chicago P.D., Chicago Med e Chicago Fire, além de realities que você só assiste por aqui, como The Traitors e Barras Invisíveis, com a história de Adriane Galisteu.", + "source_text": "Os conteúdos mais incríveis do Universal+, com filmes como Top Gun, Kill Bill e Sonic. Séries de tirar o fôlego, como Chicago P.D., Chicago Med e Chicago Fire, além de realities que você só assiste por aqui, como The Traitors e Barras Invisíveis, com a história de Adriane Galisteu.", + "segmentation_method": "independent_source_sections" + } + ] + }, + "ground_truth_main_content": "Assine qualquer COMBO ANUAL do UOL Play e receba o valor de volta para usar na ingresso.com. Todo mês você recebe seu voucher por e-mail.\n\nAcompanhe todas as emoções do Brasileirão Sub-20, Brasileirão Feminino, WGP e K1 Kickboxing, além de acessar todos os conteúdos do Canal UOL e os originais do UOL, que você só encontra aqui.\n\nVocê também pode contratar seus streamings favoritos pelo UOL Play, aproveitar vários conteúdos originais e ainda ter acesso a todos os benefícios do Clube UOL.\n\nOs conteúdos mais incríveis do Universal+, com filmes como Top Gun, Kill Bill e Sonic. Séries de tirar o fôlego, como Chicago P.D., Chicago Med e Chicago Fire, além de realities que você só assiste por aqui, como The Traitors e Barras Invisíveis, com a história de Adriane Galisteu." + }, + "artifact": "Assinaturas individuais\nAssine qualquer COMBO ANUAL do UOL Play e receba o valor de volta para usar na ingresso.com. Todo mês você recebe seu voucher por e-mail.\n\nAcompanhe todas as emoções do Brasileirão Sub-20, Brasileirão Feminino, WGP e K1 Kickboxing, além de acessar todos os conteúdos do Canal UOL e os originais do UOL, que você só encontra aqui.\n\nVocê também pode contratar seus streamings favoritos pelo UOL Play, aproveitar vários conteúdos originais e ainda ter acesso a todos os benefícios do Clube UOL.\n\nOs conteúdos mais incríveis do Universal+, com filmes como Top Gun, Kill Bill e Sonic. Séries de tirar o fôlego, como Chicago P.D., Chicago Med e Chicago Fire, além de realities que você só assiste por aqui, como The Traitors e Barras Invisíveis, com a história de Adriane Galisteu.", + "ok": true, + "expected_task_pass": true + }, + { + "id": "remaining7-bound-body-missing-programme", + "category": "structured-extraction", + "root_cause": "volatile_body_requires_bound_source", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "intro", + "alternatives": [ + "Assinaturas individuais" + ] + } + ] + }, + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 4, + "segmentation_method": "independent_source_sections", + "regions": [ + { + "id": "source-section-1", + "text": "Assine qualquer COMBO ANUAL do UOL Play e receba o valor de volta para usar na ingresso.com. Todo mês você recebe seu voucher por e-mail.", + "source_text": "Assine qualquer COMBO ANUAL do UOL Play e receba o valor de volta para usar na ingresso.com. Todo mês você recebe seu voucher por e-mail.", + "segmentation_method": "independent_source_sections" + }, + { + "id": "source-section-2", + "text": "Acompanhe todas as emoções do Brasileirão Sub-20, Brasileirão Feminino, WGP e K1 Kickboxing, além de acessar todos os conteúdos do Canal UOL e os originais do UOL, que você só encontra aqui.", + "source_text": "Acompanhe todas as emoções do Brasileirão Sub-20, Brasileirão Feminino, WGP e K1 Kickboxing, além de acessar todos os conteúdos do Canal UOL e os originais do UOL, que você só encontra aqui.", + "segmentation_method": "independent_source_sections" + }, + { + "id": "source-section-3", + "text": "Você também pode contratar seus streamings favoritos pelo UOL Play, aproveitar vários conteúdos originais e ainda ter acesso a todos os benefícios do Clube UOL.", + "source_text": "Você também pode contratar seus streamings favoritos pelo UOL Play, aproveitar vários conteúdos originais e ainda ter acesso a todos os benefícios do Clube UOL.", + "segmentation_method": "independent_source_sections" + }, + { + "id": "source-section-4", + "text": "Os conteúdos mais incríveis do Universal+, com filmes como Top Gun, Kill Bill e Sonic. Séries de tirar o fôlego, como Chicago P.D., Chicago Med e Chicago Fire, além de realities que você só assiste por aqui, como The Traitors e Barras Invisíveis, com a história de Adriane Galisteu.", + "source_text": "Os conteúdos mais incríveis do Universal+, com filmes como Top Gun, Kill Bill e Sonic. Séries de tirar o fôlego, como Chicago P.D., Chicago Med e Chicago Fire, além de realities que você só assiste por aqui, como The Traitors e Barras Invisíveis, com a história de Adriane Galisteu.", + "segmentation_method": "independent_source_sections" + } + ] + }, + "ground_truth_main_content": "Assine qualquer COMBO ANUAL do UOL Play e receba o valor de volta para usar na ingresso.com. Todo mês você recebe seu voucher por e-mail.\n\nAcompanhe todas as emoções do Brasileirão Sub-20, Brasileirão Feminino, WGP e K1 Kickboxing, além de acessar todos os conteúdos do Canal UOL e os originais do UOL, que você só encontra aqui.\n\nVocê também pode contratar seus streamings favoritos pelo UOL Play, aproveitar vários conteúdos originais e ainda ter acesso a todos os benefícios do Clube UOL.\n\nOs conteúdos mais incríveis do Universal+, com filmes como Top Gun, Kill Bill e Sonic. Séries de tirar o fôlego, como Chicago P.D., Chicago Med e Chicago Fire, além de realities que você só assiste por aqui, como The Traitors e Barras Invisíveis, com a história de Adriane Galisteu." + }, + "artifact": "Assinaturas individuais\nAssine qualquer COMBO ANUAL do UOL Play e receba o valor de volta para usar na ingresso.com. Todo mês você recebe seu voucher por e-mail.\n\nAcompanhe todas as emoções do Brasileirão Sub-20, Brasileirão Feminino, WGP e K1 Kickboxing, além de acessar todos os conteúdos do Canal UOL e os originais do UOL, que você só encontra aqui.\n\nVocê também pode contratar seus streamings favoritos pelo UOL Play, aproveitar vários conteúdos originais e ainda ter acesso a todos os benefícios do Clube UOL.\n\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-featured-card", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$" + } + } + }, + "artifact": "<main><a href=\"/serien/a\"><h3>A</h3><p>A complete visible description.</p></a><a href=\"/filme/b\"><h3>B</h3><p>Another description.</p></a></main>", + "ok": true, + "expected_task_pass": true + }, + { + "id": "remaining7-plain-navigation", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)<br>\\s*(?P<label>[^<>\\n]+?)\\s*<br>\\s*$" + } + } + }, + "artifact": "<main><a href=\"/serien/a\">A</a><a href=\"/serien/b\">B</a><p>Some unrelated description</p></main>", + "ok": true, + "expected_task_pass": false + }, + { + "id": "record-owned-stable-author-text-visible", + "category": "acceptance-contract", + "root_cause": "source_owned_stable_text_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/post/\\d+", + "stable_text_anchors": [ + { + "id": "source-visible-author", + "alternatives": [ + "编辑部" + ] + } + ] + } + }, + "artifact": "[文章标题 编辑部](https://example.org/post/1)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_anchor_hits": 1, + "stable_complete_record_count": 1 + } + }, + { + "id": "record-owned-stable-author-text-absent", + "category": "acceptance-contract", + "root_cause": "source_owned_stable_text_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/post/\\d+", + "stable_text_anchors": [ + { + "id": "source-visible-author", + "alternatives": [ + "编辑部" + ] + } + ] + } + }, + "artifact": "[文章标题](https://example.org/post/1)", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_anchor_hits": 0, + "stable_complete_record_count": 1 + } + }, + { + "id": "inline-emphasis-preserves-visible-product-fields", + "category": "format-normalization", + "root_cause": "inline_emphasis_in_structured_fields", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "product", + "minimum_records": 1, + "field_minimum_records": { + "financing": 1, + "shipping": 1 + } + } + }, + "artifact": "Alpha shirt\n$10.90\nFree shipping\nor 9 x<strong>$1.21</strong>", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_complete_record_count": 1 + } + }, + { + "id": "inline-emphasis-cannot-supply-missing-shipping", + "category": "format-normalization", + "root_cause": "inline_emphasis_in_structured_fields", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "product", + "minimum_records": 1, + "field_minimum_records": { + "financing": 1, + "shipping": 1 + } + } + }, + "artifact": "Alpha shirt\n$10.90\nor 9 x<strong>$1.21</strong>", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_complete_record_count": 1 + } + }, + { + "id": "ambiguous-query-candidate-does-not-discard-valid-sibling", + "category": "structured-extraction", + "root_cause": "ambiguous_query_candidate_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item\\.htm", + "identity_query_keys": [ + "id", + "v" + ] + } + }, + "artifact": "[Valid](https://example.test/item.htm?id=1)\n[Ambiguous](https://example.test/item.htm?id=2&id=3)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "ambiguous-query-candidate-alone-cannot-form-record", + "category": "structured-extraction", + "root_cause": "ambiguous_query_candidate_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item\\.htm", + "identity_query_keys": [ + "id", + "v" + ] + } + }, + "artifact": "[Ambiguous](https://example.test/item.htm?id=2&id=3)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dense-flattened-link-collection-remains-a-list", + "category": "structured-extraction", + "root_cause": "flattened_markdown_collection_classification", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 3, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+" + } + }, + "artifact": "- Group - [First](https://example.test/item/1) - [Second](https://example.test/item/2) - [Third](https://example.test/item/3)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "prose-with-three-links-remains-prose", + "category": "structured-extraction", + "root_cause": "flattened_markdown_collection_classification", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 3, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+" + } + }, + "artifact": "Read [First](https://example.test/item/1), compare it carefully with [Second](https://example.test/item/2), then see [Third](https://example.test/item/3) for context.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "redirected-relative-markdown-uses-response-origin", + "category": "structured-extraction", + "root_cause": "redirected_markdown_link_base", + "failure_mode": "false-negative", + "case": { + "url": "https://old.example.test/start", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "new.example.test" + ], + "entry_path": "/news/item/\\d+" + } + }, + "artifact": "[Story](item/1)", + "final_url": "https://new.example.test/news/", + "ok": true, + "expected_task_pass": true + }, + { + "id": "wrong-redirect-cannot-authorize-relative-markdown", + "category": "structured-extraction", + "root_cause": "redirected_markdown_link_base", + "failure_mode": "false-positive", + "case": { + "url": "https://old.example.test/start", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "old.example.test" + ], + "entry_path": "/news/item/\\d+" + } + }, + "artifact": "[Story](item/1)", + "final_url": "https://other.example.test/news/", + "ok": true, + "expected_task_pass": false + }, + { + "id": "malformed-link-is-reported-without-erasing-valid-sibling", + "category": "structured-extraction", + "root_cause": "candidate_local_malformed_link_validation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+" + } + }, + "artifact": "[Broken](https://[invalid/item/1)\n[Valid](https://example.test/item/2)", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_complete_record_count": 1, + "stable_task_errors": [ + "invalid_link_output" + ] + } + }, + { + "id": "well-formed-link-is-not-marked-as-malformed", + "category": "structured-extraction", + "root_cause": "candidate_local_malformed_link_validation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+" + } + }, + "artifact": "[Valid](https://example.test/item/2)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_complete_record_count": 1, + "stable_task_errors": [] + } + }, + { + "id": "linked-product-invalid-identity-does-not-discard-valid-sibling", + "category": "structured-extraction", + "root_cause": "linked_product_candidate_identity_isolation", + "failure_mode": "false-negative", + "case": { + "url": "https://shop.example/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "linked_product", + "minimum_records": 1, + "hosts": [ + "shop.example" + ], + "entry_path": "/product", + "identity_query_keys": [ + "id" + ], + "price_patterns": [ + "\\$\\s*\\d+(?:\\.\\d{2})?" + ], + "price_exclusion_patterns": [] + } + }, + "artifact": "[Missing identity](https://shop.example/product)\n$10.00\n\n[Valid product](https://shop.example/product?id=2)\n$20.00", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_complete_record_count": 1 + } + }, + { + "id": "linked-product-invalid-identity-alone-cannot-form-record", + "category": "structured-extraction", + "root_cause": "linked_product_candidate_identity_isolation", + "failure_mode": "false-positive", + "case": { + "url": "https://shop.example/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "linked_product", + "minimum_records": 1, + "hosts": [ + "shop.example" + ], + "entry_path": "/product", + "identity_query_keys": [ + "id" + ], + "price_patterns": [ + "\\$\\s*\\d+(?:\\.\\d{2})?" + ], + "price_exclusion_patterns": [] + } + }, + "artifact": "[Missing identity](https://shop.example/product)\n$10.00", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_complete_record_count": 0 + } + }, + { + "id": "html-doctype-hidden-challenge-is-not-visible", + "category": "format-normalization", + "root_cause": "html_visible_forbidden_surface", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/products", + "task_kind": "body", + "required_facts": [ + { + "id": "products", + "alternatives": [ + "Available products" + ] + } + ], + "forbidden_any": [ + "CAPTCHA" + ] + }, + "artifact": "<!DOCTYPE html><!--[if IE 7]><html lang=\"en\"><![endif]--><html><head><script>const gate = 'CAPTCHA';</script></head><body><main><h1>Available products</h1></main></body></html>", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "content_pass": true, + "forbidden_requirement_hits": [] + } + }, + { + "id": "html-doctype-visible-challenge-still-blocks", + "category": "format-normalization", + "root_cause": "html_visible_forbidden_surface", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/products", + "task_kind": "body", + "required_facts": [ + { + "id": "products", + "alternatives": [ + "Available products" + ] + } + ], + "forbidden_any": [ + "CAPTCHA" + ] + }, + "artifact": "<!DOCTYPE html><html><body><main><h1>Available products</h1><p>CAPTCHA</p></main></body></html>", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "content_pass": false, + "forbidden_requirement_hits": [ + "CAPTCHA" + ] + } + }, + { + "id": "html-inline-code-preserves-visible-anchor", + "category": "format-normalization", + "root_cause": "stable_anchor_html_inline_markup", + "failure_mode": "false-negative", + "case": { + "url": "https://docs.example.test/search", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "search-instruction", + "alternatives": [ + "Set search: true to enable the search plugin:" + ] + } + ] + } + }, + "artifact": "<!doctype html><html><body><main><p>Set <code>search: true</code> to enable the search plugin:</p></main></body></html>", + "title": "Search | Documentation", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_anchor_hits": 1 + } + }, + { + "id": "html-hidden-script-cannot-satisfy-anchor", + "category": "format-normalization", + "root_cause": "stable_anchor_html_inline_markup", + "failure_mode": "false-positive", + "case": { + "url": "https://docs.example.test/search", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "search-instruction", + "alternatives": [ + "Set search: true to enable the search plugin:" + ] + } + ] + } + }, + "artifact": "<!doctype html><html><head><script>const help = 'Set search: true to enable the search plugin:';</script></head><body><main><p>Search examples</p></main></body></html>", + "title": "Search | Documentation", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_anchor_hits": 0 + } + }, + { + "id": "html-inline-code-counts-for-body-coverage", + "category": "format-normalization", + "root_cause": "body_coverage_html_visible_text", + "failure_mode": "false-negative", + "case": { + "url": "https://docs.example.test/sorting", + "task_kind": "body", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 1.0, + "minimum_floor": 1.0, + "minimum_region_passes": 1, + "regions": [ + { + "id": "instruction", + "text": "To enable sorting simply add sort true to your config" + } + ] + } + }, + "artifact": "<!doctype html><html><body><main><p>To enable sorting, simply add <code>sort: true</code> to your config:</p></main></body></html>", + "title": "Sorting | Documentation", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "body_coverage_pass": true + } + }, + { + "id": "html-hidden-script-cannot-satisfy-body-coverage", + "category": "format-normalization", + "root_cause": "body_coverage_html_visible_text", + "failure_mode": "false-positive", + "case": { + "url": "https://docs.example.test/sorting", + "task_kind": "body", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 1.0, + "minimum_floor": 1.0, + "minimum_region_passes": 1, + "regions": [ + { + "id": "instruction", + "text": "To enable sorting simply add sort true to your config" + } + ] + } + }, + "artifact": "<!doctype html><html><head><script>const help = 'To enable sorting simply add sort true to your config';</script></head><body><main><p>Sorting examples</p></main></body></html>", + "title": "Sorting | Documentation", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "body_coverage_pass": false + } + }, + { + "id": "html-visible-body-ignores-hidden-challenge-heading", + "category": "format-normalization", + "root_cause": "html_visible_answer_surface", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "content", + "alternatives": [ + "Visible product catalog" + ] + } + ] + }, + "min_content_chars": 20 + }, + "artifact": "<!doctype html><html><head><script>\nCAPTCHA\n</script></head><body><main><p>Visible product catalog is ready.</p></main></body></html>", + "title": "Products", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "blocked_page": false, + "length_pass": true + } + }, + { + "id": "html-hidden-script-cannot-inflate-visible-length", + "category": "format-normalization", + "root_cause": "html_visible_answer_surface", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "content", + "alternatives": [ + "Hello" + ] + } + ] + }, + "min_content_chars": 20 + }, + "artifact": "<!doctype html><html><head><script>const hidden = 'This lengthy source code is not the page body';</script></head><body><main>Hello</main></body></html>", + "title": "Home", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "length_pass": false + } + }, + { + "id": "dynamic-undeclared-source-language-can-verify-identity", + "category": "acceptance-contract", + "root_cause": "source_document_language_absent", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/untagged", + "evaluation_url": "https://example.test/untagged", + "expected_final_url": "https://example.test/untagged", + "target_language": "en", + "target_region": "US", + "expected_document_language_present": false, + "track": "dynamic", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "content", + "alternatives": [ + "Visible content" + ] + } + ] + } + }, + "artifact": "Visible content", + "ok": true, + "final_url": "https://example.test/untagged", + "metadata": { + "route": "dynamic_selected_text", + "document_language": "" + }, + "expected_task_pass": true, + "expected_metrics": { + "page_identity_status": "verified", + "language_consistent": true + } + }, + { + "id": "dynamic-undeclared-source-language-rejects-declared-language", + "category": "acceptance-contract", + "root_cause": "source_document_language_absent", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/untagged", + "evaluation_url": "https://example.test/untagged", + "expected_final_url": "https://example.test/untagged", + "target_language": "en", + "target_region": "US", + "expected_document_language_present": false, + "track": "dynamic", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "content", + "alternatives": [ + "Visible content" + ] + } + ] + } + }, + "artifact": "Visible content", + "ok": true, + "final_url": "https://example.test/untagged", + "metadata": { + "route": "dynamic_selected_text", + "document_language": "en" + }, + "expected_task_pass": false, + "expected_metrics": { + "page_identity_status": "mismatch", + "language_consistent": false + } + }, + { + "id": "long-metadata-title-keeps-visible-html-anchor", + "category": "format-normalization", + "root_cause": "html_title_prefix_detection", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Required body" + ] + } + ] + } + }, + "artifact": "<html><body><p>Required <b>body</b></p></body></html>", + "title": "Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title ", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_anchor_hits": 1 + } + }, + { + "id": "long-metadata-title-cannot-expose-script-anchor", + "category": "format-normalization", + "root_cause": "html_title_prefix_detection", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Required body" + ] + } + ] + } + }, + "artifact": "<html><body><script>Required body</script><p>Other text</p></body></html>", + "title": "Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title Metadata title ", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_anchor_hits": 0 + } + }, + { + "id": "html-inline-challenge-explanation-is-prose", + "category": "format-normalization", + "root_cause": "html_block_line_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Required body" + ] + } + ] + } + }, + "artifact": "<html><body><p>Required body. If you see <strong>Verify you are human</strong>, complete the verification before continuing.</p></body></html>", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "blocked_page": false, + "stable_blocked_page": false + } + }, + { + "id": "combined-problem-cells-across-sections-pass", + "category": "structured-extraction", + "root_cause": "composite_table_equivalent_fields", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/training", + "task_kind": "body", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "problems", + "kind": "table", + "minimum_records": 3, + "start_pattern": "(?m)^Section\\s+1\\.", + "end_pattern": "(?m)^Section\\s+6\\.", + "required_columns": [ + { + "id": "number", + "kind": "text", + "pattern": "^[A-Z]?[0-9]+$", + "aliases": [ + "Number" + ] + }, + { + "id": "name", + "kind": "text", + "aliases": [ + "Name" + ] + }, + { + "id": "attempts", + "kind": "integer", + "aliases": [ + "Tried" + ] + }, + { + "id": "accepted", + "kind": "integer", + "aliases": [ + "AC" + ] + }, + { + "id": "difficulty", + "kind": "integer", + "aliases": [ + "Difficulty" + ] + } + ], + "identity_column": "number", + "combined_columns": [ + { + "aliases": [ + "Problem" + ], + "pattern": "^(?P<number>[A-Z]?[0-9]+)\\s+(?P<name>.+)$" + } + ] + } + ] + } + }, + "artifact": "Section 1. First\nProblem\tTried\tAC\tDifficulty\nP1 Hello World\t1\t1\t7\nP2 Output Practice\t2\t1\t5\nSection 2. Next\nProblem\tTried\tAC\tDifficulty\nB003 Addition Exercise\t3\t2\t4\nSection 6. End", + "ok": true, + "expected_task_pass": true + }, + { + "id": "combined-problem-cells-cannot-omit-name", + "category": "structured-extraction", + "root_cause": "composite_table_equivalent_fields", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/training", + "task_kind": "body", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "problems", + "kind": "table", + "minimum_records": 3, + "start_pattern": "(?m)^Section\\s+1\\.", + "end_pattern": "(?m)^Section\\s+6\\.", + "required_columns": [ + { + "id": "number", + "kind": "text", + "pattern": "^[A-Z]?[0-9]+$", + "aliases": [ + "Number" + ] + }, + { + "id": "name", + "kind": "text", + "aliases": [ + "Name" + ] + }, + { + "id": "attempts", + "kind": "integer", + "aliases": [ + "Tried" + ] + }, + { + "id": "accepted", + "kind": "integer", + "aliases": [ + "AC" + ] + }, + { + "id": "difficulty", + "kind": "integer", + "aliases": [ + "Difficulty" + ] + } + ], + "identity_column": "number", + "combined_columns": [ + { + "aliases": [ + "Problem" + ], + "pattern": "^(?P<number>[A-Z]?[0-9]+)\\s+(?P<name>.+)$" + } + ] + } + ] + } + }, + "artifact": "Section 1. First\nProblem\tTried\tAC\tDifficulty\nP1 Hello World\t1\t1\t7\nP2 Output Practice\t2\t1\t5\nSection 2. Next\nProblem\tTried\tAC\tDifficulty\nB003\t3\t2\t4\nSection 6. End", + "ok": true, + "expected_task_pass": false + }, + { + "id": "html-head-title-cannot-supply-body", + "category": "format-normalization", + "root_cause": "html_head_is_not_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Required body" + ] + } + ] + } + }, + "artifact": "<html><head><title>Required body

Other text

", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_anchor_hits": 0 + } + }, + { + "id": "html-body-anchor-with-head-title-passes", + "category": "format-normalization", + "root_cause": "html_head_is_not_body", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Required body" + ] + } + ] + } + }, + "artifact": "Required body

Required body

", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_anchor_hits": 1 + } + }, + { + "id": "card-single-line-distinct-typed-fields-pass", + "category": "structured-extraction", + "root_cause": "card_field_span_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/homes", + "task_kind": "body", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "homes", + "kind": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "required_fields": [ + { + "id": "price", + "kind": "price" + }, + { + "id": "beds", + "kind": "integer", + "pattern": "\\b(?:bds|beds?)\\b" + }, + { + "id": "baths", + "kind": "integer", + "pattern": "\\b(?:ba|baths?)\\b" + }, + { + "id": "area", + "kind": "area" + } + ] + } + ] + } + }, + "artifact": "[One Main Street](https://example.test/item/1)\n$900,000\n3 beds · 2 baths · 1,200 sqft", + "ok": true, + "expected_task_pass": true + }, + { + "id": "card-unrelated-quantity-cannot-supply-typed-fields", + "category": "structured-extraction", + "root_cause": "card_field_span_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/homes", + "task_kind": "body", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "homes", + "kind": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+", + "required_fields": [ + { + "id": "price", + "kind": "price" + }, + { + "id": "beds", + "kind": "integer", + "pattern": "\\b(?:bds|beds?)\\b" + }, + { + "id": "baths", + "kind": "integer", + "pattern": "\\b(?:ba|baths?)\\b" + }, + { + "id": "area", + "kind": "area" + } + ] + } + ] + } + }, + "artifact": "[One Main Street](https://example.test/item/1)\n$900,000\n1200 sqft beds\n1200 sqft baths\n1200 sqft", + "ok": true, + "expected_task_pass": false + }, + { + "id": "metadata-title-body-anchor-missing-fails", + "category": "format-normalization", + "root_cause": "metadata_title_is_not_declared_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Required body" + ] + } + ] + } + }, + "artifact": "Other text", + "title": "Required body", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_anchor_hits": 0 + } + }, + { + "id": "metadata-title-body-anchor-visible-passes", + "category": "format-normalization", + "root_cause": "metadata_title_is_not_declared_body", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Required body" + ] + } + ] + } + }, + "artifact": "Required body", + "title": "Other title", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_anchor_hits": 1 + } + }, + { + "id": "metadata-title-body-coverage-missing-fails", + "category": "body-scope", + "root_cause": "metadata_title_is_not_declared_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "required body paragraph explains independent visible source content and complete record ownership", + "source_text": "required body paragraph explains independent visible source content and complete record ownership", + "segmentation_method": "balanced_tokens_fallback" + } + ] + } + }, + "artifact": "Other text", + "title": "Required body paragraph explains independent visible source content and complete record ownership.", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "body_coverage_pass": false + } + }, + { + "id": "metadata-title-body-coverage-visible-passes", + "category": "body-scope", + "root_cause": "metadata_title_is_not_declared_body", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "required body paragraph explains independent visible source content and complete record ownership", + "source_text": "required body paragraph explains independent visible source content and complete record ownership", + "segmentation_method": "balanced_tokens_fallback" + } + ] + } + }, + "artifact": "Required body paragraph explains independent visible source content and complete record ownership.", + "title": "Other title", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "body_coverage_pass": true + } + }, + { + "id": "metadata-title-composite-missing-fails", + "category": "structured-extraction", + "root_cause": "metadata_title_is_not_declared_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/homes", + "task_kind": "body", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "homes", + "kind": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+" + } + ] + } + }, + "artifact": "Other text", + "title": "[One Main Street](https://example.test/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "metadata-title-composite-visible-passes", + "category": "structured-extraction", + "root_cause": "metadata_title_is_not_declared_body", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/homes", + "task_kind": "body", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "homes", + "kind": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+" + } + ] + } + }, + "artifact": "[One Main Street](https://example.test/item/1)", + "title": "Other title", + "ok": true, + "expected_task_pass": true + }, + { + "id": "legacy-title-marker-missing-fails", + "category": "format-normalization", + "root_cause": "legacy_title_is_not_page_text", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/order", + "task_kind": "body", + "expected_any": [ + "Order ID 123" + ] + }, + "artifact": "Other text", + "title": "Order ID 123", + "ok": true, + "expected_task_pass": false + }, + { + "id": "legacy-title-marker-visible-passes", + "category": "format-normalization", + "root_cause": "legacy_title_is_not_page_text", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/order", + "task_kind": "body", + "expected_any": [ + "Order ID 123" + ] + }, + "artifact": "Order ID 123", + "title": "Other title", + "ok": true, + "expected_task_pass": true + }, + { + "id": "legacy-title-pattern-missing-fails", + "category": "format-normalization", + "root_cause": "legacy_title_is_not_page_text", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/order", + "task_kind": "body", + "required_patterns": [ + { + "id": "order", + "pattern": "Order ID [0-9]+", + "minimum": 1 + } + ] + }, + "artifact": "Other text", + "title": "Order ID 123", + "ok": true, + "expected_task_pass": false + }, + { + "id": "legacy-title-pattern-visible-passes", + "category": "format-normalization", + "root_cause": "legacy_title_is_not_page_text", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/order", + "task_kind": "body", + "required_patterns": [ + { + "id": "order", + "pattern": "Order ID [0-9]+", + "minimum": 1 + } + ] + }, + "artifact": "Order ID 123", + "title": "Other title", + "ok": true, + "expected_task_pass": true + }, + { + "id": "html-misplaced-title-missing-body-fails", + "category": "format-normalization", + "root_cause": "html_head_is_not_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Required body" + ] + } + ] + } + }, + "artifact": "Required body

Other text

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "html-misplaced-title-visible-body-passes", + "category": "format-normalization", + "root_cause": "html_head_is_not_body", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/article", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Required body" + ] + } + ] + } + }, + "artifact": "Required body

Required body

", + "ok": true, + "expected_task_pass": true + }, + { + "id": "declared-page-chrome-is-required-stable-text", + "category": "acceptance-contract", + "root_cause": "declared_full_page_stable_text_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/post/\\d+", + "stable_text_anchors": [ + { + "id": "navigation", + "alternatives": [ + "翻译" + ] + } + ] + } + }, + "artifact": "翻译\n[文章标题](https://example.org/post/1)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_anchor_hits": 1, + "stable_complete_record_count": 1 + } + }, + { + "id": "missing-declared-page-chrome-fails-full-page-contract", + "category": "acceptance-contract", + "root_cause": "declared_full_page_stable_text_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/post/\\d+", + "stable_text_anchors": [ + { + "id": "navigation", + "alternatives": [ + "翻译" + ] + } + ] + } + }, + "artifact": "[文章标题](https://example.org/post/1)", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_anchor_hits": 0, + "stable_complete_record_count": 1 + } + }, + { + "id": "markdown-plus-list-keeps-named-records", + "category": "structured-extraction", + "root_cause": "markdown_list_prefix_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "structure", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_context": "list_or_card", + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Catalog" + ] + } + ] + } + }, + "artifact": "Catalog\n + [Alpha](/item/1)\n + [Beta](/item/2)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_structure_pass": true, + "stable_unique_record_count": 2 + } + }, + { + "id": "markdown-plus-prose-is-not-a-record-list", + "category": "structured-extraction", + "root_cause": "markdown_list_prefix_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "structure", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_context": "list_or_card", + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Catalog" + ] + } + ] + } + }, + "artifact": "Catalog\nRead [Alpha](/item/1) + consider [Beta](/item/2).", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_structure_pass": false, + "stable_unique_record_count": 0 + } + }, + { + "id": "adjacent-multiline-markdown-cards-keep-records", + "category": "structured-extraction", + "root_cause": "markdown_list_prefix_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "structure", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_context": "list_or_card", + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Catalog" + ] + } + ] + } + }, + "artifact": "Catalog\n[Alpha\nDetails](/item/1) [Beta\nDetails](/item/2)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_structure_pass": true, + "stable_unique_record_count": 2 + } + }, + { + "id": "multiline-card-followed-by-prose-does-not-borrow-record", + "category": "structured-extraction", + "root_cause": "markdown_list_prefix_ownership", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "structure", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_context": "list_or_card", + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Catalog" + ] + } + ] + } + }, + "artifact": "Catalog\n[Alpha\nDetails](/item/1) consider [Beta\nDetails](/item/2)", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_structure_pass": false, + "stable_unique_record_count": 1 + } + }, + { + "id": "emphasized-numeric-rank-is-valid-entry", + "category": "format-normalization", + "root_cause": "balanced_numeric_rank_emphasis", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+" + } + }, + "artifact": "* _10_ [A new paper](/item/10) [Journal](/journal/1)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_unique_record_count": 1 + } + }, + { + "id": "emphasized-recommendation-is-not-list-entry", + "category": "format-normalization", + "root_cause": "balanced_numeric_rank_emphasis", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+" + } + }, + "artifact": "Read _10_ [A new paper](/item/10)", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_unique_record_count": 0 + } + }, + { + "id": "markdown-heading-opens-declared-list-scope", + "category": "format-normalization", + "root_cause": "markdown_heading_scope_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": "main", + "start_pattern": "^News\\s*$", + "end_pattern": "^Related items\\s*$" + } + } + }, + "artifact": "## News\n[A new article](/item/1)\n## [Related **items**](/archive)\n[Outside article](/item/2)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_unique_record_count": 1 + } + }, + { + "id": "outside-section-links-do-not-fill-declared-scope", + "category": "format-normalization", + "root_cause": "markdown_heading_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": "main", + "start_pattern": "^News\\s*$", + "end_pattern": "^Related items\\s*$" + } + } + }, + "artifact": "[Outside before](/item/1)\n## News\n## [Related **items**](/archive)\n[Outside after](/item/2)", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_unique_record_count": 0 + } + }, + { + "id": "inline-layout-breaks-preserve-table-fields", + "category": "format-normalization", + "root_cause": "inline_break_table_cell_layout", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "(?m)^(?=\\|\\s*Name\\s*\\|)", + "end_pattern": "(?m)^End$", + "plain_row_pattern": "(?P[A-Za-z]+)\\s+(?P\\d+)" + } + } + }, + "artifact": "| Name

| Age
|\n| --- | --- |\n| Alice
| 25
|\nEnd", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_unique_record_count": 1 + } + }, + { + "id": "inline-layout-breaks-do-not-repair-missing-table-fields", + "category": "format-normalization", + "root_cause": "inline_break_table_cell_layout", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "(?m)^(?=\\|\\s*Name\\s*\\|)", + "end_pattern": "(?m)^End$", + "plain_row_pattern": "(?P[A-Za-z]+)\\s+(?P\\d+)" + } + } + }, + "artifact": "| Name
| Age
|\n| --- | --- |\n| Alice
|
|\nEnd", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_unique_record_count": 0 + } + }, + { + "id": "declared-redirect-link-base-resolves-relative-records", + "category": "acceptance-contract", + "root_cause": "independently_declared_document_link_base", + "failure_mode": "false-negative", + "case": { + "url": "https://old.example/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "new.example" + ], + "entry_path": "/article/\\d+", + "link_base_url": "https://new.example/" + } + }, + "artifact": "[A new article](/article/1)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_unique_record_count": 1 + } + }, + { + "id": "declared-redirect-link-base-does-not-allow-other-hosts", + "category": "acceptance-contract", + "root_cause": "independently_declared_document_link_base", + "failure_mode": "false-positive", + "case": { + "url": "https://old.example/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "new.example" + ], + "entry_path": "/article/\\d+", + "link_base_url": "https://new.example/" + } + }, + "artifact": "[A new article](https://outside.example/article/1)", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_unique_record_count": 0 + } + }, + { + "id": "escaped-list-punctuation-retains-entry", + "category": "format-normalization", + "root_cause": "commonmark_escaped_punctuation_in_entry_prefix", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/[0-9]+" + } + }, + "artifact": "2026-10-08 \\- [Actual article](/article/1)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "escaped-recommendation-remains-prose", + "category": "format-normalization", + "root_cause": "commonmark_escaped_punctuation_in_entry_prefix", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/article/[0-9]+" + } + }, + "artifact": "2026-10-08 Read \\- [Actual article](/article/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "unreserved-path-encoding-retains-card-fields", + "category": "format-normalization", + "root_cause": "composite_card_identity_disagrees_on_unreserved_url_encoding", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "offers", + "kind": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+_detail", + "required_fields": [ + { + "id": "price", + "kind": "price" + } + ] + } + ] + } + }, + "artifact": "[Actual offer](/item/12%5Fdetail)\n\n$120", + "ok": true, + "expected_task_pass": true + }, + { + "id": "encoded-card-cannot-borrow-next-price", + "category": "format-normalization", + "root_cause": "composite_card_identity_disagrees_on_unreserved_url_encoding", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "offers", + "kind": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/[0-9]+_detail", + "required_fields": [ + { + "id": "price", + "kind": "price" + } + ] + } + ] + } + }, + "artifact": "[Actual offer](/item/12%5Fdetail)\n\n[Other offer](/item/13_detail)\n\n$120", + "ok": true, + "expected_task_pass": false + }, + { + "id": "markdown-table-header-opens-declared-scope", + "category": "format-normalization", + "root_cause": "markdown_table_header_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "amount", + "label": "Amount", + "type": "integer" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "^Name\\s*$", + "end_pattern": "^Footer$", + "plain_row_pattern": "(?P[^\\t]+)\\t(?P\\d+)" + } + } + }, + "artifact": "| Name | Amount |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_unique_record_count": 2 + } + }, + { + "id": "markdown-table-header-cannot-bypass-declared-scope", + "category": "format-normalization", + "root_cause": "markdown_table_header_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "amount", + "label": "Amount", + "type": "integer" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "^Target section$", + "end_pattern": "^Footer$", + "plain_row_pattern": "(?P[^\\t]+)\\t(?P\\d+)" + } + } + }, + "artifact": "| Name | Amount |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_unique_record_count": 0 + } + }, + { + "id": "missing-page-identity-does-not-hide-empty-body", + "category": "task-predicate", + "root_cause": "unknown_identity_masks_independent_requirement_failure", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/start", + "track": "risk_control", + "task_kind": "body", + "expected_any": [ + "Expected body" + ], + "min_content_chars": 1, + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Expected body" + ] + } + ] + }, + "source_benchmark": { + "risk_control": { + "navigation_contract": { + "requested_url": "https://example.test/start", + "observed_final_url": "https://example.test/final" + } + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": false, + "expected_task_measured": true, + "expected_metrics": { + "page_identity_status": "unavailable", + "length_pass": false + } + }, + { + "id": "missing-page-identity-with-valid-body-stays-unknown", + "category": "task-predicate", + "root_cause": "unknown_identity_masks_independent_requirement_failure", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/start", + "track": "risk_control", + "task_kind": "body", + "expected_any": [ + "Expected body" + ], + "min_content_chars": 1, + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Expected body" + ] + } + ] + }, + "source_benchmark": { + "risk_control": { + "navigation_contract": { + "requested_url": "https://example.test/start", + "observed_final_url": "https://example.test/final" + } + } + } + }, + "artifact": "Expected body", + "ok": true, + "expected_task_pass": false, + "expected_task_measured": false, + "expected_metrics": { + "page_identity_status": "unavailable", + "length_pass": true + } + }, + { + "id": "observed-final-url-accepts-declared-destination", + "category": "answer-surface", + "root_cause": "requested_url_misreported_as_final_url", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/start", + "track": "risk_control", + "task_kind": "body", + "expected_any": [ + "Expected body" + ], + "min_content_chars": 1, + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Expected body" + ] + } + ] + }, + "source_benchmark": { + "risk_control": { + "navigation_contract": { + "requested_url": "https://example.test/start", + "observed_final_url": "https://example.test/final" + } + } + } + }, + "artifact": "Expected body", + "ok": true, + "final_url": "https://example.test/final", + "expected_task_pass": true, + "expected_metrics": { + "page_identity_status": "verified" + } + }, + { + "id": "observed-final-url-rejects-undeclared-destination", + "category": "answer-surface", + "root_cause": "requested_url_misreported_as_final_url", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/start", + "track": "risk_control", + "task_kind": "body", + "expected_any": [ + "Expected body" + ], + "min_content_chars": 1, + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Expected body" + ] + } + ] + }, + "source_benchmark": { + "risk_control": { + "navigation_contract": { + "requested_url": "https://example.test/start", + "observed_final_url": "https://example.test/final" + } + } + } + }, + "artifact": "Expected body", + "ok": true, + "final_url": "https://example.test/other", + "expected_task_pass": false, + "expected_metrics": { + "page_identity_status": "mismatch" + } + }, + { + "id": "sort-icons-preserve-declared-table-columns", + "category": "format-normalization", + "root_cause": "markdown_table_sort_icon_headers", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "amount", + "label": "Amount", + "type": "integer" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "^Name$", + "end_pattern": "^Footer$", + "plain_row_pattern": "(?P[^\\t]+)\\t(?P\\d+)" + } + } + }, + "artifact": "| Name ↕ | Amount ↕ |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "sort-icons-cannot-substitute-a-required-column", + "category": "format-normalization", + "root_cause": "markdown_table_sort_icon_headers", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "amount", + "label": "Amount", + "type": "integer" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "^Name$", + "end_pattern": "^Footer$", + "plain_row_pattern": "(?P[^\\t]+)\\t(?P\\d+)" + } + } + }, + "artifact": "| Name ↕ | Other ↕ |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "split-sticky-header-preserves-explicit-body-columns", + "category": "format-normalization", + "root_cause": "markdown_table_split_sticky_header", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "amount", + "label": "Amount", + "type": "integer" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "^Name$", + "end_pattern": "^Footer$", + "plain_row_pattern": "(?P[^\\t]+)\\t(?P\\d+)" + } + } + }, + "artifact": "| Name | Amount |\n| --- | --- |\n\n| | |\n| --- | --- |\n| Name | Amount |\n| Alpha | 7 |\n| Beta | 8 |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "split-sticky-header-cannot-inherit-unobserved-body-columns", + "category": "format-normalization", + "root_cause": "markdown_table_split_sticky_header", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "amount", + "label": "Amount", + "type": "integer" + } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "^Name$", + "end_pattern": "^Footer$", + "plain_row_pattern": "(?P[^\\t]+)\\t(?P\\d+)" + } + } + }, + "artifact": "| Name | Amount |\n| --- | --- |\n\n| | |\n| --- | --- |\n| Alpha | 7 |\n| Beta | 8 |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "inline-html-labels-preserve-markdown-entry-scope", + "category": "format-normalization", + "root_cause": "inline_html_misclassified_as_document", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": "main a[href]", + "start_pattern": "^Target list$", + "end_pattern": "^Footer$" + } + } + }, + "artifact": "## Target list\n[

First entry

](/item/1)\n[

Second entry

](/item/2)\n## Footer", + "metadata": {}, + "ok": true, + "expected_task_pass": true + }, + { + "id": "document-html-cannot-borrow-footer-links", + "category": "format-normalization", + "root_cause": "inline_html_misclassified_as_document", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": "main a[href]", + "start_pattern": "^Target list$", + "end_pattern": "^Footer$" + } + } + }, + "artifact": "
Target list
", + "metadata": {}, + "ok": true, + "expected_task_pass": false + }, + { + "id": "unique-delivered-section-labels-bind-native-named-links", + "category": "answer-surface", + "root_cause": "native_named_links_lose_proven_text_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": "main a[href]", + "start_pattern": "^Target list$", + "end_pattern": "^Footer$" + } + } + }, + "artifact": "## Target list\nFirst entry\nSecond entry\n## Footer", + "metadata": { + "links": [ + { + "text": "First entry", + "href": "https://example.test/item/1" + }, + { + "text": "Second entry", + "href": "https://example.test/item/2" + } + ] + }, + "ok": true, + "expected_task_pass": true + }, + { + "id": "native-named-links-cannot-supply-missing-section-label", + "category": "answer-surface", + "root_cause": "native_named_links_lose_proven_text_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": "main a[href]", + "start_pattern": "^Target list$", + "end_pattern": "^Footer$" + } + } + }, + "artifact": "## Target list\nFirst entry\n## Footer\nSecond entry", + "metadata": { + "links": [ + { + "text": "First entry", + "href": "https://example.test/item/1" + }, + { + "text": "Second entry", + "href": "https://example.test/item/2" + } + ] + }, + "ok": true, + "expected_task_pass": false + }, + { + "id": "composite-html-retains-visible-link-destinations", + "category": "answer-surface", + "root_cause": "html_link_destinations_lost_before_composite_acceptance", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/base/", + "task_kind": "body", + "expected_any": [ + "Directory" + ], + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "entries", + "kind": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/base/[^/]+" + } + ] + } + }, + "artifact": "

Directory

AlphaBeta", + "ok": true, + "expected_task_pass": true + }, + { + "id": "composite-html-visible-names-without-destinations-fail", + "category": "answer-surface", + "root_cause": "html_link_destinations_lost_before_composite_acceptance", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/base/", + "task_kind": "body", + "expected_any": [ + "Directory" + ], + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "entries", + "kind": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/base/[^/]+" + } + ] + } + }, + "artifact": "

Directory

AlphaBeta", + "ok": true, + "expected_task_pass": false + }, + { + "id": "composite-html-hidden-links-cannot-fill-records", + "category": "answer-surface", + "root_cause": "html_link_destinations_lost_before_composite_acceptance", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/base/", + "task_kind": "body", + "expected_any": [ + "Directory" + ], + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "entries", + "kind": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/base/[^/]+" + } + ] + } + }, + "artifact": "

Directory

Alpha", + "ok": true, + "expected_task_pass": false + }, + { + "id": "visible-character-references-preserve-body-coverage", + "category": "format-normalization", + "root_cause": "visible_character_references_not_decoded", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/body", + "task_kind": "body", + "expected_any": [], + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 1.0, + "minimum_floor": 1.0, + "minimum_region_passes": 1, + "regions": [ + { + "id": "body", + "text": "Alpha Beta Gamma Delta" + } + ] + } + }, + "artifact": "Alpha Beta Gamma Delta", + "ok": true, + "expected_task_pass": true + }, + { + "id": "character-references-cannot-replace-missing-body", + "category": "format-normalization", + "root_cause": "visible_character_references_not_decoded", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/body", + "task_kind": "body", + "expected_any": [], + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 1.0, + "minimum_floor": 1.0, + "minimum_region_passes": 1, + "regions": [ + { + "id": "body", + "text": "Alpha Beta Gamma Delta" + } + ] + } + }, + "artifact": "Alpha Beta Gamma Wrong", + "ok": true, + "expected_task_pass": false + }, + { + "id": "character-references-are-decoded-once", + "category": "format-normalization", + "root_cause": "visible_character_references_not_decoded", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/body", + "task_kind": "body", + "expected_any": [], + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 1.0, + "minimum_floor": 1.0, + "minimum_region_passes": 1, + "regions": [ + { + "id": "body", + "text": "Alpha Beta Gamma Delta" + } + ] + } + }, + "artifact": "Alpha&#32;Beta&#x20;Gamma&nbsp;Delta", + "ok": true, + "expected_task_pass": false + }, + { + "id": "linked-column-labels-preserve-table-headers", + "category": "format-normalization", + "root_cause": "linked_table_headers_not_normalized_as_visible_labels", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "rows", + "kind": "linked_table", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "required_columns": [ + { + "id": "name", + "kind": "text", + "aliases": [ + "Name" + ] + }, + { + "id": "version", + "kind": "version", + "aliases": [ + "Version" + ] + } + ], + "identity_column": "name", + "link_column": "name" + } + ] + } + }, + "artifact": "| [Name](/sort/name) | [Version](/sort/version \"Sort packages\") |\n| --- | --- |\n| [Alpha](/item/1) | 1.2.3 |\n| [Beta](/item/2) | 2.3.4 |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "column-label-cannot-be-inferred-from-header-url", + "category": "format-normalization", + "root_cause": "linked_table_headers_not_normalized_as_visible_labels", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "rows", + "kind": "linked_table", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "required_columns": [ + { + "id": "name", + "kind": "text", + "aliases": [ + "Name" + ] + }, + { + "id": "version", + "kind": "version", + "aliases": [ + "Version" + ] + } + ], + "identity_column": "name", + "link_column": "name" + } + ] + } + }, + "artifact": "| [Other](/sort/Name) | Version |\n| --- | --- |\n| [Alpha](/item/1) | 1.2.3 |\n| [Beta](/item/2) | 2.3.4 |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "linked-headers-cannot-repair-invalid-row-fields", + "category": "format-normalization", + "root_cause": "linked_table_headers_not_normalized_as_visible_labels", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "rows", + "kind": "linked_table", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "required_columns": [ + { + "id": "name", + "kind": "text", + "aliases": [ + "Name" + ] + }, + { + "id": "version", + "kind": "version", + "aliases": [ + "Version" + ] + } + ], + "identity_column": "name", + "link_column": "name" + } + ] + } + }, + "artifact": "| [Name](/sort/name) | Version |\n| --- | --- |\n| [Alpha](/item/1) | 1.2.3 |\n| [Beta](/item/2) | unknown |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "thumbnail-link-binds-immediate-visible-caption", + "category": "format-normalization", + "root_cause": "thumbnail_link_and_visible_caption_split_across_markdown_blocks", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".card", + "title_selector": "p.name", + "link_selector": "a[href]", + "start_pattern": "^Catalog$", + "end_pattern": "^Footer$" + } + } + }, + "artifact": "Catalog\n[![](/cover1.jpg)](/item/1)\n\nFirst delivered title\n\n[![](/cover2.jpg)](/item/2)\n\nSecond delivered title\n\nFooter", + "ok": true, + "expected_task_pass": true + }, + { + "id": "thumbnail-alt-cannot-supply-missing-visible-caption", + "category": "format-normalization", + "root_cause": "thumbnail_link_and_visible_caption_split_across_markdown_blocks", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".card", + "title_selector": "p.name", + "link_selector": "a[href]", + "start_pattern": "^Catalog$", + "end_pattern": "^Footer$" + } + } + }, + "artifact": "Catalog\n[![](/cover1.jpg)](/item/1)\n\nFirst delivered title\n\n[![Second title only in image alt](/cover2.jpg)](/item/2)\n\nFooter", + "ok": true, + "expected_task_pass": false + }, + { + "id": "thumbnail-link-cannot-borrow-next-card-title", + "category": "format-normalization", + "root_cause": "thumbnail_link_and_visible_caption_split_across_markdown_blocks", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".card", + "title_selector": "p.name", + "link_selector": "a[href]", + "start_pattern": "^Catalog$", + "end_pattern": "^Footer$" + } + } + }, + "artifact": "Catalog\n[![](/cover1.jpg)](/item/1)\n\n[![](/cover2.jpg)](/item/2)\n\nSecond delivered title\n\nFooter", + "ok": true, + "expected_task_pass": false + }, + { + "id": "thumbnail-link-cannot-borrow-outside-scope-caption", + "category": "format-normalization", + "root_cause": "thumbnail_link_and_visible_caption_split_across_markdown_blocks", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".card", + "title_selector": "p.name", + "link_selector": "a[href]", + "start_pattern": "^Catalog$", + "end_pattern": "^Footer$" + } + } + }, + "artifact": "Catalog\n[![](/cover1.jpg)](/item/1)\n\nFirst delivered title\n\n[![](/cover2.jpg)](/item/2)\n\n## Footer\nSecond title outside scope", + "ok": true, + "expected_task_pass": false + }, + { + "id": "parallel-table-columns-preserve-independent-named-lists", + "category": "structured-extraction", + "root_cause": "parallel_table_columns_treated_as_linear_sections", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "first", + "kind": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "start_pattern": "First Division", + "end_pattern": "Second Division" + }, + { + "id": "second", + "kind": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "start_pattern": "Second Division", + "end_pattern": "End" + } + ] + } + }, + "artifact": "
First DivisionSecond Division
AlphaGamma
BetaDelta

End

", + "ok": true, + "expected_task_pass": true + }, + { + "id": "parallel-column-cannot-borrow-sibling-records", + "category": "structured-extraction", + "root_cause": "parallel_table_columns_treated_as_linear_sections", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "first", + "kind": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "start_pattern": "First Division", + "end_pattern": "Second Division" + }, + { + "id": "second", + "kind": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "start_pattern": "Second Division", + "end_pattern": "End" + } + ] + } + }, + "artifact": "
First DivisionSecond Division
AlphaGamma
Delta

End

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "parallel-column-requires-declared-terminal-boundary", + "category": "structured-extraction", + "root_cause": "parallel_table_columns_treated_as_linear_sections", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "composite_acceptance": { + "version": 1, + "groups": [ + { + "id": "first", + "kind": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "start_pattern": "First Division", + "end_pattern": "Second Division" + }, + { + "id": "second", + "kind": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "start_pattern": "Second Division", + "end_pattern": "End" + } + ] + } + }, + "artifact": "
First DivisionSecond Division
AlphaGamma
BetaDelta
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "inline-html-break-preserves-body-statement", + "category": "format-normalization", + "root_cause": "inline_html_break_treated_as_visible_body_text", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "statement", + "alternatives": [ + "One platform. Endless ways to work together." + ] + } + ] + } + }, + "artifact": "## One platform.
Endless ways to work together.", + "ok": true, + "expected_task_pass": true + }, + { + "id": "encoded-break-keeps-literal-markup", + "category": "format-normalization", + "root_cause": "inline_html_break_treated_as_visible_body_text", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "statement", + "alternatives": [ + "One platform. Endless ways to work together." + ] + } + ] + } + }, + "artifact": "## One platform. <br>Endless ways to work together.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "code-break-keeps-literal-markup", + "category": "format-normalization", + "root_cause": "inline_html_break_treated_as_visible_body_text", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "statement", + "alternatives": [ + "One platform. Endless ways to work together." + ] + } + ] + } + }, + "artifact": "`One platform.
Endless ways to work together.`", + "ok": true, + "expected_task_pass": false + }, + { + "id": "inline-break-cannot-repair-missing-words", + "category": "format-normalization", + "root_cause": "inline_html_break_treated_as_visible_body_text", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "statement", + "alternatives": [ + "One platform. Endless ways to work together." + ] + } + ] + } + }, + "artifact": "## One platform.
Different ways to work together.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "visible-html-structured-reviews-retain-owned-fields", + "category": "structured-extraction", + "root_cause": "structured_record_parser_received_raw_html_instead_of_visible_body", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "review", + "minimum_records": 2, + "stable_text_anchors": [] + } + }, + "artifact": "

2026-09-01

First independently delivered review

2026-09-02

Second independently delivered review

", + "ok": true, + "expected_task_pass": true + }, + { + "id": "script-only-review-payload-cannot-supply-records", + "category": "structured-extraction", + "root_cause": "structured_record_parser_received_raw_html_instead_of_visible_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "review", + "minimum_records": 2, + "stable_text_anchors": [] + } + }, + "artifact": "
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "visible-html-review-cannot-borrow-missing-date", + "category": "structured-extraction", + "root_cause": "structured_record_parser_received_raw_html_instead_of_visible_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "review", + "minimum_records": 2, + "stable_text_anchors": [] + } + }, + "artifact": "

2026-09-01

First independently delivered review

Second independently delivered review

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "hidden-html-review-cannot-supply-record", + "category": "structured-extraction", + "root_cause": "structured_record_parser_received_raw_html_instead_of_visible_body", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "review", + "minimum_records": 2, + "stable_text_anchors": [] + } + }, + "artifact": "

2026-09-01

First independently delivered review

2026-09-02

Second independently delivered review

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "card-caption-retains-native-closed-heading-boundary", + "category": "format-normalization", + "root_cause": "native_card_caption_boundaries_serialized_as_closed_heading_and_link_end", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/", + "task_kind": "body", + "expected_any": [], + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "(?m)^# Catalog$", + "end_pattern": "(?m)^# End$", + "markdown_label_pattern": "(?s)
\\s*(?P