From b851056b58a49ff49bfbf058df2bcd70758aa146 Mon Sep 17 00:00:00 2001 From: Neelabh Kumar Date: Tue, 20 Oct 2026 07:26:54 -0700 Subject: [PATCH 1/2] Derender site URLs in markdown comparison MarkdownString overrides _normalize_string to keep Unicode and case, and the override also dropped the URL derendering that the base class does first. The expected README of template 88 (tasks 562 to 566) writes its links with the site template (__REDDIT__/f/DIY/...), while a committed README holds the site's address (http://localhost:9999/f/DIY/...), so the content check could not match a real commit. Derender the same way the base class does. --- .../evaluation/data_types/markdown_string.py | 9 ++++- .../data_types/test_markdown_string.py | 35 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/webarena_verified/core/evaluation/data_types/markdown_string.py b/src/webarena_verified/core/evaluation/data_types/markdown_string.py index 34a0054..fc7179e 100644 --- a/src/webarena_verified/core/evaluation/data_types/markdown_string.py +++ b/src/webarena_verified/core/evaluation/data_types/markdown_string.py @@ -1,5 +1,6 @@ """Markdown string data type for normalized comparison.""" +import contextlib import re from typing import Any @@ -38,8 +39,14 @@ def _normalize_string(self, s: str) -> str: - Case sensitivity - Special characters - We only strip leading/trailing whitespace. + We only derender site URLs (as the base class does) and strip leading/trailing whitespace. """ + # Derender site URLs (best effort), so a link written with the site's address matches + # an expected link written with its template (e.g. __REDDIT__/f/DIY/...) + if self.derender_url_fct is not None and "http" in s: + with contextlib.suppress(Exception): + s = self.derender_url_fct(s, strict=False) + return s.strip() def _type_normalize(self, value: Any) -> str: diff --git a/tests/core/evaluation/data_types/test_markdown_string.py b/tests/core/evaluation/data_types/test_markdown_string.py index deb75fc..6a422f2 100644 --- a/tests/core/evaluation/data_types/test_markdown_string.py +++ b/tests/core/evaluation/data_types/test_markdown_string.py @@ -1,8 +1,12 @@ """Unit tests for MarkdownString data type.""" +from functools import partial + import pytest from webarena_verified.core.evaluation.data_types import MarkdownString +from webarena_verified.types.config import WebArenaVerifiedConfig +from webarena_verified.types.task import WebArenaSite # ===== Basic Functionality Tests ===== @@ -223,6 +227,37 @@ def test_real_world_example(): assert markdown1.normalized == markdown2.normalized +@pytest.fixture +def derender_url_fct(): + """Derender function as NetworkEventEvaluator passes it for a GitLab and Reddit task.""" + config = WebArenaVerifiedConfig.model_validate( + {"environments": {"gitlab": {"urls": ["http://localhost:8023"]}, "reddit": {"urls": ["http://localhost:9999"]}}} + ) + return partial(config.derender_url, sites=(WebArenaSite.GITLAB, WebArenaSite.REDDIT)) + + +def test_site_urls_derendered(derender_url_fct): + """Test that links with the site's address match expected links with its template (task 564).""" + # Expected README content of task 564 + expected = ( + "# live_a_life\n\n## Most Active DIY Threads\n\n" + "- [Separate glued plastic parts](__REDDIT__/f/DIY/118903/separate-glued-plastic-parts)\n" + "- [How would you fix this dryer vent mess?](__REDDIT__/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess)\n" + "- [Basement Bulkhead/Soffit + Wall Framing](__REDDIT__/f/DIY/118935/basement-bulkhead-soffit-wall-framing)" + ) + # README content committed by an agent run on task 564 + actual = ( + "# live_a_life\n\n## Most Active DIY Threads\n\n" + "- [Separate glued plastic parts](http://localhost:9999/f/DIY/118903/separate-glued-plastic-parts)\n" + "- [How would you fix this dryer vent mess?](http://localhost:9999/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess)\n" + "- [Basement Bulkhead/Soffit + Wall Framing](http://localhost:9999/f/DIY/118935/basement-bulkhead-soffit-wall-framing)\n" + ) + + assert MarkdownString(expected, derender_url_fct=derender_url_fct) == MarkdownString( + actual, derender_url_fct=derender_url_fct + ) + + # ===== Alternatives Support Tests ===== From 2528b181116067378626105a95c43a9b1b3e9db6 Mon Sep 17 00:00:00 2001 From: Neelabh Kumar Date: Tue, 20 Oct 2026 07:27:54 -0700 Subject: [PATCH 2/2] Fold typographic quotes and apostrophes in markdown comparison The forum renders the title "GE Water Heater Pilot Light Won't Stay Lit" with a typographic apostrophe (U+2019), and an agent that copies the title from the page commits it so; the expected README of tasks 562, 563, 565 and 566 writes a straight one. MarkdownString keeps Unicode as is, so the content check fails on that one character. Fold the four typographic quotes and apostrophes to their ASCII forms, as NormalizedString already does through unidecode. --- .../evaluation/data_types/markdown_string.py | 9 +++- .../data_types/test_markdown_string.py | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/webarena_verified/core/evaluation/data_types/markdown_string.py b/src/webarena_verified/core/evaluation/data_types/markdown_string.py index fc7179e..b1daa70 100644 --- a/src/webarena_verified/core/evaluation/data_types/markdown_string.py +++ b/src/webarena_verified/core/evaluation/data_types/markdown_string.py @@ -6,6 +6,9 @@ from .base import NormalizedType +# Typographic quotes and apostrophes (U+2018, U+2019, U+201C, U+201D) and their ASCII forms +_TYPOGRAPHIC_QUOTES = str.maketrans({"\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"'}) + class MarkdownString(NormalizedType[str]): """Markdown string normalization for basic comparison. @@ -39,7 +42,8 @@ def _normalize_string(self, s: str) -> str: - Case sensitivity - Special characters - We only derender site URLs (as the base class does) and strip leading/trailing whitespace. + We only derender site URLs (as the base class does), fold typographic quotes and apostrophes + to ASCII, and strip leading/trailing whitespace. """ # Derender site URLs (best effort), so a link written with the site's address matches # an expected link written with its template (e.g. __REDDIT__/f/DIY/...) @@ -47,7 +51,8 @@ def _normalize_string(self, s: str) -> str: with contextlib.suppress(Exception): s = self.derender_url_fct(s, strict=False) - return s.strip() + # Fold typographic quotes and apostrophes (a site may render "Won't" with U+2019) + return s.translate(_TYPOGRAPHIC_QUOTES).strip() def _type_normalize(self, value: Any) -> str: """Normalize markdown string for comparison. diff --git a/tests/core/evaluation/data_types/test_markdown_string.py b/tests/core/evaluation/data_types/test_markdown_string.py index 6a422f2..08585ec 100644 --- a/tests/core/evaluation/data_types/test_markdown_string.py +++ b/tests/core/evaluation/data_types/test_markdown_string.py @@ -258,6 +258,56 @@ def test_site_urls_derendered(derender_url_fct): ) +@pytest.mark.parametrize( + ("value1", "value2"), + [ + ("Won\u2019t", "Won't"), + ("\u2018single\u2019", "'single'"), + ("\u201cdouble\u201d", '"double"'), + ], +) +def test_typographic_quotes_folded(value1, value2): + """Test that typographic quotes and apostrophes match their ASCII forms.""" + md1 = MarkdownString(value1) + md2 = MarkdownString(value2) + assert md1 == md2 + assert md2 == md1 # Symmetric + + +def test_site_rendered_title_matches(derender_url_fct): + """Test a README committed for task 562 against its expected content. + + The forum renders the fourth title with a typographic apostrophe and the README + holds the site's address; the expected content writes "Won't" and __REDDIT__. + """ + # Expected README content of task 562 + expected = ( + "# Awesome_DIY_ideas\n\n## Most Active DIY Threads\n\n" + "- [Separate glued plastic parts](__REDDIT__/f/DIY/118903/separate-glued-plastic-parts)\n" + "- [How would you fix this dryer vent mess?](__REDDIT__/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess)\n" + "- [Basement Bulkhead/Soffit + Wall Framing](__REDDIT__/f/DIY/118935/basement-bulkhead-soffit-wall-framing)\n" + "- [GE Water Heater Pilot Light Won't Stay Lit](__REDDIT__/f/DIY/118904/ge-water-heater-pilot-light-won-t-stay-lit)\n" + "- [Attempting to move a wall outlet in my basement a few inches to the left and am totally stumped. " + "Any help GREATLY appreciated!!](__REDDIT__/f/DIY/118960/attempting-to-move-a-wall-outlet-in-my-basement-a-few-inches)\n" + "- [AFCI Outlet Question](__REDDIT__/f/DIY/118931/afci-outlet-question)" + ) + # README content committed by an agent run on task 562 + actual = ( + "# Awesome_DIY_ideas\n\n## Most Active DIY Threads\n\n" + "- [Separate glued plastic parts](http://localhost:9999/f/DIY/118903/separate-glued-plastic-parts)\n" + "- [How would you fix this dryer vent mess?](http://localhost:9999/f/DIY/118923/how-would-you-fix-this-dryer-vent-mess)\n" + "- [Basement Bulkhead/Soffit + Wall Framing](http://localhost:9999/f/DIY/118935/basement-bulkhead-soffit-wall-framing)\n" + "- [GE Water Heater Pilot Light Won\u2019t Stay Lit](http://localhost:9999/f/DIY/118904/ge-water-heater-pilot-light-won-t-stay-lit)\n" + "- [Attempting to move a wall outlet in my basement a few inches to the left and am totally stumped. " + "Any help GREATLY appreciated!!](http://localhost:9999/f/DIY/118960/attempting-to-move-a-wall-outlet-in-my-basement-a-few-inches)\n" + "- [AFCI Outlet Question](http://localhost:9999/f/DIY/118931/afci-outlet-question)\n" + ) + + assert MarkdownString(expected, derender_url_fct=derender_url_fct) == MarkdownString( + actual, derender_url_fct=derender_url_fct + ) + + # ===== Alternatives Support Tests =====