diff --git a/docling/backend/webvtt_backend.py b/docling/backend/webvtt_backend.py index 2a0aa462f3..46374ba712 100644 --- a/docling/backend/webvtt_backend.py +++ b/docling/backend/webvtt_backend.py @@ -53,6 +53,30 @@ class AnnotatedPar: items: list[AnnotatedText] +def _detach_span_edge_whitespace(items: list[AnnotatedText]) -> list[AnnotatedText]: + """Drop the whitespace at the edges of the spans that carry formatting. + + Emphasis markers wrap an item's text verbatim, so a space at the edge of a + bold/italic/underline item lands *inside* the markers (``* text*``). + CommonMark does not treat a space-adjacent ``*`` as a flanking delimiter, so + the markup is emitted but never renders. The inline-group serializer already + inserts a separator between the items it joins, so the whitespace can simply + be dropped instead of being moved elsewhere. + """ + detached: list[AnnotatedText] = [] + for item in items: + if item.formatting is None: + detached.append(item) + continue + core = item.text.strip() + if not core: + # A whitespace-only span would emit bare markers that render as + # literal asterisks. + continue + detached.append(item.copy_meta(core)) + return detached + + class WebVTTDocumentBackend(DeclarativeDocumentBackend): """Declarative backend for WebVTT (.vtt) files. @@ -193,10 +217,11 @@ def _add_text_item( identifier = str(block.identifier) if block.identifier else None _extract_components(block.payload) for par in cue_text: - if not par.items: + items = _detach_span_edge_whitespace(par.items) + if not items: continue - if len(par.items) == 1: - item = par.items[0] + if len(items) == 1: + item = items[0] _add_text_item( text=item.text, formatting=item.formatting, @@ -206,7 +231,7 @@ def _add_text_item( group = doc.add_inline_group( "WebVTT cue span", content_layer=ContentLayer.BODY ) - for item in par.items: + for item in items: _add_text_item( text=item.text, formatting=item.formatting, diff --git a/tests/data/webvtt/groundtruth/webvtt_example_04.vtt.itxt b/tests/data/webvtt/groundtruth/webvtt_example_04.vtt.itxt index 93feba5e9a..cc7a9fb3b2 100644 --- a/tests/data/webvtt/groundtruth/webvtt_example_04.vtt.itxt +++ b/tests/data/webvtt/groundtruth/webvtt_example_04.vtt.itxt @@ -2,13 +2,12 @@ item-0 at level 0: unspecified: group _root_ item-1 at level 1: text: Last night the chef surprised us with a culinary adventure. item-2 at level 1: inline: group WebVTT cue span item-3 at level 2: text: The waiter offered a - item-4 at level 2: text: steaming bowl of + item-4 at level 2: text: steaming bowl of item-5 at level 2: text: paella item-6 at level 2: text: that instantly transported the diners to a sunny Mediterranean coast. item-7 at level 1: inline: group WebVTT cue span item-8 at level 2: text: The dessert’s item-9 at level 2: text: unexpected - item-10 at level 2: text: - item-11 at level 2: text: arcobaleno - item-12 at level 2: text: of flavors - item-13 at level 2: text: left everyone in awe. \ No newline at end of file + item-10 at level 2: text: arcobaleno + item-11 at level 2: text: of flavors + item-12 at level 2: text: left everyone in awe. \ No newline at end of file diff --git a/tests/data/webvtt/groundtruth/webvtt_example_04.vtt.json b/tests/data/webvtt/groundtruth/webvtt_example_04.vtt.json index b6c9fa4453..40829e3f2b 100644 --- a/tests/data/webvtt/groundtruth/webvtt_example_04.vtt.json +++ b/tests/data/webvtt/groundtruth/webvtt_example_04.vtt.json @@ -75,9 +75,6 @@ }, { "$ref": "#/texts/9" - }, - { - "$ref": "#/texts/10" } ], "content_layer": "body", @@ -143,8 +140,8 @@ "identifier": "agcvs-08234" } ], - "orig": "steaming bowl of ", - "text": "steaming bowl of ", + "orig": "steaming bowl of", + "text": "steaming bowl of", "formatting": { "bold": false, "italic": true, @@ -264,33 +261,6 @@ "identifier": "agcvs-08234" } ], - "orig": " ", - "text": " ", - "formatting": { - "bold": false, - "italic": true, - "underline": false, - "strikethrough": false, - "script": "baseline" - } - }, - { - "self_ref": "#/texts/8", - "parent": { - "$ref": "#/groups/1" - }, - "children": [], - "content_layer": "body", - "label": "text", - "prov": [], - "source": [ - { - "kind": "track", - "start_time": 14760.0, - "end_time": 14818.239, - "identifier": "agcvs-08234" - } - ], "orig": "arcobaleno", "text": "arcobaleno", "formatting": { @@ -302,7 +272,7 @@ } }, { - "self_ref": "#/texts/9", + "self_ref": "#/texts/8", "parent": { "$ref": "#/groups/1" }, @@ -318,8 +288,8 @@ "identifier": "agcvs-08234" } ], - "orig": " of flavors", - "text": " of flavors", + "orig": "of flavors", + "text": "of flavors", "formatting": { "bold": false, "italic": true, @@ -329,7 +299,7 @@ } }, { - "self_ref": "#/texts/10", + "self_ref": "#/texts/9", "parent": { "$ref": "#/groups/1" }, diff --git a/tests/data/webvtt/groundtruth/webvtt_example_04.vtt.md b/tests/data/webvtt/groundtruth/webvtt_example_04.vtt.md index f2312a059c..5649fbb7c1 100644 --- a/tests/data/webvtt/groundtruth/webvtt_example_04.vtt.md +++ b/tests/data/webvtt/groundtruth/webvtt_example_04.vtt.md @@ -1,5 +1,5 @@ Last night the chef surprised us with a culinary adventure. -The waiter offered a *steaming bowl of * *paella* that instantly transported the diners to a sunny Mediterranean coast. +The waiter offered a *steaming bowl of* *paella* that instantly transported the diners to a sunny Mediterranean coast. -The dessert’s ***unexpected*** * * *arcobaleno* * of flavors* left everyone in awe. \ No newline at end of file +The dessert’s ***unexpected*** *arcobaleno* *of flavors* left everyone in awe. \ No newline at end of file diff --git a/tests/test_backend_vtt.py b/tests/test_backend_vtt.py index 257aa861b4..db85fdb11e 100644 --- a/tests/test_backend_vtt.py +++ b/tests/test_backend_vtt.py @@ -150,6 +150,56 @@ def test_styling_and_voice_tags_stripped(converter): assert _process_vtt_doc(doc) == expected +def test_emphasis_markers_stay_clear_of_span_edge_whitespace(converter): + """A space inside the emphasis markers stops CommonMark from flanking them. + + The italic span ends with the space that separates it from the nested voice + span. Emitting that space inside the markers produces ``*steaming bowl of *``, + where the closing ``*`` is preceded by whitespace and therefore is not a + right-flanking delimiter: the markup is written out but never renders. + """ + vtt = """ +WEBVTT + +00:00:01.000 --> 00:00:04.000 +The waiter offered a steaming bowl of paella that arrived. +""" + stream = _create_vtt_stream(vtt) + doc = converter.convert(stream).document + md = doc.export_to_markdown(escape_html=False) + + assert "*steaming bowl of*" in md + assert "*paella*" in md + assert "*steaming bowl of *" not in md + + +def test_whitespace_only_span_emits_no_bare_markers(converter): + """A span holding nothing but whitespace used to emit unrenderable markers. + + The space between ```` and ```` is itself an italic span, so it was + exported as a pair of markers with the space between them, which shows up as + literal asterisks. The following span started with the same separator space, + which broke its opening marker in the same way. + """ + vtt = """ +WEBVTT + +00:00:01.000 --> 00:00:04.000 +The dessert's unexpected arcobaleno of flavors left everyone in awe. +""" + stream = _create_vtt_stream(vtt) + doc = converter.convert(stream).document + md = doc.export_to_markdown(escape_html=False) + + assert "***unexpected***" in md + assert "*arcobaleno*" in md + assert "*of flavors*" in md + # the whitespace-only span contributed a bare pair of markers, and the span + # after it opened with a space + assert "* * *" not in md + assert "* of flavors*" not in md + + def test_blank_cue_contributes_no_text(converter): # First cue has text; second cue is intentionally blank (zero transcript lines). vtt = """