fix: a holiday offset written outside the construction moves the date

ovos_date_parser/holidays.py

@@ -287,23 +287,89 @@ def _holiday_extent(text: str, lang: str, anchor: datetime
     return None
 
 
+def _caller_spelling(text: str, written: str, remainder: str) -> Optional[str]:
+    """``remainder``, which is cut from ``written``, re-spelled from ``text``.
+
+    :func:`_as_written` may have put an accent back that the caller never
+    typed, and the remainder is handed to the caller, so it must come back in
+    the caller's own characters. Every substitution there replaces the same
+    number of characters, so the two strings are the same length and a
+    position in one is the same position in the other; that equality is
+    checked rather than assumed, and a mismatch returns None so the caller can
+    fall back.
+
+    Each word of the remainder is found in ``written`` left to right, never
+    searched for in the whole string, so a word that occurs twice takes its
+    own occurrence. A word that is not found returns None: chronologia owns
+    the remainder's spelling and may normalise a word one day, and a wrong
+    offset must not be guessed at.
+    """
+    if len(written) != len(text):
+        return None
+    out = []
+    at = 0
+    for word in remainder.split():
+        found = written.find(word, at)
+        if found < 0:
+            return None
+        out.append(text[found:found + len(word)])
+        at = found + len(word)
+    return " ".join(out)
+
+
 def extract_holiday_span(text: str, lang: str,
                          anchorDate: Optional[datetime] = None
                          ) -> Optional[Tuple[datetime, str]]:
     """Resolve a named holiday in ``text`` to ``(datetime, remainder)``.
 
-    The date is the occurrence the holiday phrase asks for: the next one by
-    default, and the one a determiner inside the phrase names when it
-    carries one ("next easter", "last christmas", "christmas eve"), as
-    chronologia reckons it from ``anchorDate``. A verb tense is not read —
-    "when was easter" answers with the next Easter, because chronologia
-    reads no verb — so this claims the determiner only. A word outside the holiday phrase is not read: it
-    stays in the remainder, so "how many days until christmas" and the
-    French "combien de jours avant noël" both answer with Christmas and both
-    keep their question.
-
-    The remainder is the caller's own text with the holiday phrase cut out of
-    it, and nothing else removed.
+    The date is the occurrence the phrase asks for: the next one by default,
+    the one a determiner inside the phrase names when it carries one ("next
+    easter", "last christmas", "christmas eve"), and the one an offset
+    outside the phrase names ("the day after christmas", "two days after
+    christmas", "le jour après noël"), as chronologia reckons it from
+    ``anchorDate``. A verb tense is not read — "when was easter" answers with
+    the next Easter, because chronologia reads no verb — so this claims the
+    determiner and the offset, never the verb.
+
+    The whole text is parsed, not the holiday construction's own characters.
+    Parsing the substring was what lost the offset: a modifier outside the
+    construction was never applied, so "the day after christmas" answered
+    25 December and handed "the day after" back as though it were question
+    words. :func:`_holiday_extent` stays, as the gate alone: a holiday
+    reading is kept only when a winning match is a holiday construction. What
+    the answer covers is then chronologia's to say, and it says it through
+    ``DateSpanResult.remainder``.
+
+    The remainder is therefore chronologia's, re-spelled with the caller's own
+    characters by :func:`_caller_spelling`, and it keeps every word the parse
+    did not consume: "how many days until christmas" and "quantos dias faltam
+    para o natal" both answer with Christmas and both keep their question.
+
+    **One known wrong answer, and it is chronologia's.** The French "combien
+    de jours avant noël" asks how many days remain before Christmas, so the
+    date is Christmas, and this answers 24 December with the remainder
+    "combien": chronologia reads "avant" after an interrogative quantity as an
+    offset. The English "how many days until christmas" and the French
+    "combien de jours jusqu'a noël" are both read correctly. Telling the two
+    French shapes apart needs a table of interrogatives per language, which
+    belongs to chronologia and is not copied here.
+    ``test_holidays_chronologia.py`` carries a cell holding that answer, so
+    the day chronologia fixes it the cell says so.
+
+    **A part-of-day offset is read as a time of day on the holiday, and the
+    direction is dropped.** "the night before christmas" answers 21:00 on
+    25 December, not the night of the 24th, and "the morning after christmas"
+    answers 06:00 on the 25th, not the morning of the 26th. The before and the
+    after make no difference to the answer, and every row of this family comes
+    back with an EMPTY remainder, so a caller cannot see that the modifier was
+    read at all. Eight rows are measured, in English, French and Portuguese.
+
+    This is the same defect as the French row above and is chronologia's in the
+    same way: the day offset is read correctly, so "the day after christmas"
+    gives 26 December, and only the part-of-day word is misread. Task T-7350
+    carries it. ``test_holidays_chronologia.py`` holds all eight answers in one
+    parametrised cell, so the day chronologia fixes any of them the cell says
+    which.
 
     Returns None when the utterance names no holiday in its own language, or
     when chronologia answered from something other than a holiday.
@@ -317,10 +383,9 @@ def extract_holiday_span(text: str, lang: str,
     extent = _holiday_extent(written, lang, anchor)
     if extent is None:
         return None
-    first, last = extent
     try:
         from chronologia import extract_timespan
-        result = extract_timespan(written[first:last], lang=_base_lang(lang),
+        result = extract_timespan(written, lang=_base_lang(lang),
                                   anchor=anchor,
                                   jurisdiction=_jurisdiction(lang))
     except Exception:
@@ -330,8 +395,13 @@ def extract_holiday_span(text: str, lang: str,
     start = result.span.start_datetime
     if start is None:  # a span outside the datetime range
         return None
-    remainder = re.sub(r"\s{2,}", " ", text[:first] + " " + text[last:])
-    return start, remainder.strip()
+    remainder = _caller_spelling(text, written, result.remainder)
+    if remainder is None:
+        # the offsets do not line up, so the holiday phrase is cut out by the
+        # extent instead; the date stands, only the remainder falls back
+        first, last = extent
+        remainder = text[:first] + " " + text[last:]
+    return start, re.sub(r"\s{2,}", " ", remainder).strip()
 
 
 def extract_holiday_date(text: str, lang: str,

test/test_holidays_chronologia.py

@@ -161,28 +161,150 @@ def test_a_sentence_that_merely_names_a_holiday_costs_milliseconds():
 
 @pytest.mark.parametrize("lang,utterance,kept", [
     ("en-US", "how many days until christmas", "how many days until"),
-    ("fr-FR", "combien de jours avant noël", "combien de jours avant"),
     ("pt-PT", "quantos dias faltam para o natal", "quantos dias faltam para o"),
     ("en-US", "play some christmas music", "play some music"),
 ])
-def test_the_remainder_keeps_every_word_outside_the_holiday_phrase(
+def test_the_remainder_keeps_every_word_the_parse_did_not_consume(
         lang, utterance, kept):
-    """Only the holiday phrase leaves the remainder.
+    """A question word the parse did not read stays in the remainder.
 
-    "combien de jours avant noël" came back as 'combien': chronologia
-    applied "avant" as an offset from outside the match and took "de jours"
-    with it, so a French question lost words its English sibling kept.
+    The remainder is chronologia's, re-spelled with the caller's own
+    characters, so it holds exactly what the parse left. The French sibling
+    of the first row is not here: it is a known wrong answer and has its own
+    cell below.
     """
     got = extract_datetime(utterance, lang, REF)
     assert got is not None
     assert got[1] == kept
 
 
-def test_the_same_question_reads_the_same_in_english_and_french():
-    """The French question asked about Christmas and was answered Christmas Eve."""
+def test_the_english_question_keeps_its_words_and_answers_christmas():
+    """The shape the French row below should have, and does not."""
     english = extract_datetime("how many days until christmas", "en-US", REF)
-    french = extract_datetime("combien de jours avant noël", "fr-FR", REF)
-    assert english[0] == french[0] == CHRISTMAS
+    assert english[0] == CHRISTMAS
+    assert english[1] == "how many days until"
+
+
+# --- known answers, not correct answers -------------------------------------
+
+def test_a_french_interrogative_quantity_before_a_holiday_is_read_as_an_offset():
+    """Known answer, and the defect is chronologia's: task T-6896.
+
+    "combien de jours avant noël" asks how many days remain before Christmas,
+    so the date is Christmas, 25 December. chronologia reads "avant" after an
+    interrogative quantity as an offset and answers the day before, taking
+    "de jours avant" into the match and leaving "combien" behind.
+
+    The two controls are the shapes that ARE read correctly, so the cell
+    blames the interrogative and not the language or the preposition: the
+    English "how many days until christmas" above, and the French
+    "combien de jours jusqu'a noël" here.
+
+    Parsing the holiday construction's own substring hid this row, at the
+    price of four silently wrong offset dates ("the day after christmas" and
+    its siblings). The whole text is right on those four and wrong on this
+    one. The day chronologia fixes it, this cell fails and says so.
+    """
+    got = extract_datetime("combien de jours avant noël", "fr-FR", REF)
+    assert got is not None
+    assert got[0] == datetime(2026, 12, 24, 0, 0)
+    assert got[0] != CHRISTMAS
+    assert got[1] == "combien"
+
+    control = extract_datetime("combien de jours jusqu'à noël", "fr-FR", REF)
+    assert control[0] == CHRISTMAS
+
+
+#: The C1 rows: an offset written OUTSIDE the holiday construction. Each date
+#: is counted by hand from Christmas, 25 December 2026.
+#: The C1 rows: an offset written OUTSIDE the holiday construction. Each date
+#: is counted by hand from Christmas, 25 December 2026.
+OFFSET_UTTERANCES = [
+    ("en-US", "the day after christmas", datetime(2026, 12, 26, 0, 0)),
+    ("en-US", "the day before christmas", datetime(2026, 12, 24, 0, 0)),
+    ("en-US", "two days after christmas", datetime(2026, 12, 27, 0, 0)),
+    ("fr-FR", "le jour après noël", datetime(2026, 12, 26, 0, 0)),
+    ("fr-FR", "le jour apres noel", datetime(2026, 12, 26, 0, 0)),
+]
+
+#: The rows above that reach the holiday layer through `extract_datetime`.
+#: "two days after christmas" is not one of them and has its own cell below.
+OFFSET_THROUGH_EXTRACT_DATETIME = [
+    row for row in OFFSET_UTTERANCES if not row[1].startswith("two days")
+]
+
+
+@pytest.mark.parametrize("lang,utterance,expected", OFFSET_UTTERANCES)
+def test_an_offset_outside_the_holiday_phrase_moves_the_date(
+        lang, utterance, expected):
+    """The C1 regression, one cell per row, on the layer that owns it.
+
+    Each of these answered 25 December, the holiday itself, and handed the
+    offset back in the remainder as though it were question words. The parse
+    read the holiday construction's own characters, so a modifier outside the
+    construction was never applied.
+
+    The last row is the same French sentence without its accents, which is
+    what speech to text produces; it must read the same.
+    """
+    got = extract_holiday_span(utterance, lang, REF)
+    assert got is not None
+    assert got[0] == expected
+    assert got[0] != CHRISTMAS
+
+
+@pytest.mark.parametrize("lang,utterance,expected", OFFSET_UTTERANCES)
+def test_an_offset_phrase_leaves_no_remainder(lang, utterance, expected):
+    """The other half: the offset words are consumed, not handed back.
+
+    A caller that reads the remainder as the rest of the command would have
+    been given "the day after" to act on.
+    """
+    got = extract_holiday_span(utterance, lang, REF)
+    assert got is not None
+    assert got[1] == ""
+
+
+@pytest.mark.parametrize("lang,utterance,expected",
+                         OFFSET_THROUGH_EXTRACT_DATETIME)
+def test_an_offset_phrase_reads_the_same_through_extract_datetime(
+        lang, utterance, expected):
+    """The whole call, not the layer alone, for the rows that reach it.
+
+    Without this the module could pass on a library whose entry point never
+    consults the holiday layer at all.
+    """
+    got = extract_datetime(utterance, lang, REF)
+    assert got is not None
+    assert got[0] == expected
+
+
+def test_two_days_after_christmas_is_answered_by_the_engine_not_the_holiday():
+    """Known answer, and a different defect from the one above.
+
+    The holiday layer reads this row correctly, and the cells above assert
+    that. `extract_datetime` never asks it: the per-language engine reads
+    "two days" as an offset from the anchor, answers 27 September 2026 and
+    hands back "after christmas", so the engine-first order ends the walk
+    before the holiday layer is reached.
+
+    That is the shadowing family of finding 5 of the #369 review, but not the
+    case `holiday_overrides_engine` covers: its rule asks whether the words
+    the engine consumed all lie inside the holiday phrase, and "two days"
+    does not lie inside "christmas". The rule is right to decline here; the
+    engine's partial read is the defect, and it is filed on its own.
+
+    The cell holds the answer this library gives today so the day that is
+    fixed it fails and says so.
+    """
+    got = extract_datetime("two days after christmas", "en-US", REF)
+    assert got is not None
+    assert got[0] == datetime(2026, 9, 27, 0, 0)
+    assert got[1] == "after christmas"
+
+    # the control: the layer the walk skipped has the right answer
+    layer = extract_holiday_span("two days after christmas", "en-US", REF)
+    assert layer[0] == datetime(2026, 12, 27, 0, 0)
 
 
 @pytest.mark.parametrize("lang,plain,written", [
@@ -227,3 +349,79 @@ def test_a_tense_inside_the_holiday_phrase_still_reads(utterance, expected):
     "christmas eve" each as one match over all their words.
     """
     assert extract_datetime(utterance, "en-US", REF)[0] == expected
+
+
+# --- the part-of-day offset family, T-7244 from reviewer-d's review of #381 -
+
+# A part-of-day word written as the offset is read as a time of day ON the
+# holiday, and the before/after is dropped: "before" and "after" give the same
+# answer. Every row comes back with an EMPTY remainder, so a caller cannot see
+# that the modifier was read at all. That is what makes the family worth a cell
+# rather than a note: dev at least handed the offset back in the remainder.
+#
+# The defect is chronologia's, like the French interrogative above, and task
+# T-7350 carries it. The day offset is read correctly on the same anchor
+# ("the day after christmas" gives 26 December), so the part-of-day word is the
+# whole of it.
+#
+# These call extract_holiday_span rather than extract_datetime, because it is
+# that function's contract this family bounds. Through extract_datetime the
+# per-language engine answers these utterances first and the holiday layer is
+# never reached, so extract_datetime would assert the engine's answer, which is
+# a different wrong answer and not this one.
+
+PART_OF_DAY_OFFSETS = [
+    # lang, utterance, the answer given today, what the utterance names
+    ("en-US", "the night before christmas",
+     datetime(2026, 12, 25, 21, 0), "the night of 24 December"),
+    ("en-US", "the morning after christmas",
+     datetime(2026, 12, 25, 6, 0), "the morning of 26 December"),
+    ("en-US", "the evening after christmas",
+     datetime(2026, 12, 25, 18, 0), "the evening of 26 December"),
+    ("en-US", "the night after christmas",
+     datetime(2026, 12, 25, 21, 0), "the night of 26 December"),
+    ("en-US", "the morning before christmas",
+     datetime(2026, 12, 25, 6, 0), "the morning of 24 December"),
+    ("fr-FR", "le soir avant noël",
+     datetime(2026, 12, 25, 18, 0), "le soir du 24 décembre"),
+    ("fr-FR", "le matin après noël",
+     datetime(2026, 12, 25, 4, 0), "le matin du 26 décembre"),
+    ("pt-PT", "a noite antes do natal",
+     datetime(2026, 12, 25, 19, 0), "a noite de 24 de dezembro"),
+]
+
+
+@pytest.mark.parametrize("lang,utterance,given,asked", PART_OF_DAY_OFFSETS)
+def test_a_part_of_day_offset_is_read_as_a_time_on_the_holiday(
+        lang, utterance, given, asked):
+    """Known answer, not a correct answer: the defect is chronologia's, T-7350.
+
+    Each expected value is the answer this library gives today. What the
+    utterance actually names is written beside it. The day chronologia fixes
+    any of these the cell fails and says which.
+    """
+    got = extract_holiday_span(utterance, lang, REF)
+    assert got is not None, f"{utterance!r} no longer reaches the holiday layer"
+    assert got[0] == given, f"{utterance!r} asks for {asked}"
+    assert got[1] == "", (
+        f"{utterance!r} left {got[1]!r} over; an empty remainder is what makes "
+        "this family invisible to a caller, and the cell holds that too")
+
+
+def test_the_direction_makes_no_difference_to_a_part_of_day_offset():
+    """The sharpest statement of the defect: before and after agree.
+
+    A cell per row could pass while the two directions still collapsed onto
+    one answer, so the collapse is asserted on its own.
+    """
+    before = extract_holiday_span("the night before christmas", "en-US", REF)
+    after = extract_holiday_span("the night after christmas", "en-US", REF)
+    assert before[0] == after[0] == datetime(2026, 12, 25, 21, 0)
+
+
+def test_a_day_offset_is_still_read_correctly():
+    """Control: the direction IS honoured for a day offset, so the family
+    above blames the part-of-day word and not the offset machinery."""
+    got = extract_holiday_span("the day after christmas", "en-US", REF)
+    assert got is not None
+    assert got[0] == datetime(2026, 12, 26, 0, 0)