fix(xlsx): keep currency, percent, and date number formats

docling/backend/msexcel_backend.py

@@ -6,10 +6,12 @@
 import collections
 import logging
 import posixpath
+import re
 import shutil
 import warnings
 from copy import deepcopy
-from datetime import datetime, timezone
+from datetime import date, datetime, time, timezone
+from decimal import ROUND_HALF_UP, Decimal, localcontext
 from io import BytesIO
 from pathlib import Path
 from tempfile import mkdtemp
@@ -101,6 +103,7 @@
     from openpyxl.packaging.relationship import get_dependents, get_rels_path
     from openpyxl.reader.excel import ExcelReader
     from openpyxl.styles import PatternFill
+    from openpyxl.styles.numbers import is_date_format
     from openpyxl.utils.cell import range_boundaries
     from openpyxl.worksheet.merge import MergedCellRange
     from openpyxl.worksheet.worksheet import Worksheet
@@ -131,6 +134,181 @@
     "data without it."
 )
 
+_GENERAL_NUMBER_FORMATS: Final[frozenset[str]] = frozenset({"general", "@", ""})
+_EXCEL_DATE_TOKEN_RE: Final[re.Pattern[str]] = re.compile(
+    r'"[^"]*"|\\.|am/pm|yyyy|yy|mmmmm|mmmm|mmm|mm|m|dddd|ddd|dd|d|hh|h|ss|s|[ :/.,-]',
+    re.IGNORECASE,
+)
+
+
+def _displayed_cell_text(cell: Cell | MergedCell) -> str:
+    """Return the cell text as Excel would display it.
+
+    ``data_only=True`` gives the cached computed value. Applying the stored
+    number format keeps currency, percentages, and dates from collapsing to
+    ``str(value)`` (``12.5``, ``0.1``, ``2025-02-01 00:00:00``).
+    """
+    value = cell.value
+    if value is None:
+        return ""
+    return _format_excel_value(value, cell.number_format)
+
+
+def _format_excel_value(value: Any, number_format: str) -> str:
+    """Apply well-known Excel formats; leave custom formats as ``str(value)``."""
+    if isinstance(value, bool):
+        return str(value)
+    if isinstance(value, str):
+        return value
+
+    if number_format.lower() in _GENERAL_NUMBER_FORMATS:
+        return str(value)
+    if isinstance(value, (datetime, date, time)):
+        return _format_excel_date(value, number_format)
+    if isinstance(value, (int, float)):
+        if is_date_format(number_format):
+            # Workbooks are loaded by openpyxl with their own date epoch. Do not
+            # reinterpret a remaining numeric value with an assumed 1900 epoch.
+            return str(value)
+        return _format_excel_number(value, number_format)
+    return str(value)
+
+
+def _format_excel_number(value: int | float, fmt: str) -> str:
+    """Apply simple currency/percent sections, with Excel's rounding rule."""
+    # Conditions, unsupported locale markers, optional decimals and scaling
+    # must not be silently removed and mistaken for a supported number format.
+    without_colors = re.sub(
+        r"\[(?:Black|Blue|Cyan|Green|Magenta|Red|White|Yellow)\]",
+        "",
+        fmt,
+        flags=re.IGNORECASE,
+    )
+    currency_format = re.sub(r"\[\$\$-[0-9a-fA-F]+\]", '"$"', without_colors)
+    if "[" in currency_format or r"\;" in currency_format:
+        return str(value)
+    sections = re.split(r';(?=(?:[^"]*"[^"]*")*[^"]*$)', currency_format)
+    if len(sections) > 4:
+        return str(value)
+    section_index = (
+        1
+        if value < 0 and len(sections) > 1
+        else 2
+        if value == 0 and len(sections) > 2
+        else 0
+    )
+    section = sections[section_index]
+    if not section:
+        return ""
+    if re.fullmatch(r'"[^"]*"', section):
+        return section[1:-1]
+    if any(
+        re.search(r"[0#.,%]", literal) for literal in re.findall(r'"([^"]*)"', section)
+    ):
+        return str(value)
+    normalized = re.sub(r"_.", "", section).replace('"', "")
+    normalized = re.sub(r"\\([()$ -])", r"\1", normalized)
+    match = re.fullmatch(
+        r"(?P<prefix>[$( -]*)(?P<number>#,##0|0)(?:\.(?P<decimals>0{1,15}))?(?P<suffix>[$% )-]*)",
+        normalized,
+    )
+    if match is None or normalized.count("%") > 1 or normalized.count("$") > 1:
+        return str(value)
+    if "$" not in normalized and "%" not in normalized:
+        return str(value)
+    amount = Decimal(str(abs(value)))
+    if not amount.is_finite():
+        return str(value)
+    if "%" in normalized:
+        amount *= 100
+    decimals = len(match.group("decimals") or "")
+    # A local context also keeps large finite values from failing quantize.
+    with localcontext() as context:
+        context.prec = max(28, amount.adjusted() + decimals + 2)
+        rounded = amount.quantize(Decimal(1).scaleb(-decimals), rounding=ROUND_HALF_UP)
+    grouping = "," if "," in match.group("number") else ""
+    sign = "-" if value < 0 and section_index == 0 else ""
+    return f"{sign}{match.group('prefix')}{rounded:{grouping}.{decimals}f}{match.group('suffix')}"
+
+
+def _format_excel_date(value: Any, fmt: str) -> str:
+    """Render supported date tokens, preserving quoted and escaped literals."""
+    if not isinstance(value, (datetime, date, time)):
+        return str(value)
+    if fmt.lower() == "mm-dd-yy" and isinstance(value, date):
+        return value.isoformat().split("T")[0]
+    if fmt.lower() == "m/d/yy h:mm" and isinstance(value, datetime):
+        return value.isoformat(sep=" ")
+    parts = _EXCEL_DATE_TOKEN_RE.findall(fmt)
+    if "".join(parts) != fmt:
+        return str(value)
+    tokens = [part.lower() for part in parts if part[0].isalpha()]
+    if isinstance(value, time) and any(
+        token.startswith(("y", "d")) for token in tokens
+    ):
+        return str(value)
+    if not isinstance(value, (datetime, time)) and any(
+        token.startswith(("h", "s", "a")) for token in tokens
+    ):
+        return str(value)
+    twelve_hour = "am/pm" in tokens
+    dt = datetime.combine(date(1900, 1, 1), value) if isinstance(value, time) else value
+    rendered: list[str] = []
+    token_index = 0
+    for part in parts:
+        if part.startswith('"'):
+            rendered.append(part[1:-1])
+        elif part.startswith("\\"):
+            rendered.append(part[1:])
+        elif not part[0].isalpha():
+            rendered.append(part)
+        else:
+            token = tokens[token_index]
+            before = tokens[token_index - 1] if token_index else ""
+            after = tokens[token_index + 1] if token_index + 1 < len(tokens) else ""
+            token_index += 1
+            if token in ("m", "mm", "d", "dd", "h", "hh", "s", "ss"):
+                minute = token in ("m", "mm") and (
+                    before in ("h", "hh") or after in ("s", "ss")
+                )
+                if isinstance(value, time) and token in ("m", "mm") and not minute:
+                    return str(value)
+                if minute or token.startswith(("h", "s")):
+                    if not isinstance(dt, datetime):
+                        return str(value)
+                    number = (
+                        dt.minute
+                        if minute
+                        else dt.second
+                        if token.startswith("s")
+                        else dt.hour
+                    )
+                    if token.startswith("h") and twelve_hour:
+                        number = number % 12 or 12
+                else:
+                    number = dt.day if token.startswith("d") else dt.month
+                rendered.append(str(number).zfill(len(token)))
+            elif token == "am/pm":
+                if not isinstance(dt, datetime):
+                    return str(value)
+                rendered.append(part[:2] if dt.hour < 12 else part[3:])
+            elif token == "mmmmm":
+                if isinstance(value, time):
+                    return str(value)
+                rendered.append(dt.strftime("%B")[:1])
+            else:
+                directive = {
+                    "yyyy": "%Y",
+                    "yy": "%y",
+                    "mmm": "%b",
+                    "mmmm": "%B",
+                    "ddd": "%a",
+                    "dddd": "%A",
+                }[token]
+                rendered.append(dt.strftime(directive))
+    return "".join(rendered)
+
+
 # Maps an openpyxl chart object's ``tagname`` (the DrawingML element name, e.g.
 # "barChart") to the docling picture-classification label we tag the emitted
 # PictureItem with. Chart types not listed fall back to OTHER_CHART.
@@ -1423,7 +1601,7 @@ def has_content(r: int, c: int) -> bool:
                 if merged_cell_index.is_shadow(cell):
                     continue
 
-                cell_text = str(cell.value) if cell.value is not None else ""
+                cell_text = _displayed_cell_text(cell)
 
                 # Compute Spans
                 row_span, col_span = merged_cell_index.span_at(ri, rj)
@@ -2262,7 +2440,9 @@ def _resolve_reference(self, ref: str, max_cells: int) -> list[str]:
         ``"'Duck Observations'!$B$2:$B$7"``.  ``_chart_range`` finds the sheet
         and the bounds to read, and we read the *cached* values from the
         workbook (the backend loads with ``data_only=True``, so we get computed
-        numbers, not formulas).  Values are returned in row-major order.
+        numbers, not formulas). Values are returned in row-major order without
+        display formatting, so chart data keeps the precision of the cached
+        values.
 
         Args:
             ref: A range reference string, optionally sheet-qualified.