feat: avoid padded-grid allocation in table exports

.github/workflows/stackgen-serve.yml

@@ -0,0 +1,99 @@
+name: StackGen Serve image
+on:
+  pull_request:
+    paths:
+      - 'docling/**'
+      - 'pyproject.toml'
+      - 'integrations/stackgen_serve/**'
+      - '.github/workflows/stackgen-serve.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'docling/**'
+      - 'pyproject.toml'
+      - 'integrations/stackgen_serve/**'
+      - '.github/workflows/stackgen-serve.yml'
+  workflow_dispatch:
+    inputs:
+      publish:
+        description: Publish the image (main branch only)
+        type: boolean
+        default: false
+permissions:
+  contents: read
+jobs:
+  image-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [ubuntu-24.04, ubuntu-24.04-arm]
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: astral-sh/setup-uv@v7
+      - name: Python format and lint
+        run: |
+          uvx --from ruff==0.15.12 ruff format --check --config pyproject.toml integrations/stackgen_serve
+          uvx --from ruff==0.15.12 ruff check --config pyproject.toml integrations/stackgen_serve
+      - name: Build checkout wheels and local image
+        run: sh integrations/stackgen_serve/build.sh
+      - name: Verify fork, unit tests, HTTP default, workers and reload
+        run: sh integrations/stackgen_serve/test-image.sh
+
+  publish:
+    needs: image-test
+    if: >-
+      github.repository == 'stackgenhq/docling'
+      && github.ref == 'refs/heads/main'
+      && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish))
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      packages: write
+    concurrency:
+      group: stackgen-serve-publication
+      cancel-in-progress: false
+    env:
+      IMAGE: ghcr.io/stackgenhq/docling-serve
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: astral-sh/setup-uv@v7
+      - name: Build wheels from the tested revision
+        run: |
+          uv build --wheel --build-constraints integrations/stackgen_serve/build-constraints.txt --out-dir integrations/stackgen_serve/dist
+          uv build --wheel --build-constraints integrations/stackgen_serve/build-constraints.txt integrations/stackgen_serve --out-dir integrations/stackgen_serve/dist
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Reject an outdated main revision
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          latest=$(gh api "repos/$GITHUB_REPOSITORY/commits/main" --jq .sha)
+          test "$latest" = "$GITHUB_SHA" || { echo 'Main advanced; do not overwrite its image tag.' >&2; exit 1; }
+      - name: Publish multi-platform image
+        id: image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: integrations/stackgen_serve/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          network: none
+          push: true
+          tags: |
+            ${{ env.IMAGE }}:main
+            ${{ env.IMAGE }}:sha-${{ github.sha }}
+          labels: |
+            org.opencontainers.image.source=https://github.com/stackgenhq/docling
+            org.opencontainers.image.revision=${{ github.sha }}
+      - name: Report deployment pin
+        env:
+          DIGEST: ${{ steps.image.outputs.digest }}
+        run: |
+          printf 'Published image pin: `%s@%s`\n' "$IMAGE" "$DIGEST" >> "$GITHUB_STEP_SUMMARY"

integrations/stackgen_serve/Dockerfile

@@ -0,0 +1,11 @@
+# Build the wheels from this checkout before building with the repository root context.
+# CPU execution only. Use the upstream CPU variant instead of CUDA dependencies.
+FROM ghcr.io/docling-project/docling-serve-cpu:v1.36.0@sha256:225c8586e20d5d0fc6811a9e0e044fa602bcc4393f00389009bad42d6787b58f
+COPY --chown=1001:0 integrations/stackgen_serve/dist/*.whl /tmp/fork-wheels/
+# The base has docling-slim, not the legacy overlapping docling distribution.
+# Fail if that layout changes. Do not resolve or upgrade dependencies.
+RUN python -c "from importlib.metadata import distributions; assert 'docling' not in {d.metadata['Name'] for d in distributions()}" \
+    && python -m pip install --no-index --no-deps --force-reinstall /tmp/fork-wheels/*.whl \
+    && rm -rf /tmp/fork-wheels
+ENTRYPOINT ["/opt/app-root/bin/python", "-m", "stackgen_docling"]
+CMD ["run"]

integrations/stackgen_serve/Dockerfile.dockerignore

@@ -0,0 +1,6 @@
+# Only the two built wheels belong in the Docker build context.
+**
+!integrations/
+!integrations/stackgen_serve/
+!integrations/stackgen_serve/dist/
+!integrations/stackgen_serve/dist/*.whl

integrations/stackgen_serve/README.md

@@ -0,0 +1,79 @@
+# StackGen record-table Serve integration
+
+## Benefit
+
+Markdown exports iterate actual table cells instead of constructing
+`TableData.grid`. Sparse tables do not allocate a padded rectangle. Explicit
+empty values, source coordinates, column/row header flags, merged ranges and
+rich content remain available. Each table uses continuous `Record N` blocks, not headings.
+The structured document and its JSON export remain unchanged. Parsing, sorting
+and output still use memory; this is not protection against all OOM failures.
+
+## Image and startup
+
+This repository owns the exporter and Serve startup adapter. The image installs
+both wheels built from this checkout. It uses the digest-pinned upstream
+`docling-serve-cpu` base: Serve 1.36.0, Core 2.99.0 and Jobkit 3.8.1. It supports
+CPU execution, not GPU/CUDA. Runtime installation is offline with `--no-deps`;
+other base dependencies are not upgraded. Do not install an overlapping legacy
+`docling` distribution alongside `docling-slim`.
+
+The launcher enables the exporter before Serve imports. An importable factory
+also enables it in spawned workers and reload children. Importing the adapter
+alone does not change SDK behavior. Explicit table serializers override its
+default. The adapter uses private Core helpers for annotations and caption
+placement; update the regression tests before changing the pinned runtime.
+
+## Local validation
+
+Run from the repository root with Python 3.11+, uv and Docker:
+
+```sh
+sh integrations/stackgen_serve/build.sh
+sh integrations/stackgen_serve/test-image.sh
+uvx --from ruff==0.15.12 ruff check --config pyproject.toml integrations/stackgen_serve
+uvx --from ruff==0.15.12 ruff format --check --config pyproject.toml integrations/stackgen_serve
+make validate
+```
+
+Image tests mount only tests and a source-hash manifest, not application source.
+They compare the installed converter with the checkout, exercise CSV and HTML
+in default/workers/reload modes, and require `pip check` to pass. Containers
+have no external network, a 3 GiB memory limit and two CPUs. Models are not loaded.
+Process identity and OOM guards prevent health responses from hiding worker loss.
+The CSV fixture includes blank lines and explicit empty fields.
+
+For a larger approved UTF-8 CSV, set `CSV` to its absolute path:
+
+```sh
+docker run --rm --network=none --memory=4g --cpus=2 \
+  --entrypoint /opt/app-root/bin/python \
+  -v "$PWD/integrations/stackgen_serve/tests:/tests:ro" \
+  -v "$CSV:/fixture/input.csv:ro" stackgen-docling:fork-review \
+  /tests/http_smoke.py default --csv /fixture/input.csv --requests 1 --timeout 600
+```
+
+Repeat with `workers` and `reload`. The test compares every field and record
+label. One worker-mode request does not prove coverage of both workers or
+concurrent upload capacity. Memory counters include the test client and startup;
+they are not a server-only profile. Do not commit customer files or large fixtures.
+These checks do not establish PDF model behavior or upload-to-search acceptance.
+
+## CI and publication
+
+`stackgen-serve.yml` tests native AMD64 and ARM64 images on pull requests and
+relevant `main` pushes. After both test jobs pass, a `main` push publishes
+`ghcr.io/stackgenhq/docling-serve:main` and `:sha-<commit>` as a multi-platform image.
+Manual dispatch is test-only unless `publish` is selected on `main`. PRs cannot
+publish. Publication rebuilds wheels from the tested revision with the same
+pinned base and build constraints; it does not promote the local test image IDs.
+
+GHCR login uses `GITHUB_TOKEN` with package-write permission only in the publish
+job. Organisation package policy and native runner access must permit this.
+The workflow summary reports the manifest digest; deployments should use it.
+Publishing does not deploy or update downstream deployment image references.
+
+Remote CI and publication have not been executed. Local checks cover native
+ARM64 and emulated AMD64, not native AMD64 capacity. Vulnerability remediation
+is deferred; these tests are not a security approval. Production registry access,
+concurrent large uploads, and upload-through-search acceptance remain required.

integrations/stackgen_serve/build-constraints.txt

@@ -0,0 +1 @@
+hatchling==1.30.1

integrations/stackgen_serve/build.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -eu
+# Run from the repository root. Keep wheel inputs explicit and remove stale wheels.
+rm -f integrations/stackgen_serve/dist/*.whl
+uv build --wheel --build-constraints integrations/stackgen_serve/build-constraints.txt --out-dir integrations/stackgen_serve/dist
+uv build --wheel --build-constraints integrations/stackgen_serve/build-constraints.txt integrations/stackgen_serve --out-dir integrations/stackgen_serve/dist
+docker build --network=none -f integrations/stackgen_serve/Dockerfile -t stackgen-docling:fork-review .

integrations/stackgen_serve/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+requires = ["hatchling==1.30.1"]
+build-backend = "hatchling.build"
+
+[project]
+name = "stackgen-docling"
+version = "0.1.0"
+description = "Opt-in record-table export and StackGen Docling Serve launcher"
+requires-python = ">=3.10,<4"
+dependencies = ["docling-core==2.99.0"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/stackgen_docling"]

integrations/stackgen_serve/src/stackgen_docling/__init__.py

@@ -0,0 +1 @@
+"""Pinned Docling Serve integration; only table-to-Markdown rendering changes."""

integrations/stackgen_serve/src/stackgen_docling/__main__.py

@@ -0,0 +1,29 @@
+"""Preserve Serve's CLI while directing every server process to our factory."""
+
+import importlib
+from typing import Any
+from unittest.mock import patch
+
+import uvicorn
+
+from stackgen_docling.install import install
+
+
+class _ServerRunner:
+    """Redirect the pinned CLI's factory without patching the shared Uvicorn module."""
+
+    def run(self, *, app: str, **kwargs: Any) -> None:
+        """Keep upstream options, but install inside spawned workers/reload children."""
+        uvicorn.run(app="stackgen_docling.app:create_app", **kwargs)
+
+
+def main() -> None:
+    """Install for CLI work; delegate serving-process installation to our factory."""
+    install()
+    upstream = importlib.import_module("docling_serve.__main__")
+    with patch.object(upstream, "uvicorn", _ServerRunner()):
+        upstream.app()
+
+
+if __name__ == "__main__":
+    main()

integrations/stackgen_serve/src/stackgen_docling/app.py

@@ -0,0 +1,26 @@
+"""Process-local factory used by both direct and spawned Uvicorn servers."""
+
+import logging
+import os
+from typing import TYPE_CHECKING
+
+from stackgen_docling.install import install
+
+if TYPE_CHECKING:
+    from fastapi import FastAPI
+
+
+def create_app() -> "FastAPI":
+    """Install before upstream imports in every worker, including reload children.
+
+    Parent-process module replacements are not inherited by multiprocessing spawn.
+    Without this factory, workers silently restore upstream's dense-grid exporter.
+    """
+    install()
+    from docling_serve.app import create_app as upstream_factory
+
+    application = upstream_factory()
+    logging.getLogger(__name__).info(
+        "Record table factory installed pid=%s", os.getpid()
+    )
+    return application

integrations/stackgen_serve/src/stackgen_docling/install.py

@@ -0,0 +1,40 @@
+"""Install the replacement before Serve imports its conversion/export machinery."""
+
+import logging
+from importlib.metadata import version
+
+from docling_core.transforms.serializer import markdown
+from docling_core.transforms.serializer.base import BaseTableSerializer
+from pydantic import Field
+
+from stackgen_docling.record_tables import RecordTableSerializer
+
+
+class RecordMarkdownDocSerializer(markdown.MarkdownDocSerializer):
+    """Keep upstream document serialization; replace its default table strategy."""
+
+    table_serializer: BaseTableSerializer = Field(default_factory=RecordTableSerializer)
+
+
+def install() -> None:
+    """Select record tables for Markdown exports, including callback character counts.
+
+    Document.export_to_markdown imports this class at call time, so the original
+    signature/defaults/filtering stay intact. Install once before starting Serve.
+    An explicit table_serializer passed by a caller still overrides the default.
+    """
+    expected = {
+        "docling-core": "2.99.0",
+        "docling-serve": "1.36.0",
+        "docling-jobkit": "3.8.1",
+    }
+    for distribution, pinned in expected.items():
+        installed = version(distribution)
+        if installed != pinned:
+            raise RuntimeError(
+                f"Unsupported {distribution} {installed}; expected {pinned}"
+            )
+    if markdown.MarkdownDocSerializer is RecordMarkdownDocSerializer:
+        return
+    markdown.MarkdownDocSerializer = RecordMarkdownDocSerializer
+    logging.getLogger(__name__).info("Record-oriented table exporter installed")