From f3722c7762f31ca99dc7c6afae0e7a713faf8a33 Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Tue, 27 Oct 2026 16:59:56 +0100 Subject: [PATCH 1/3] fix: resolve /CIDToGIDMap streams for CIDFontType2 glyph lookup CIDFontType2 subsets that carry no cmap table address their glyphs only through a /CIDToGIDMap stream (ISO 32000-1, 9.7.4.2). Blend2D refuses such a face (no character mapping), and the FreeType fallback honoured only an /Identity map, so every glyph fell through to a system font, or to a stroked outline box where none is installed. Decode the stream (2 bytes per CID, big-endian GID; CIDs past its end map to GID 0) into the embedded font blob, and use it wherever the CID identity was used: FreeType glyph resolution and Blend2D's char-code glyph recovery. Identity and absent maps behave as before. Signed-off-by: Christoph Auer --- src/parse/page_items/embedded_font_blob.h | 31 ++++- src/parse/pdf_resources/page_font.h | 49 +++++++- src/render/blend2d_renderer.h | 24 ++-- src/render/freetype_font_cache.h | 15 ++- tests/test_unit_cid_to_gid_map.py | 144 ++++++++++++++++++++++ 5 files changed, 238 insertions(+), 25 deletions(-) create mode 100644 tests/test_unit_cid_to_gid_map.py diff --git a/src/parse/page_items/embedded_font_blob.h b/src/parse/page_items/embedded_font_blob.h index be6d68644..a4896dd25 100644 --- a/src/parse/page_items/embedded_font_blob.h +++ b/src/parse/page_items/embedded_font_blob.h @@ -32,7 +32,8 @@ namespace pdflib bool is_cid_font, bool cid_to_gid_identity, bool uses_builtin_encoding, - std::shared_ptr > bytes); + std::shared_ptr > bytes, + std::vector cid_to_gid_map = {}); // Content-derived identity (FNV-1a over the decoded bytes + size). // PDF object ids are NOT used: they are only unique within one document, @@ -53,6 +54,22 @@ namespace pdflib // /CIDToGIDMap is /Identity or absent bool get_cid_to_gid_identity() const { return cid_to_gid_identity; } + // True when a CID font's character codes address glyphs directly: an + // identity /CIDToGIDMap, or an explicit /CIDToGIDMap stream. + bool has_cid_to_gid() const + { + return is_cid_font and (cid_to_gid_identity or not cid_to_gid_map.empty()); + } + + // Glyph index of a CID under has_cid_to_gid(). A CID beyond the end of + // the /CIDToGIDMap stream maps to GID 0 (ISO 32000-1, 9.7.4.2). + uint32_t cid_to_gid(int64_t cid) const + { + if(cid_to_gid_identity) { return static_cast(cid); } + return (cid >= 0 and static_cast(cid) < cid_to_gid_map.size()) + ? cid_to_gid_map[static_cast(cid)] : 0; + } + // True for a simple font that is symbolic (/FontDescriptor /Flags bit 3) // and has no /Encoding override: its PDF character codes address the // font program's builtin cmap directly (PDF 32000-1, 9.6.6.4). Shaping @@ -83,6 +100,9 @@ namespace pdflib bool uses_builtin_encoding; std::shared_ptr > bytes; + + // decoded /CIDToGIDMap stream (CID -> GID); empty when identity or absent + std::vector cid_to_gid_map; }; inline embedded_font_blob::embedded_font_blob(): @@ -94,7 +114,8 @@ namespace pdflib is_cid_font(false), cid_to_gid_identity(false), uses_builtin_encoding(false), - bytes(nullptr) + bytes(nullptr), + cid_to_gid_map() {} inline embedded_font_blob::embedded_font_blob(std::string cache_key_, @@ -105,7 +126,8 @@ namespace pdflib bool is_cid_font_, bool cid_to_gid_identity_, bool uses_builtin_encoding_, - std::shared_ptr > bytes_): + std::shared_ptr > bytes_, + std::vector cid_to_gid_map_): cache_key(std::move(cache_key_)), font_name(std::move(font_name_)), base_font(std::move(base_font_)), @@ -114,7 +136,8 @@ namespace pdflib is_cid_font(is_cid_font_), cid_to_gid_identity(cid_to_gid_identity_), uses_builtin_encoding(uses_builtin_encoding_), - bytes(std::move(bytes_)) + bytes(std::move(bytes_)), + cid_to_gid_map(std::move(cid_to_gid_map_)) {} inline bool embedded_font_blob::has_bytes() const diff --git a/src/parse/pdf_resources/page_font.h b/src/parse/pdf_resources/page_font.h index f2e44a69e..290a535a3 100644 --- a/src/parse/pdf_resources/page_font.h +++ b/src/parse/pdf_resources/page_font.h @@ -186,6 +186,7 @@ namespace pdflib embedded_font_format resolve_embedded_font_format() const; bool resolve_cid_to_gid_identity() const; + std::vector resolve_cid_to_gid_map() const; void build_embedded_font_blob(); // Resolution of this font onto one of the built-in base fonts. See the @@ -1761,8 +1762,8 @@ namespace pdflib return false; } - // PDF spec: /CIDToGIDMap defaults to /Identity when absent. Any stream - // value means an explicit map that we do not resolve here. + // PDF spec: /CIDToGIDMap defaults to /Identity when absent. A stream is + // an explicit map, read by resolve_cid_to_gid_map(). QPDFObjectHandle value = qpdf_object::get_path(qpdf_desc_font, {"/CIDToGIDMap"}); if(value.isNull()) { @@ -1773,6 +1774,46 @@ namespace pdflib return qpdf_object::get_name_or_string(value, name) and name == "/Identity"; } + std::vector pdf_resource::resolve_cid_to_gid_map() const + { + std::vector map; + + // Only a CIDFontType2 (TrueType) program is addressed by GID through + // /CIDToGIDMap (ISO 32000-1, 9.7.4.2); CFF CIDFonts map through their + // own charset. + std::string desc_subtype; + QPDFObjectHandle value = qpdf_object::get_path(qpdf_desc_font, {"/CIDToGIDMap"}); + if(subtype != TYPE_0 or not value.isStream() or + not qpdf_object::get_name_or_string(qpdf_desc_font, {"/Subtype"}, desc_subtype) or + desc_subtype != "/CIDFontType2") + { + return map; + } + + try + { + auto buffer = value.getStreamData(qpdf_dl_all); + if(buffer) + { + // 2 bytes per CID, big-endian GID; CID n at bytes 2n..2n+1 + const unsigned char* data = buffer->getBuffer(); + map.resize(buffer->getSize() / 2); + for(size_t cid = 0; cid < map.size(); cid++) + { + map[cid] = static_cast((data[2*cid] << 8) | data[2*cid + 1]); + } + } + } + catch(const std::exception& e) + { + LOG_S(WARNING) << "could not decode /CIDToGIDMap for font-key=" + << font_key << ": " << e.what(); + map.clear(); + } + + return map; + } + void pdf_resource::build_embedded_font_blob() { if(not font_program.found or @@ -1813,7 +1854,8 @@ namespace pdflib subtype == TYPE_0, resolve_cid_to_gid_identity(), uses_builtin_encoding, - std::move(bytes)); + std::move(bytes), + resolve_cid_to_gid_map()); LOG_S(INFO) << __FUNCTION__ << ": font-key=" << font_key @@ -1824,6 +1866,7 @@ namespace pdflib << " cache-key=" << font_blob->get_cache_key() << " cid=" << font_blob->get_is_cid_font() << " cid-to-gid-identity=" << font_blob->get_cid_to_gid_identity() + << " has-cid-to-gid=" << font_blob->has_cid_to_gid() << " uses-builtin-encoding=" << font_blob->get_uses_builtin_encoding(); } diff --git a/src/render/blend2d_renderer.h b/src/render/blend2d_renderer.h index 8e3422121..1adb4adb2 100644 --- a/src/render/blend2d_renderer.h +++ b/src/render/blend2d_renderer.h @@ -438,13 +438,13 @@ namespace pdflib double h_scale); // Glyph-identity mapping by PDF character code against an embedded - // (Blend2D-loaded) face: CID fonts with an identity CIDToGIDMap use the - // character code as glyph index directly; simple fonts are tried through - // the 0xF000 private-use convention ((3,0) symbol cmaps) and then with - // the raw character code ((1,0) / format-0 builtin cmaps, as written by - // Cairo/LibreOffice subsetters). Used both as the primary path for - // symbolic fonts and as recovery when Unicode shaping failed or - // produced only .notdef. On success gb holds a positioned glyph run + // (Blend2D-loaded) face: CID fonts map the character code to the glyph + // index through /CIDToGIDMap (identity or stream); simple fonts are + // tried through the 0xF000 private-use convention ((3,0) symbol cmaps) + // and then with the raw character code ((1,0) / format-0 builtin cmaps, + // as written by Cairo/LibreOffice subsetters). Used both as the primary + // path for symbolic fonts and as recovery when Unicode shaping failed + // or produced only .notdef. On success gb holds a positioned glyph run // ready for fill_glyph_run. static bool recover_embedded_glyphs(const BLFont& font, text_instruction& instr, @@ -455,16 +455,16 @@ namespace pdflib const auto& blob = instr.get_embedded_font(); - if (blob->get_is_cid_font() and blob->get_cid_to_gid_identity()) + if (blob->has_cid_to_gid()) { - const uint32_t glyph_id = static_cast(char_code); + const uint32_t glyph_id = blob->cid_to_gid(char_code); gb.set_glyphs(&glyph_id, 1); // shape() rejects glyph content, so position directly. if (font.position_glyphs(gb) == BL_SUCCESS and not gb.is_empty() and gb.placement_data() != nullptr) { - LOG_S(INFO) << "render_text: recovered glyph via CID identity" + LOG_S(INFO) << "render_text: recovered glyph via CIDToGIDMap" << " cid=" << char_code << " font_name=`" << instr.get_font_name() << "`"; return true; @@ -2889,7 +2889,7 @@ namespace pdflib // glyph (e.g. U+0020 hitting subset code 0x20). Resolve by // character code first for such faces. // - // CID fonts with an identity CIDToGIDMap must also resolve by + // CID fonts with a /CIDToGIDMap must also resolve by // character code first: the content-stream CID is the exact glyph // the producer chose (contextual Arabic forms, lam-alef ligatures, // …). Round-tripping through the Unicode cmap instead picks the @@ -2900,7 +2900,7 @@ namespace pdflib using_embedded_font and instr.has_embedded_font() and (embedded_blob->get_is_cid_font() - ? embedded_blob->get_cid_to_gid_identity() + ? embedded_blob->has_cid_to_gid() : embedded_blob->get_uses_builtin_encoding()); if (char_code_first) diff --git a/src/render/freetype_font_cache.h b/src/render/freetype_font_cache.h index 0e1695a7f..a322639dc 100644 --- a/src/render/freetype_font_cache.h +++ b/src/render/freetype_font_cache.h @@ -41,7 +41,7 @@ namespace pdflib // those files, so they are drawn from here instead. // // Glyph identity is resolved in decreasing order of authority: - // 1. CID with identity CIDToGIDMap — the CID is the glyph index; + // 1. CID through /CIDToGIDMap (identity or stream) — the glyph index; // 2. the glyph name assigned by /Encoding /Differences (FT_Get_Name_Index); // 3. the decoded PDF character code through the font's builtin encoding // (Adobe custom / standard / Latin-1, and MS symbol with the 0xF000 @@ -603,13 +603,16 @@ namespace pdflib { glyph_indices.clear(); - // CID-keyed font with identity CIDToGIDMap: the character code is the - // CID is the glyph index. - if(blob->get_is_cid_font() and blob->get_cid_to_gid_identity() and char_code >= 0) + // CID-keyed font with a /CIDToGIDMap (identity or stream): the + // character code is the CID, which the map turns into the glyph index. + if(blob->has_cid_to_gid() and char_code >= 0) { - if(char_code < entry.face->num_glyphs) + const uint32_t gid = blob->cid_to_gid(char_code); + // An explicit map sends unused CIDs to GID 0 (.notdef): no glyph. + const bool mapped = blob->get_cid_to_gid_identity() or gid != 0; + if(mapped and gid < static_cast(entry.face->num_glyphs)) { - glyph_indices.push_back(static_cast(char_code)); + glyph_indices.push_back(static_cast(gid)); return true; } return false; diff --git a/tests/test_unit_cid_to_gid_map.py b/tests/test_unit_cid_to_gid_map.py new file mode 100644 index 000000000..a1a526b82 --- /dev/null +++ b/tests/test_unit_cid_to_gid_map.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +"""A CIDFontType2 draws the glyph its /CIDToGIDMap stream names (9.7.4.2). + +Subset TrueType programs embedded under a /Type0 font often carry no `cmap` +(and no `post`) table: the PDF addresses glyphs by CID, and a /CIDToGIDMap +stream turns each CID into a glyph index -- two big-endian bytes per CID. Blend2D +refuses such a face outright (no character mapping), and the FreeType path only +honoured an /Identity map, so every glyph fell through to a system font, or, +with none installed, to a stroked outline box. Annual reports set in subset +Times New Roman came out as rows of blue rectangles. + +The fixture maps CID 1 to GID 2, a large solid rectangle, while GID 1 is a +small square: an identity reading, a system substitute for the ToUnicode 'A', +and an outline box all draw something else. +""" + +from __future__ import annotations + +from io import BytesIO + +import pytest + +from tests.pdf_builder import render_page, simple_page_pdf, stream_object +from tests.rendering_regression import flatten_on_white + +EM = 1000 +FONT_SIZE = 100.0 +TEXT_X = 20.0 +TEXT_Y = 60.0 +MEASURE_SCALE = 4.0 + +# GID 2: the glyph CID 1 must reach. 0.8 x 0.7 em of solid ink. +BIG = (100, 0, 900, 700) +# GID 1: what an identity CID->GID reading would draw instead. +SMALL = (100, 0, 200, 100) + + +def _rect_glyph(box): + from fontTools.pens.ttGlyphPen import TTGlyphPen + + x0, y0, x1, y1 = box + pen = TTGlyphPen(None) + pen.moveTo((x0, y0)) + pen.lineTo((x0, y1)) + pen.lineTo((x1, y1)) + pen.lineTo((x1, y0)) + pen.closePath() + return pen.glyph() + + +def _cmapless_font() -> bytes: + """A TrueType with only the tables a CID subset keeps: no cmap, no post.""" + pytest.importorskip("fontTools", reason="building the test font needs fontTools") + + from fontTools.fontBuilder import FontBuilder + from fontTools.pens.ttGlyphPen import TTGlyphPen + + builder = FontBuilder(EM, isTTF=True) + builder.setupGlyphOrder([".notdef", "small", "big"]) + builder.setupGlyf( + { + ".notdef": TTGlyphPen(None).glyph(), + "small": _rect_glyph(SMALL), + "big": _rect_glyph(BIG), + } + ) + builder.setupHorizontalMetrics( + {".notdef": (EM, 0), "small": (EM, SMALL[0]), "big": (EM, BIG[0])} + ) + builder.setupHorizontalHeader(ascent=800, descent=-200) + + out = BytesIO() + builder.save(out) + + from fontTools.ttLib import TTFont + + font = TTFont(BytesIO(out.getvalue())) + assert "cmap" not in font and "post" not in font + + return out.getvalue() + + +def _page() -> bytes: + program = _cmapless_font() + + content = f"BT /F1 {FONT_SIZE} Tf {TEXT_X} {TEXT_Y} Td <0001> Tj ET\n" + + # CID 0 -> GID 0, CID 1 -> GID 2, CID 2 -> GID 1 + cid_to_gid = bytes([0, 0, 0, 2, 0, 1]) + + to_unicode = ( + b"/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n" + b"/CMapName /Adobe-Identity-UCS def /CMapType 2 def\n" + b"1 begincodespacerange <0000> endcodespacerange\n" + b"1 beginbfchar <0001> <0041> endbfchar\n" + b"endcmap CMapName currentdict /CMap defineresource pop end end\n" + ) + + objects: list[str | bytes] = [ + # 5: the Type0 font + "<< /Type /Font /Subtype /Type0 /BaseFont /AAAAAA+DoclingCidMapped " + "/Encoding /Identity-H /DescendantFonts [6 0 R] /ToUnicode 10 0 R >>", + # 6: its CIDFontType2 descendant + "<< /Type /Font /Subtype /CIDFontType2 /BaseFont /AAAAAA+DoclingCidMapped " + "/CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> " + "/FontDescriptor 7 0 R /DW 1000 /CIDToGIDMap 9 0 R >>", + # 7 + "<< /Type /FontDescriptor /FontName /AAAAAA+DoclingCidMapped /Flags 4 " + "/FontBBox [0 -200 1000 800] /ItalicAngle 0 /Ascent 800 /Descent -200 " + "/CapHeight 700 /StemV 80 /FontFile2 8 0 R >>", + # 8 + stream_object(f"/Length1 {len(program)}", program), + # 9 + stream_object("", cid_to_gid), + # 10 + stream_object("", to_unicode), + ] + + return simple_page_pdf( + content, + resources="/Font << /F1 5 0 R >>", + extra_objects=objects, + ) + + +def test_cid_to_gid_map_stream_selects_the_glyph(): + """CID 1 draws GID 2 from the embedded program, not a box or a substitute.""" + result = render_page(_page(), scale=MEASURE_SCALE) + + image = flatten_on_white(result.get_image(scale=MEASURE_SCALE)).convert("L") + mask = image.point(lambda value: 255 if value < 128 else 0) + + box = mask.getbbox() + assert box is not None, "the page came out blank" + + width = (box[2] - box[0]) / MEASURE_SCALE + height = (box[3] - box[1]) / MEASURE_SCALE + assert width == pytest.approx((BIG[2] - BIG[0]) / EM * FONT_SIZE, abs=4.0) + assert height == pytest.approx((BIG[3] - BIG[1]) / EM * FONT_SIZE, abs=4.0) + + # A stroked outline box spans a similar area but is almost all white. + inked = mask.crop(box).histogram()[255] + area = (box[2] - box[0]) * (box[3] - box[1]) + assert inked / area > 0.9, "the glyph cell was stroked, not filled" From 94db2f46d6ee42cf91c16498475ed9b25dbf0837 Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Tue, 27 Oct 2026 18:30:14 +0100 Subject: [PATCH 2/3] fix: resolve the character code to its CID before /CIDToGIDMap The renderer indexed the /CIDToGIDMap stream with the content-stream character code, which is the CID only under Identity-H/V. Fonts whose /Encoding is an embedded or predefined CMap drew the wrong glyphs: "50-99" came out as "xso" and two greater-or-equal signs. Compose the map with the encoding's code -> CID when the font blob is built. Identity-H/V use the map as is; an embedded CMap reuses the code -> CID table that the codespace scan already parses; a predefined CMap takes it from its cmap resource. When no complete table is known (an embedded CMap with usecmap, or any other encoding), the stream is not applied and Unicode shaping draws the text as before. Rename the blob helpers to has_code_to_gid()/code_to_gid(), and drop the constructor default that no caller used. Correct the test's account of why Blend2D rejects these subsets: they lack a name table (BL_ERROR_FONT_MISSING_IMPORTANT_TABLE), not a character mapping. Signed-off-by: Christoph Auer --- src/parse/page_items/embedded_font_blob.h | 33 ++--- src/parse/pdf_resources/page_font.h | 62 ++++++++-- src/parse/pdf_resources/page_font/font_cid.h | 2 + src/render/blend2d_renderer.h | 10 +- src/render/freetype_font_cache.h | 10 +- tests/test_unit_cid_to_gid_map.py | 122 ++++++++++++++++--- 6 files changed, 189 insertions(+), 50 deletions(-) diff --git a/src/parse/page_items/embedded_font_blob.h b/src/parse/page_items/embedded_font_blob.h index a4896dd25..3e33a42d9 100644 --- a/src/parse/page_items/embedded_font_blob.h +++ b/src/parse/page_items/embedded_font_blob.h @@ -33,7 +33,7 @@ namespace pdflib bool cid_to_gid_identity, bool uses_builtin_encoding, std::shared_ptr > bytes, - std::vector cid_to_gid_map = {}); + std::vector code_to_gid_map); // Content-derived identity (FNV-1a over the decoded bytes + size). // PDF object ids are NOT used: they are only unique within one document, @@ -55,19 +55,21 @@ namespace pdflib bool get_cid_to_gid_identity() const { return cid_to_gid_identity; } // True when a CID font's character codes address glyphs directly: an - // identity /CIDToGIDMap, or an explicit /CIDToGIDMap stream. - bool has_cid_to_gid() const + // identity /CIDToGIDMap, or a /CIDToGIDMap stream whose CIDs the + // /Encoding CMap resolves. + bool has_code_to_gid() const { - return is_cid_font and (cid_to_gid_identity or not cid_to_gid_map.empty()); + return is_cid_font and (cid_to_gid_identity or not code_to_gid_map.empty()); } - // Glyph index of a CID under has_cid_to_gid(). A CID beyond the end of - // the /CIDToGIDMap stream maps to GID 0 (ISO 32000-1, 9.7.4.2). - uint32_t cid_to_gid(int64_t cid) const + // Glyph index of a PDF character code under has_code_to_gid(). An + // identity map takes the code itself, as before (exact for + // Identity-H/V); an unmapped code gives GID 0. + uint32_t code_to_gid(int64_t code) const { - if(cid_to_gid_identity) { return static_cast(cid); } - return (cid >= 0 and static_cast(cid) < cid_to_gid_map.size()) - ? cid_to_gid_map[static_cast(cid)] : 0; + if(cid_to_gid_identity) { return static_cast(code); } + return (code >= 0 and static_cast(code) < code_to_gid_map.size()) + ? code_to_gid_map[static_cast(code)] : 0; } // True for a simple font that is symbolic (/FontDescriptor /Flags bit 3) @@ -101,8 +103,9 @@ namespace pdflib std::shared_ptr > bytes; - // decoded /CIDToGIDMap stream (CID -> GID); empty when identity or absent - std::vector cid_to_gid_map; + // /CIDToGIDMap stream composed with the /Encoding CMap (code -> GID); + // empty when the map is identity or absent, or its CIDs are unresolved + std::vector code_to_gid_map; }; inline embedded_font_blob::embedded_font_blob(): @@ -115,7 +118,7 @@ namespace pdflib cid_to_gid_identity(false), uses_builtin_encoding(false), bytes(nullptr), - cid_to_gid_map() + code_to_gid_map() {} inline embedded_font_blob::embedded_font_blob(std::string cache_key_, @@ -127,7 +130,7 @@ namespace pdflib bool cid_to_gid_identity_, bool uses_builtin_encoding_, std::shared_ptr > bytes_, - std::vector cid_to_gid_map_): + std::vector code_to_gid_map_): cache_key(std::move(cache_key_)), font_name(std::move(font_name_)), base_font(std::move(base_font_)), @@ -137,7 +140,7 @@ namespace pdflib cid_to_gid_identity(cid_to_gid_identity_), uses_builtin_encoding(uses_builtin_encoding_), bytes(std::move(bytes_)), - cid_to_gid_map(std::move(cid_to_gid_map_)) + code_to_gid_map(std::move(code_to_gid_map_)) {} inline bool embedded_font_blob::has_bytes() const diff --git a/src/parse/pdf_resources/page_font.h b/src/parse/pdf_resources/page_font.h index 290a535a3..7ebc34a23 100644 --- a/src/parse/pdf_resources/page_font.h +++ b/src/parse/pdf_resources/page_font.h @@ -186,7 +186,7 @@ namespace pdflib embedded_font_format resolve_embedded_font_format() const; bool resolve_cid_to_gid_identity() const; - std::vector resolve_cid_to_gid_map() const; + std::vector resolve_code_to_gid_map() const; void build_embedded_font_blob(); // Resolution of this font onto one of the built-in base fonts. See the @@ -258,6 +258,10 @@ namespace pdflib bool has_explicit_encoding; // true if encoding was found in PDF, false if defaulted bool is_symbolic = false; // /FontDescriptor /Flags bit 3 (PDF 32000-1 table 123) + // code -> CID of an embedded /Encoding CMap; left empty when the CMap + // inherits through usecmap, whose entries are not read + std::unordered_map embedded_code_to_cid; + font_subtype_name subtype; std::string font_key; @@ -1197,8 +1201,10 @@ namespace pdflib // An embedded CMap may inherit from a predefined one. Only its name is // available here, so the inherited codespace is reported rather than // silently assumed. - auto report_usecmap = [](const std::string& parent) + bool inherits = false; + auto report_usecmap = [&inherits](const std::string& parent) { + inherits = true; LOG_S(WARNING) << "embedded cmap inherits from " << parent << ": the inherited entries are not applied"; }; @@ -1221,6 +1227,11 @@ namespace pdflib LOG_S(INFO) << "embedded cmap: " << cmap_codespaces.size() << " codespace-range(s), " << tables.code_to_cid.size() << " code-to-cid entries"; + + if(not inherits) + { + embedded_code_to_cid = std::move(tables.code_to_cid); + } } void pdf_resource::init_subtype() @@ -1763,7 +1774,7 @@ namespace pdflib } // PDF spec: /CIDToGIDMap defaults to /Identity when absent. A stream is - // an explicit map, read by resolve_cid_to_gid_map(). + // an explicit map, read by resolve_code_to_gid_map(). QPDFObjectHandle value = qpdf_object::get_path(qpdf_desc_font, {"/CIDToGIDMap"}); if(value.isNull()) { @@ -1774,7 +1785,7 @@ namespace pdflib return qpdf_object::get_name_or_string(value, name) and name == "/Identity"; } - std::vector pdf_resource::resolve_cid_to_gid_map() const + std::vector pdf_resource::resolve_code_to_gid_map() const { std::vector map; @@ -1811,7 +1822,44 @@ namespace pdflib map.clear(); } - return map; + // The renderer holds character codes, and the map is indexed by CID: + // compose it with the /Encoding CMap's code -> CID (9.7.6.2). Under + // Identity-H/V the code is the CID already. + if(map.empty() or encoding == IDENTITY_H or encoding == IDENTITY_V) + { + return map; + } + + const std::unordered_map* code_to_cid = nullptr; + if(encoding == CMAP_STREAM) + { + code_to_cid = &embedded_code_to_cid; + } + else if(encoding == CMAP_RESOURCES and cids.decode_cmap_resource(encoding_name)) + { + code_to_cid = &cids.get(encoding_name).get_code_to_cid(); + } + + // Without a code -> CID table the map cannot be applied: an empty result + // keeps the Unicode-shaping path. + if(code_to_cid == nullptr) + { + return {}; + } + + // shortcut: codes above 0xFFFF (four-byte codespaces) are left unmapped + // and render through Unicode shaping; widen the table if one ever matters. + std::vector code_to_gid; + for(const auto& [code, cid] : *code_to_cid) + { + if(code <= 0xFFFF and cid < map.size() and map[cid] != 0) + { + if(code >= code_to_gid.size()) { code_to_gid.resize(code + 1, 0); } + code_to_gid[code] = map[cid]; + } + } + + return code_to_gid; } void pdf_resource::build_embedded_font_blob() @@ -1855,7 +1903,7 @@ namespace pdflib resolve_cid_to_gid_identity(), uses_builtin_encoding, std::move(bytes), - resolve_cid_to_gid_map()); + resolve_code_to_gid_map()); LOG_S(INFO) << __FUNCTION__ << ": font-key=" << font_key @@ -1866,7 +1914,7 @@ namespace pdflib << " cache-key=" << font_blob->get_cache_key() << " cid=" << font_blob->get_is_cid_font() << " cid-to-gid-identity=" << font_blob->get_cid_to_gid_identity() - << " has-cid-to-gid=" << font_blob->has_cid_to_gid() + << " has-code-to-gid=" << font_blob->has_code_to_gid() << " uses-builtin-encoding=" << font_blob->get_uses_builtin_encoding(); } diff --git a/src/parse/pdf_resources/page_font/font_cid.h b/src/parse/pdf_resources/page_font/font_cid.h index e5e1303fb..98a14b712 100644 --- a/src/parse/pdf_resources/page_font/font_cid.h +++ b/src/parse/pdf_resources/page_font/font_cid.h @@ -313,6 +313,8 @@ namespace pdflib // resource declared none. const std::vector& get_codespaces() const; + const std::unordered_map& get_code_to_cid() const { return cmap2cid; } + void decode_cmap_resource(std::string filename, std::string cid2code, std::vector columns); diff --git a/src/render/blend2d_renderer.h b/src/render/blend2d_renderer.h index 1adb4adb2..eef9ddfb7 100644 --- a/src/render/blend2d_renderer.h +++ b/src/render/blend2d_renderer.h @@ -439,7 +439,7 @@ namespace pdflib // Glyph-identity mapping by PDF character code against an embedded // (Blend2D-loaded) face: CID fonts map the character code to the glyph - // index through /CIDToGIDMap (identity or stream); simple fonts are + // index through /Encoding and /CIDToGIDMap; simple fonts are // tried through the 0xF000 private-use convention ((3,0) symbol cmaps) // and then with the raw character code ((1,0) / format-0 builtin cmaps, // as written by Cairo/LibreOffice subsetters). Used both as the primary @@ -455,9 +455,9 @@ namespace pdflib const auto& blob = instr.get_embedded_font(); - if (blob->has_cid_to_gid()) + if (blob->has_code_to_gid()) { - const uint32_t glyph_id = blob->cid_to_gid(char_code); + const uint32_t glyph_id = blob->code_to_gid(char_code); gb.set_glyphs(&glyph_id, 1); // shape() rejects glyph content, so position directly. if (font.position_glyphs(gb) == BL_SUCCESS and @@ -465,7 +465,7 @@ namespace pdflib gb.placement_data() != nullptr) { LOG_S(INFO) << "render_text: recovered glyph via CIDToGIDMap" - << " cid=" << char_code + << " char_code=" << char_code << " font_name=`" << instr.get_font_name() << "`"; return true; } @@ -2900,7 +2900,7 @@ namespace pdflib using_embedded_font and instr.has_embedded_font() and (embedded_blob->get_is_cid_font() - ? embedded_blob->has_cid_to_gid() + ? embedded_blob->has_code_to_gid() : embedded_blob->get_uses_builtin_encoding()); if (char_code_first) diff --git a/src/render/freetype_font_cache.h b/src/render/freetype_font_cache.h index a322639dc..d9b8f8eb6 100644 --- a/src/render/freetype_font_cache.h +++ b/src/render/freetype_font_cache.h @@ -41,7 +41,7 @@ namespace pdflib // those files, so they are drawn from here instead. // // Glyph identity is resolved in decreasing order of authority: - // 1. CID through /CIDToGIDMap (identity or stream) — the glyph index; + // 1. CID font: the code through /Encoding and /CIDToGIDMap — the glyph index; // 2. the glyph name assigned by /Encoding /Differences (FT_Get_Name_Index); // 3. the decoded PDF character code through the font's builtin encoding // (Adobe custom / standard / Latin-1, and MS symbol with the 0xF000 @@ -604,11 +604,11 @@ namespace pdflib glyph_indices.clear(); // CID-keyed font with a /CIDToGIDMap (identity or stream): the - // character code is the CID, which the map turns into the glyph index. - if(blob->has_cid_to_gid() and char_code >= 0) + // character code selects a CID, which the map turns into the glyph index. + if(blob->has_code_to_gid() and char_code >= 0) { - const uint32_t gid = blob->cid_to_gid(char_code); - // An explicit map sends unused CIDs to GID 0 (.notdef): no glyph. + const uint32_t gid = blob->code_to_gid(char_code); + // An explicit map sends unused codes to GID 0 (.notdef): no glyph. const bool mapped = blob->get_cid_to_gid_identity() or gid != 0; if(mapped and gid < static_cast(entry.face->num_glyphs)) { diff --git a/tests/test_unit_cid_to_gid_map.py b/tests/test_unit_cid_to_gid_map.py index a1a526b82..6ba0111a6 100644 --- a/tests/test_unit_cid_to_gid_map.py +++ b/tests/test_unit_cid_to_gid_map.py @@ -3,15 +3,19 @@ Subset TrueType programs embedded under a /Type0 font often carry no `cmap` (and no `post`) table: the PDF addresses glyphs by CID, and a /CIDToGIDMap -stream turns each CID into a glyph index -- two big-endian bytes per CID. Blend2D -refuses such a face outright (no character mapping), and the FreeType path only -honoured an /Identity map, so every glyph fell through to a system font, or, -with none installed, to a stroked outline box. Annual reports set in subset -Times New Roman came out as rows of blue rectangles. +stream turns each CID into a glyph index -- two big-endian bytes per CID. Such +subsets also lack a `name` table, so Blend2D refuses the face at load +(BL_ERROR_FONT_MISSING_IMPORTANT_TABLE), and the FreeType path only honoured an +/Identity map, so every glyph fell through to a system font, or, with none +installed, to a stroked outline box. Annual reports set in subset Times New +Roman came out as rows of blue rectangles. The fixture maps CID 1 to GID 2, a large solid rectangle, while GID 1 is a small square: an identity reading, a system substitute for the ToUnicode 'A', and an outline box all draw something else. + +The map is indexed by CID, not by character code: under any /Encoding other +than Identity-H/V the code reaches its CID through the encoding CMap first. """ from __future__ import annotations @@ -80,26 +84,71 @@ def _cmapless_font() -> bytes: return out.getvalue() -def _page() -> bytes: - program = _cmapless_font() +def _complete_font() -> bytes: + """The same glyphs with the cmap, name, OS/2 and post tables Blend2D needs. + + Its Unicode cmap sends 'A' to the small square, so Unicode shaping (what + renders this face when the map is not used) draws the expected glyph. + """ + from fontTools.fontBuilder import FontBuilder + from fontTools.ttLib import TTFont + + font = TTFont(BytesIO(_cmapless_font())) + builder = FontBuilder(font=font) + # without a post table the glyphs read back as glyph00001, ... + builder.setupCharacterMap({0x41: font.getGlyphOrder()[1]}) + builder.setupNameTable( + {"familyName": "Docling", "styleName": "Regular", "psName": "Docling-Regular"} + ) + builder.setupOS2( + sTypoAscender=800, sTypoDescender=-200, usWinAscent=800, usWinDescent=200 + ) + builder.setupPost() + + out = BytesIO() + font.save(out) + return out.getvalue() + + +# An embedded /Encoding CMap that sends code 1 to CID 2. +CODE_1_TO_CID_2 = ( + b"/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n" + b"/CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> def\n" + b"/CMapName /Custom def /CMapType 1 def /WMode 0 def\n" + b"1 begincodespacerange <0000> endcodespacerange\n" + b"1 begincidchar <0001> 2 endcidchar\n" + b"endcmap CMapName currentdict /CMap defineresource pop end end\n" +) + - content = f"BT /F1 {FONT_SIZE} Tf {TEXT_X} {TEXT_Y} Td <0001> Tj ET\n" +def _page( + program: bytes | None = None, + *, + encoding: str = "/Identity-H", + code: int = 1, + cid_to_gid: bytes = bytes([0, 0, 0, 2, 0, 1]), # CID 1 -> GID 2, CID 2 -> GID 1 + encoding_cmap: bytes | None = None, +) -> bytes: + if program is None: + program = _cmapless_font() - # CID 0 -> GID 0, CID 1 -> GID 2, CID 2 -> GID 1 - cid_to_gid = bytes([0, 0, 0, 2, 0, 1]) + content = f"BT /F1 {FONT_SIZE} Tf {TEXT_X} {TEXT_Y} Td <{code:04X}> Tj ET\n" to_unicode = ( b"/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n" b"/CMapName /Adobe-Identity-UCS def /CMapType 2 def\n" b"1 begincodespacerange <0000> endcodespacerange\n" - b"1 beginbfchar <0001> <0041> endbfchar\n" - b"endcmap CMapName currentdict /CMap defineresource pop end end\n" + + f"1 beginbfchar <{code:04X}> <0041> endbfchar\n".encode() + + b"endcmap CMapName currentdict /CMap defineresource pop end end\n" ) + if encoding_cmap is not None: + encoding = "11 0 R" + objects: list[str | bytes] = [ # 5: the Type0 font "<< /Type /Font /Subtype /Type0 /BaseFont /AAAAAA+DoclingCidMapped " - "/Encoding /Identity-H /DescendantFonts [6 0 R] /ToUnicode 10 0 R >>", + f"/Encoding {encoding} /DescendantFonts [6 0 R] /ToUnicode 10 0 R >>", # 6: its CIDFontType2 descendant "<< /Type /Font /Subtype /CIDFontType2 /BaseFont /AAAAAA+DoclingCidMapped " "/CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> " @@ -115,6 +164,9 @@ def _page() -> bytes: # 10 stream_object("", to_unicode), ] + if encoding_cmap is not None: + # 11 + objects.append(stream_object("/Type /CMap", encoding_cmap)) return simple_page_pdf( content, @@ -123,9 +175,9 @@ def _page() -> bytes: ) -def test_cid_to_gid_map_stream_selects_the_glyph(): - """CID 1 draws GID 2 from the embedded program, not a box or a substitute.""" - result = render_page(_page(), scale=MEASURE_SCALE) +def _assert_draws(pdf: bytes, glyph) -> None: + """The page carries exactly `glyph`, filled.""" + result = render_page(pdf, scale=MEASURE_SCALE) image = flatten_on_white(result.get_image(scale=MEASURE_SCALE)).convert("L") mask = image.point(lambda value: 255 if value < 128 else 0) @@ -135,10 +187,44 @@ def test_cid_to_gid_map_stream_selects_the_glyph(): width = (box[2] - box[0]) / MEASURE_SCALE height = (box[3] - box[1]) / MEASURE_SCALE - assert width == pytest.approx((BIG[2] - BIG[0]) / EM * FONT_SIZE, abs=4.0) - assert height == pytest.approx((BIG[3] - BIG[1]) / EM * FONT_SIZE, abs=4.0) + assert width == pytest.approx((glyph[2] - glyph[0]) / EM * FONT_SIZE, abs=4.0) + assert height == pytest.approx((glyph[3] - glyph[1]) / EM * FONT_SIZE, abs=4.0) # A stroked outline box spans a similar area but is almost all white. inked = mask.crop(box).histogram()[255] area = (box[2] - box[0]) * (box[3] - box[1]) assert inked / area > 0.9, "the glyph cell was stroked, not filled" + + +def test_cid_to_gid_map_stream_selects_the_glyph(): + """CID 1 draws GID 2 from the embedded program, not a box or a substitute.""" + _assert_draws(_page(), BIG) + + +# CID 34 -> GID 1 (small), and CID 65 -> GID 2 (big), where code 0x41 read +# as a CID would land. +_UNIJIS_MAP = bytearray(2 * 66) +_UNIJIS_MAP[2 * 34 + 1] = 1 +_UNIJIS_MAP[2 * 65 + 1] = 2 + + +@pytest.mark.parametrize("complete", [False, True], ids=["freetype", "blend2d"]) +@pytest.mark.parametrize( + "encoding", + [ + # code 1 -> CID 2 -> GID 1; CID 1 would be GID 2 + dict(code=1, encoding_cmap=CODE_1_TO_CID_2), + # /UniJIS-UCS2-H sends U+0041 to CID 34 (Adobe-Japan1) + dict(code=0x41, encoding="/UniJIS-UCS2-H", cid_to_gid=bytes(_UNIJIS_MAP)), + ], + ids=["embedded-cmap", "predefined-cmap"], +) +def test_encoding_cmap_resolves_the_cid_before_the_map(encoding, complete): + """Code -> CID through /Encoding, then CID -> GID through /CIDToGIDMap. + + Code, CID and GID all differ, so reading the code as the CID draws the + big rectangle instead of the small square. The cmap-less face renders + through FreeType, the complete one through Blend2D. + """ + program = _complete_font() if complete else _cmapless_font() + _assert_draws(_page(program, **encoding), SMALL) From 95ed56a353e29eaad9715171d5e4743509c38d1f Mon Sep 17 00:00:00 2001 From: Christoph Auer Date: Tue, 27 Oct 2026 19:10:53 +0100 Subject: [PATCH 3/3] test: pin regression dataset with updated render groundtruth The five pages whose /CIDToGIDMap stream fonts now render with their embedded glyphs get new reference images in docling-project/regression-dataset-for-docling-parse@21374211. Signed-off-by: Christoph Auer --- tests/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/constants.py b/tests/constants.py index 982c0d144..4adf5b05a 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,7 +1,7 @@ from tests.regression_page_selection import REGRESSION_PAGE_SELECTION HF_DATASET_REPO_ID = "docling-project/regression-dataset-for-docling-parse" -HF_DATASET_REVISION = "6d3458e2a6a729defd700a7b91e57b6cb2ce0290" +HF_DATASET_REVISION = "21374211ec8e84cb58eea3c62f869c2adfcc9a54" REGRESSION_DIR = "tests/data/regression"