diff --git a/src/parse/page_items/embedded_font_blob.h b/src/parse/page_items/embedded_font_blob.h index be6d68644..3e33a42d9 100644 --- a/src/parse/page_items/embedded_font_blob.h +++ b/src/parse/page_items/embedded_font_blob.h @@ -32,7 +32,8 @@ namespace pdflib bool is_cid_font, bool cid_to_gid_identity, bool uses_builtin_encoding, - std::shared_ptr > bytes); + std::shared_ptr > bytes, + std::vector code_to_gid_map); // Content-derived identity (FNV-1a over the decoded bytes + size). // PDF object ids are NOT used: they are only unique within one document, @@ -53,6 +54,24 @@ namespace pdflib // /CIDToGIDMap is /Identity or absent bool get_cid_to_gid_identity() const { return cid_to_gid_identity; } + // True when a CID font's character codes address glyphs directly: an + // identity /CIDToGIDMap, or a /CIDToGIDMap stream whose CIDs the + // /Encoding CMap resolves. + bool has_code_to_gid() const + { + return is_cid_font and (cid_to_gid_identity or not code_to_gid_map.empty()); + } + + // Glyph index of a PDF character code under has_code_to_gid(). An + // identity map takes the code itself, as before (exact for + // Identity-H/V); an unmapped code gives GID 0. + uint32_t code_to_gid(int64_t code) const + { + if(cid_to_gid_identity) { return static_cast(code); } + return (code >= 0 and static_cast(code) < code_to_gid_map.size()) + ? code_to_gid_map[static_cast(code)] : 0; + } + // True for a simple font that is symbolic (/FontDescriptor /Flags bit 3) // and has no /Encoding override: its PDF character codes address the // font program's builtin cmap directly (PDF 32000-1, 9.6.6.4). Shaping @@ -83,6 +102,10 @@ namespace pdflib bool uses_builtin_encoding; std::shared_ptr > bytes; + + // /CIDToGIDMap stream composed with the /Encoding CMap (code -> GID); + // empty when the map is identity or absent, or its CIDs are unresolved + std::vector code_to_gid_map; }; inline embedded_font_blob::embedded_font_blob(): @@ -94,7 +117,8 @@ namespace pdflib is_cid_font(false), cid_to_gid_identity(false), uses_builtin_encoding(false), - bytes(nullptr) + bytes(nullptr), + code_to_gid_map() {} inline embedded_font_blob::embedded_font_blob(std::string cache_key_, @@ -105,7 +129,8 @@ namespace pdflib bool is_cid_font_, bool cid_to_gid_identity_, bool uses_builtin_encoding_, - std::shared_ptr > bytes_): + std::shared_ptr > bytes_, + std::vector code_to_gid_map_): cache_key(std::move(cache_key_)), font_name(std::move(font_name_)), base_font(std::move(base_font_)), @@ -114,7 +139,8 @@ namespace pdflib is_cid_font(is_cid_font_), cid_to_gid_identity(cid_to_gid_identity_), uses_builtin_encoding(uses_builtin_encoding_), - bytes(std::move(bytes_)) + bytes(std::move(bytes_)), + code_to_gid_map(std::move(code_to_gid_map_)) {} inline bool embedded_font_blob::has_bytes() const diff --git a/src/parse/pdf_resources/page_font.h b/src/parse/pdf_resources/page_font.h index f2e44a69e..7ebc34a23 100644 --- a/src/parse/pdf_resources/page_font.h +++ b/src/parse/pdf_resources/page_font.h @@ -186,6 +186,7 @@ namespace pdflib embedded_font_format resolve_embedded_font_format() const; bool resolve_cid_to_gid_identity() const; + std::vector resolve_code_to_gid_map() const; void build_embedded_font_blob(); // Resolution of this font onto one of the built-in base fonts. See the @@ -257,6 +258,10 @@ namespace pdflib bool has_explicit_encoding; // true if encoding was found in PDF, false if defaulted bool is_symbolic = false; // /FontDescriptor /Flags bit 3 (PDF 32000-1 table 123) + // code -> CID of an embedded /Encoding CMap; left empty when the CMap + // inherits through usecmap, whose entries are not read + std::unordered_map embedded_code_to_cid; + font_subtype_name subtype; std::string font_key; @@ -1196,8 +1201,10 @@ namespace pdflib // An embedded CMap may inherit from a predefined one. Only its name is // available here, so the inherited codespace is reported rather than // silently assumed. - auto report_usecmap = [](const std::string& parent) + bool inherits = false; + auto report_usecmap = [&inherits](const std::string& parent) { + inherits = true; LOG_S(WARNING) << "embedded cmap inherits from " << parent << ": the inherited entries are not applied"; }; @@ -1220,6 +1227,11 @@ namespace pdflib LOG_S(INFO) << "embedded cmap: " << cmap_codespaces.size() << " codespace-range(s), " << tables.code_to_cid.size() << " code-to-cid entries"; + + if(not inherits) + { + embedded_code_to_cid = std::move(tables.code_to_cid); + } } void pdf_resource::init_subtype() @@ -1761,8 +1773,8 @@ namespace pdflib return false; } - // PDF spec: /CIDToGIDMap defaults to /Identity when absent. Any stream - // value means an explicit map that we do not resolve here. + // PDF spec: /CIDToGIDMap defaults to /Identity when absent. A stream is + // an explicit map, read by resolve_code_to_gid_map(). QPDFObjectHandle value = qpdf_object::get_path(qpdf_desc_font, {"/CIDToGIDMap"}); if(value.isNull()) { @@ -1773,6 +1785,83 @@ namespace pdflib return qpdf_object::get_name_or_string(value, name) and name == "/Identity"; } + std::vector pdf_resource::resolve_code_to_gid_map() const + { + std::vector map; + + // Only a CIDFontType2 (TrueType) program is addressed by GID through + // /CIDToGIDMap (ISO 32000-1, 9.7.4.2); CFF CIDFonts map through their + // own charset. + std::string desc_subtype; + QPDFObjectHandle value = qpdf_object::get_path(qpdf_desc_font, {"/CIDToGIDMap"}); + if(subtype != TYPE_0 or not value.isStream() or + not qpdf_object::get_name_or_string(qpdf_desc_font, {"/Subtype"}, desc_subtype) or + desc_subtype != "/CIDFontType2") + { + return map; + } + + try + { + auto buffer = value.getStreamData(qpdf_dl_all); + if(buffer) + { + // 2 bytes per CID, big-endian GID; CID n at bytes 2n..2n+1 + const unsigned char* data = buffer->getBuffer(); + map.resize(buffer->getSize() / 2); + for(size_t cid = 0; cid < map.size(); cid++) + { + map[cid] = static_cast((data[2*cid] << 8) | data[2*cid + 1]); + } + } + } + catch(const std::exception& e) + { + LOG_S(WARNING) << "could not decode /CIDToGIDMap for font-key=" + << font_key << ": " << e.what(); + map.clear(); + } + + // The renderer holds character codes, and the map is indexed by CID: + // compose it with the /Encoding CMap's code -> CID (9.7.6.2). Under + // Identity-H/V the code is the CID already. + if(map.empty() or encoding == IDENTITY_H or encoding == IDENTITY_V) + { + return map; + } + + const std::unordered_map* code_to_cid = nullptr; + if(encoding == CMAP_STREAM) + { + code_to_cid = &embedded_code_to_cid; + } + else if(encoding == CMAP_RESOURCES and cids.decode_cmap_resource(encoding_name)) + { + code_to_cid = &cids.get(encoding_name).get_code_to_cid(); + } + + // Without a code -> CID table the map cannot be applied: an empty result + // keeps the Unicode-shaping path. + if(code_to_cid == nullptr) + { + return {}; + } + + // shortcut: codes above 0xFFFF (four-byte codespaces) are left unmapped + // and render through Unicode shaping; widen the table if one ever matters. + std::vector code_to_gid; + for(const auto& [code, cid] : *code_to_cid) + { + if(code <= 0xFFFF and cid < map.size() and map[cid] != 0) + { + if(code >= code_to_gid.size()) { code_to_gid.resize(code + 1, 0); } + code_to_gid[code] = map[cid]; + } + } + + return code_to_gid; + } + void pdf_resource::build_embedded_font_blob() { if(not font_program.found or @@ -1813,7 +1902,8 @@ namespace pdflib subtype == TYPE_0, resolve_cid_to_gid_identity(), uses_builtin_encoding, - std::move(bytes)); + std::move(bytes), + resolve_code_to_gid_map()); LOG_S(INFO) << __FUNCTION__ << ": font-key=" << font_key @@ -1824,6 +1914,7 @@ namespace pdflib << " cache-key=" << font_blob->get_cache_key() << " cid=" << font_blob->get_is_cid_font() << " cid-to-gid-identity=" << font_blob->get_cid_to_gid_identity() + << " has-code-to-gid=" << font_blob->has_code_to_gid() << " uses-builtin-encoding=" << font_blob->get_uses_builtin_encoding(); } diff --git a/src/parse/pdf_resources/page_font/font_cid.h b/src/parse/pdf_resources/page_font/font_cid.h index e5e1303fb..98a14b712 100644 --- a/src/parse/pdf_resources/page_font/font_cid.h +++ b/src/parse/pdf_resources/page_font/font_cid.h @@ -313,6 +313,8 @@ namespace pdflib // resource declared none. const std::vector& get_codespaces() const; + const std::unordered_map& get_code_to_cid() const { return cmap2cid; } + void decode_cmap_resource(std::string filename, std::string cid2code, std::vector columns); diff --git a/src/render/blend2d_renderer.h b/src/render/blend2d_renderer.h index 8e3422121..eef9ddfb7 100644 --- a/src/render/blend2d_renderer.h +++ b/src/render/blend2d_renderer.h @@ -438,13 +438,13 @@ namespace pdflib double h_scale); // Glyph-identity mapping by PDF character code against an embedded - // (Blend2D-loaded) face: CID fonts with an identity CIDToGIDMap use the - // character code as glyph index directly; simple fonts are tried through - // the 0xF000 private-use convention ((3,0) symbol cmaps) and then with - // the raw character code ((1,0) / format-0 builtin cmaps, as written by - // Cairo/LibreOffice subsetters). Used both as the primary path for - // symbolic fonts and as recovery when Unicode shaping failed or - // produced only .notdef. On success gb holds a positioned glyph run + // (Blend2D-loaded) face: CID fonts map the character code to the glyph + // index through /Encoding and /CIDToGIDMap; simple fonts are + // tried through the 0xF000 private-use convention ((3,0) symbol cmaps) + // and then with the raw character code ((1,0) / format-0 builtin cmaps, + // as written by Cairo/LibreOffice subsetters). Used both as the primary + // path for symbolic fonts and as recovery when Unicode shaping failed + // or produced only .notdef. On success gb holds a positioned glyph run // ready for fill_glyph_run. static bool recover_embedded_glyphs(const BLFont& font, text_instruction& instr, @@ -455,17 +455,17 @@ namespace pdflib const auto& blob = instr.get_embedded_font(); - if (blob->get_is_cid_font() and blob->get_cid_to_gid_identity()) + if (blob->has_code_to_gid()) { - const uint32_t glyph_id = static_cast(char_code); + const uint32_t glyph_id = blob->code_to_gid(char_code); gb.set_glyphs(&glyph_id, 1); // shape() rejects glyph content, so position directly. if (font.position_glyphs(gb) == BL_SUCCESS and not gb.is_empty() and gb.placement_data() != nullptr) { - LOG_S(INFO) << "render_text: recovered glyph via CID identity" - << " cid=" << char_code + LOG_S(INFO) << "render_text: recovered glyph via CIDToGIDMap" + << " char_code=" << char_code << " font_name=`" << instr.get_font_name() << "`"; return true; } @@ -2889,7 +2889,7 @@ namespace pdflib // glyph (e.g. U+0020 hitting subset code 0x20). Resolve by // character code first for such faces. // - // CID fonts with an identity CIDToGIDMap must also resolve by + // CID fonts with a /CIDToGIDMap must also resolve by // character code first: the content-stream CID is the exact glyph // the producer chose (contextual Arabic forms, lam-alef ligatures, // …). Round-tripping through the Unicode cmap instead picks the @@ -2900,7 +2900,7 @@ namespace pdflib using_embedded_font and instr.has_embedded_font() and (embedded_blob->get_is_cid_font() - ? embedded_blob->get_cid_to_gid_identity() + ? embedded_blob->has_code_to_gid() : embedded_blob->get_uses_builtin_encoding()); if (char_code_first) diff --git a/src/render/freetype_font_cache.h b/src/render/freetype_font_cache.h index 0e1695a7f..d9b8f8eb6 100644 --- a/src/render/freetype_font_cache.h +++ b/src/render/freetype_font_cache.h @@ -41,7 +41,7 @@ namespace pdflib // those files, so they are drawn from here instead. // // Glyph identity is resolved in decreasing order of authority: - // 1. CID with identity CIDToGIDMap — the CID is the glyph index; + // 1. CID font: the code through /Encoding and /CIDToGIDMap — the glyph index; // 2. the glyph name assigned by /Encoding /Differences (FT_Get_Name_Index); // 3. the decoded PDF character code through the font's builtin encoding // (Adobe custom / standard / Latin-1, and MS symbol with the 0xF000 @@ -603,13 +603,16 @@ namespace pdflib { glyph_indices.clear(); - // CID-keyed font with identity CIDToGIDMap: the character code is the - // CID is the glyph index. - if(blob->get_is_cid_font() and blob->get_cid_to_gid_identity() and char_code >= 0) + // CID-keyed font with a /CIDToGIDMap (identity or stream): the + // character code selects a CID, which the map turns into the glyph index. + if(blob->has_code_to_gid() and char_code >= 0) { - if(char_code < entry.face->num_glyphs) + const uint32_t gid = blob->code_to_gid(char_code); + // An explicit map sends unused codes to GID 0 (.notdef): no glyph. + const bool mapped = blob->get_cid_to_gid_identity() or gid != 0; + if(mapped and gid < static_cast(entry.face->num_glyphs)) { - glyph_indices.push_back(static_cast(char_code)); + glyph_indices.push_back(static_cast(gid)); return true; } return false; diff --git a/tests/constants.py b/tests/constants.py index 982c0d144..4adf5b05a 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,7 +1,7 @@ from tests.regression_page_selection import REGRESSION_PAGE_SELECTION HF_DATASET_REPO_ID = "docling-project/regression-dataset-for-docling-parse" -HF_DATASET_REVISION = "6d3458e2a6a729defd700a7b91e57b6cb2ce0290" +HF_DATASET_REVISION = "21374211ec8e84cb58eea3c62f869c2adfcc9a54" REGRESSION_DIR = "tests/data/regression" diff --git a/tests/test_unit_cid_to_gid_map.py b/tests/test_unit_cid_to_gid_map.py new file mode 100644 index 000000000..6ba0111a6 --- /dev/null +++ b/tests/test_unit_cid_to_gid_map.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python +"""A CIDFontType2 draws the glyph its /CIDToGIDMap stream names (9.7.4.2). + +Subset TrueType programs embedded under a /Type0 font often carry no `cmap` +(and no `post`) table: the PDF addresses glyphs by CID, and a /CIDToGIDMap +stream turns each CID into a glyph index -- two big-endian bytes per CID. Such +subsets also lack a `name` table, so Blend2D refuses the face at load +(BL_ERROR_FONT_MISSING_IMPORTANT_TABLE), and the FreeType path only honoured an +/Identity map, so every glyph fell through to a system font, or, with none +installed, to a stroked outline box. Annual reports set in subset Times New +Roman came out as rows of blue rectangles. + +The fixture maps CID 1 to GID 2, a large solid rectangle, while GID 1 is a +small square: an identity reading, a system substitute for the ToUnicode 'A', +and an outline box all draw something else. + +The map is indexed by CID, not by character code: under any /Encoding other +than Identity-H/V the code reaches its CID through the encoding CMap first. +""" + +from __future__ import annotations + +from io import BytesIO + +import pytest + +from tests.pdf_builder import render_page, simple_page_pdf, stream_object +from tests.rendering_regression import flatten_on_white + +EM = 1000 +FONT_SIZE = 100.0 +TEXT_X = 20.0 +TEXT_Y = 60.0 +MEASURE_SCALE = 4.0 + +# GID 2: the glyph CID 1 must reach. 0.8 x 0.7 em of solid ink. +BIG = (100, 0, 900, 700) +# GID 1: what an identity CID->GID reading would draw instead. +SMALL = (100, 0, 200, 100) + + +def _rect_glyph(box): + from fontTools.pens.ttGlyphPen import TTGlyphPen + + x0, y0, x1, y1 = box + pen = TTGlyphPen(None) + pen.moveTo((x0, y0)) + pen.lineTo((x0, y1)) + pen.lineTo((x1, y1)) + pen.lineTo((x1, y0)) + pen.closePath() + return pen.glyph() + + +def _cmapless_font() -> bytes: + """A TrueType with only the tables a CID subset keeps: no cmap, no post.""" + pytest.importorskip("fontTools", reason="building the test font needs fontTools") + + from fontTools.fontBuilder import FontBuilder + from fontTools.pens.ttGlyphPen import TTGlyphPen + + builder = FontBuilder(EM, isTTF=True) + builder.setupGlyphOrder([".notdef", "small", "big"]) + builder.setupGlyf( + { + ".notdef": TTGlyphPen(None).glyph(), + "small": _rect_glyph(SMALL), + "big": _rect_glyph(BIG), + } + ) + builder.setupHorizontalMetrics( + {".notdef": (EM, 0), "small": (EM, SMALL[0]), "big": (EM, BIG[0])} + ) + builder.setupHorizontalHeader(ascent=800, descent=-200) + + out = BytesIO() + builder.save(out) + + from fontTools.ttLib import TTFont + + font = TTFont(BytesIO(out.getvalue())) + assert "cmap" not in font and "post" not in font + + return out.getvalue() + + +def _complete_font() -> bytes: + """The same glyphs with the cmap, name, OS/2 and post tables Blend2D needs. + + Its Unicode cmap sends 'A' to the small square, so Unicode shaping (what + renders this face when the map is not used) draws the expected glyph. + """ + from fontTools.fontBuilder import FontBuilder + from fontTools.ttLib import TTFont + + font = TTFont(BytesIO(_cmapless_font())) + builder = FontBuilder(font=font) + # without a post table the glyphs read back as glyph00001, ... + builder.setupCharacterMap({0x41: font.getGlyphOrder()[1]}) + builder.setupNameTable( + {"familyName": "Docling", "styleName": "Regular", "psName": "Docling-Regular"} + ) + builder.setupOS2( + sTypoAscender=800, sTypoDescender=-200, usWinAscent=800, usWinDescent=200 + ) + builder.setupPost() + + out = BytesIO() + font.save(out) + return out.getvalue() + + +# An embedded /Encoding CMap that sends code 1 to CID 2. +CODE_1_TO_CID_2 = ( + b"/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n" + b"/CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> def\n" + b"/CMapName /Custom def /CMapType 1 def /WMode 0 def\n" + b"1 begincodespacerange <0000> endcodespacerange\n" + b"1 begincidchar <0001> 2 endcidchar\n" + b"endcmap CMapName currentdict /CMap defineresource pop end end\n" +) + + +def _page( + program: bytes | None = None, + *, + encoding: str = "/Identity-H", + code: int = 1, + cid_to_gid: bytes = bytes([0, 0, 0, 2, 0, 1]), # CID 1 -> GID 2, CID 2 -> GID 1 + encoding_cmap: bytes | None = None, +) -> bytes: + if program is None: + program = _cmapless_font() + + content = f"BT /F1 {FONT_SIZE} Tf {TEXT_X} {TEXT_Y} Td <{code:04X}> Tj ET\n" + + to_unicode = ( + b"/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n" + b"/CMapName /Adobe-Identity-UCS def /CMapType 2 def\n" + b"1 begincodespacerange <0000> endcodespacerange\n" + + f"1 beginbfchar <{code:04X}> <0041> endbfchar\n".encode() + + b"endcmap CMapName currentdict /CMap defineresource pop end end\n" + ) + + if encoding_cmap is not None: + encoding = "11 0 R" + + objects: list[str | bytes] = [ + # 5: the Type0 font + "<< /Type /Font /Subtype /Type0 /BaseFont /AAAAAA+DoclingCidMapped " + f"/Encoding {encoding} /DescendantFonts [6 0 R] /ToUnicode 10 0 R >>", + # 6: its CIDFontType2 descendant + "<< /Type /Font /Subtype /CIDFontType2 /BaseFont /AAAAAA+DoclingCidMapped " + "/CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> " + "/FontDescriptor 7 0 R /DW 1000 /CIDToGIDMap 9 0 R >>", + # 7 + "<< /Type /FontDescriptor /FontName /AAAAAA+DoclingCidMapped /Flags 4 " + "/FontBBox [0 -200 1000 800] /ItalicAngle 0 /Ascent 800 /Descent -200 " + "/CapHeight 700 /StemV 80 /FontFile2 8 0 R >>", + # 8 + stream_object(f"/Length1 {len(program)}", program), + # 9 + stream_object("", cid_to_gid), + # 10 + stream_object("", to_unicode), + ] + if encoding_cmap is not None: + # 11 + objects.append(stream_object("/Type /CMap", encoding_cmap)) + + return simple_page_pdf( + content, + resources="/Font << /F1 5 0 R >>", + extra_objects=objects, + ) + + +def _assert_draws(pdf: bytes, glyph) -> None: + """The page carries exactly `glyph`, filled.""" + result = render_page(pdf, scale=MEASURE_SCALE) + + image = flatten_on_white(result.get_image(scale=MEASURE_SCALE)).convert("L") + mask = image.point(lambda value: 255 if value < 128 else 0) + + box = mask.getbbox() + assert box is not None, "the page came out blank" + + width = (box[2] - box[0]) / MEASURE_SCALE + height = (box[3] - box[1]) / MEASURE_SCALE + assert width == pytest.approx((glyph[2] - glyph[0]) / EM * FONT_SIZE, abs=4.0) + assert height == pytest.approx((glyph[3] - glyph[1]) / EM * FONT_SIZE, abs=4.0) + + # A stroked outline box spans a similar area but is almost all white. + inked = mask.crop(box).histogram()[255] + area = (box[2] - box[0]) * (box[3] - box[1]) + assert inked / area > 0.9, "the glyph cell was stroked, not filled" + + +def test_cid_to_gid_map_stream_selects_the_glyph(): + """CID 1 draws GID 2 from the embedded program, not a box or a substitute.""" + _assert_draws(_page(), BIG) + + +# CID 34 -> GID 1 (small), and CID 65 -> GID 2 (big), where code 0x41 read +# as a CID would land. +_UNIJIS_MAP = bytearray(2 * 66) +_UNIJIS_MAP[2 * 34 + 1] = 1 +_UNIJIS_MAP[2 * 65 + 1] = 2 + + +@pytest.mark.parametrize("complete", [False, True], ids=["freetype", "blend2d"]) +@pytest.mark.parametrize( + "encoding", + [ + # code 1 -> CID 2 -> GID 1; CID 1 would be GID 2 + dict(code=1, encoding_cmap=CODE_1_TO_CID_2), + # /UniJIS-UCS2-H sends U+0041 to CID 34 (Adobe-Japan1) + dict(code=0x41, encoding="/UniJIS-UCS2-H", cid_to_gid=bytes(_UNIJIS_MAP)), + ], + ids=["embedded-cmap", "predefined-cmap"], +) +def test_encoding_cmap_resolves_the_cid_before_the_map(encoding, complete): + """Code -> CID through /Encoding, then CID -> GID through /CIDToGIDMap. + + Code, CID and GID all differ, so reading the code as the CID draws the + big rectangle instead of the small square. The cmap-less face renders + through FreeType, the complete one through Blend2D. + """ + program = _complete_font() if complete else _cmapless_font() + _assert_draws(_page(program, **encoding), SMALL)