From 329e4a73ae6a2b5b3452bd09581fba3e7a6b864e Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Mon, 5 Oct 2026 10:38:33 +0200 Subject: [PATCH 1/2] fix(pdf): preserve Identity font character widths --- CHANGELOG.md | 3 +++ src/odr/internal/pdf/AGENTS.md | 3 ++- src/odr/internal/pdf/pdf_cmap.cpp | 10 +++++++--- src/odr/internal/pdf/pdf_cmap.hpp | 10 ++++------ src/odr/internal/pdf/pdf_document.cpp | 14 ++++++-------- src/odr/internal/pdf/pdf_document_element.hpp | 17 ++++++++++------- test/src/internal/pdf/pdf_cmap.cpp | 4 ++-- test/src/internal/pdf/pdf_font.cpp | 19 +++++++++++++++++++ 8 files changed, 53 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c220b4f2f..e4ef61e4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ The release run heads these entries with the version and opens a fresh ## Unreleased +- PDF Identity-H/V fonts retain two-byte character codes even when their + text-extraction map declares a conflicting codespace, preserving glyphs and spacing. + - PDF annotation snapshots no longer expose mutable pending state. Pointer options are copied, and unknown option keys leave the settings unchanged. diff --git a/src/odr/internal/pdf/AGENTS.md b/src/odr/internal/pdf/AGENTS.md index 9fe609f3e..66ac46fac 100644 --- a/src/odr/internal/pdf/AGENTS.md +++ b/src/odr/internal/pdf/AGENTS.md @@ -40,7 +40,8 @@ output. `to_unicode` imposes that width instead of reading the `/ToUnicode` codespace, because producers write `<0000> ` there regardless. The entries are keyed either way, so a one-byte code is also looked up - zero-padded. Composite codes split by the codespace, as `Font::codes` does. + zero-padded. Identity-H/V fixes two-byte codes regardless of `/ToUnicode`; + other composite codes split by the codespace, as `Font::codes` does. - **`std::any` object model.** `Object` holds its value in `std::any` with typed `is_*` and `as_*` accessors, like `oldms/`'s `Entry`. The cost: no exhaustive matching, and accidental copies are easy. `resolve_object_copy` diff --git a/src/odr/internal/pdf/pdf_cmap.cpp b/src/odr/internal/pdf/pdf_cmap.cpp index e455dc586..75aa1c096 100644 --- a/src/odr/internal/pdf/pdf_cmap.cpp +++ b/src/odr/internal/pdf/pdf_cmap.cpp @@ -107,13 +107,17 @@ std::size_t CMap::code_width(const std::string_view bytes) const { } std::string CMap::translate_string(const std::string &codes, - const bool single_byte_codes) const { + const std::size_t fixed_width) const { std::u16string result; std::size_t pos = 0; while (pos < codes.size()) { const std::size_t width = - single_byte_codes ? 1 : code_width(std::string_view(codes).substr(pos)); + fixed_width != 0 ? fixed_width + : code_width(std::string_view(codes).substr(pos)); + if (width > codes.size() - pos) { + break; + } const std::string code = codes.substr(pos, width); pos += width; @@ -124,7 +128,7 @@ std::string CMap::translate_string(const std::string &codes, // Only for an imposed width — a declared mixed codespace keeps `<20>` and // `<0020>` distinct. - if (single_byte_codes) { + if (fixed_width == 1) { if (const std::optional unicode = unicode_for_code(std::string(1, '\0') + code)) { result += *unicode; diff --git a/src/odr/internal/pdf/pdf_cmap.hpp b/src/odr/internal/pdf/pdf_cmap.hpp index d886e16bc..fd77cc540 100644 --- a/src/odr/internal/pdf/pdf_cmap.hpp +++ b/src/odr/internal/pdf/pdf_cmap.hpp @@ -66,12 +66,10 @@ class CMap { /// so the glyph paths split codes exactly as `translate_string` does. [[nodiscard]] std::size_t code_width(std::string_view bytes) const; - /// `single_byte_codes` overrides the codespace ranges. An imposed - /// single-byte code is also looked up zero-padded to two bytes, producers - /// keying the entries either way. - [[nodiscard]] std::string - translate_string(const std::string &codes, - bool single_byte_codes = false) const; + /// A nonzero `fixed_width` overrides the codespace. One-byte codes are also + /// looked up zero-padded to two bytes; a trailing partial code is ignored. + [[nodiscard]] std::string translate_string(const std::string &codes, + std::size_t fixed_width = 0) const; /// True when at least one `cidchar`/`cidrange` mapping was parsed (an /// embedded CID `/Encoding` CMap). When false the composite code -> CID is diff --git a/src/odr/internal/pdf/pdf_document.cpp b/src/odr/internal/pdf/pdf_document.cpp index 01a3a0aed..b5712a8df 100644 --- a/src/odr/internal/pdf/pdf_document.cpp +++ b/src/odr/internal/pdf/pdf_document.cpp @@ -169,11 +169,10 @@ std::uint16_t Font::glyph_for_code(const std::uint32_t code) const { } std::string Font::to_unicode(const std::string &codes) const { - // A simple font's codes are one byte each (ISO 32000-1 9.10.3); its - // `ToUnicode` codespace is not to be trusted, producers writing the - // two-byte `<0000> ` boilerplate there regardless. + // Encoding fixes the width for simple and Identity fonts (9.10.3, 9.7.5.2). if (!cmap.empty()) { - return cmap.translate_string(codes, !composite); + const std::size_t width = !composite ? 1 : has_identity_encoding() ? 2 : 0; + return cmap.translate_string(codes, width); } if (composite) { // A composite (Type0) font with no `ToUnicode` CMap. A predefined @@ -194,9 +193,8 @@ std::string Font::to_unicode(const std::string &codes) const { // descendant CIDFont's `/CIDSystemInfo`. Only take this when the codes // really are CIDs (identity or an embedded stream); a named CMap we lack // tables for must not be misread as identity CIDs. - const bool identity_cids = cid_encoding_name.empty() || - cid_encoding_name == "Identity-H" || - cid_encoding_name == "Identity-V"; + const bool identity_cids = + cid_encoding_name.empty() || has_identity_encoding(); if (identity_cids && !cid_registry.empty() && !cid_ordering.empty()) { std::string result; for (const std::uint32_t cid : this->codes(codes)) { @@ -221,7 +219,7 @@ std::string Font::to_unicode(const std::string &codes) const { !unicode.empty()) { return unicode; } - return cmap.translate_string(codes, true); + return cmap.translate_string(codes, 1); } } // namespace odr::internal::pdf diff --git a/src/odr/internal/pdf/pdf_document_element.hpp b/src/odr/internal/pdf/pdf_document_element.hpp index bc64b4ccc..6cafa3c16 100644 --- a/src/odr/internal/pdf/pdf_document_element.hpp +++ b/src/odr/internal/pdf/pdf_document_element.hpp @@ -216,10 +216,8 @@ struct Pattern final : Element { /// big-endian codes on iteration; must not outlive the bytes. A trailing /// partial code is dropped, as the text-showing operators do. /// -/// A `codespace` CMap picks each code's width from its first byte (ISO 32000-1 -/// 9.7.6.2), so a mixed 1-/2-byte encoding (a 1-byte space among 2-byte CIDs, -/// which PDFClown et al. emit) stays aligned; without one the width is a fixed -/// `Font::code_byte_width()`. +/// A codespace matches every byte (ISO 32000-1 9.7.6.2); without one, +/// `Font::code_byte_width()` supplies the fixed width. class CodeRange { public: class Iterator { @@ -362,15 +360,20 @@ struct Font final : Element { return composite ? 2 : 1; } + [[nodiscard]] bool has_identity_encoding() const { + return cid_encoding_name == "Identity-H" || + cid_encoding_name == "Identity-V"; + } + /// View `codes` as character codes, each yielded as the CID it selects. The /// result borrows `codes`. /// /// The split is by the embedded CID `/Encoding` CMap's codespace if there is - /// one, else the `/ToUnicode` codespace (which keeps the split identical to - /// `to_unicode`'s), else the fixed `code_byte_width`. + /// one, else the `/ToUnicode` codespace. Identity-H/V always use two bytes + /// (ISO 32000-1 9.7.5.2). [[nodiscard]] CodeRange codes(std::string_view codes) const { const std::size_t width = code_byte_width(); - if (!composite) { + if (!composite || has_identity_encoding()) { return {codes, width, nullptr, nullptr}; } const CMap *cid_map = cid_encoding.has_cid_map() ? &cid_encoding : nullptr; diff --git a/test/src/internal/pdf/pdf_cmap.cpp b/test/src/internal/pdf/pdf_cmap.cpp index c5651b1c1..b8119f68b 100644 --- a/test/src/internal/pdf/pdf_cmap.cpp +++ b/test/src/internal/pdf/pdf_cmap.cpp @@ -241,7 +241,7 @@ TEST(PdfCMap, imposed_code_width_overrides_codespace) { "endbfchar\n"); EXPECT_EQ(cmap.translate_string("\x41\x42"), "\xe4\x85\x82"); // U+4142 - EXPECT_EQ(cmap.translate_string("\x41\x42", true), "AB"); + EXPECT_EQ(cmap.translate_string("\x41\x42", 1), "AB"); } TEST(PdfCMap, imposed_code_width_falls_back_to_a_padded_entry) { @@ -253,7 +253,7 @@ TEST(PdfCMap, imposed_code_width_falls_back_to_a_padded_entry) { "<0042> <0042>\n" "endbfchar\n"); - EXPECT_EQ(cmap.translate_string("\x41\x42", true), "AB"); + EXPECT_EQ(cmap.translate_string("\x41\x42", 1), "AB"); } TEST(PdfCMap, imposed_code_width_keeps_a_mixed_codespace_distinct) { diff --git a/test/src/internal/pdf/pdf_font.cpp b/test/src/internal/pdf/pdf_font.cpp index cc2198c79..c46356f4a 100644 --- a/test/src/internal/pdf/pdf_font.cpp +++ b/test/src/internal/pdf/pdf_font.cpp @@ -113,6 +113,25 @@ TEST(PdfFont, composite_to_unicode_splits_by_cmap_codespace) { EXPECT_EQ(font.to_unicode("\x41\x42"), "Z"); } +TEST(PdfFont, identity_encoding_ignores_tounicode_codespace) { + Font font; + font.composite = true; + font.cmap.add_codespace_range(codes2({0x00e0}), codes2({0x028c})); + font.cmap.map_single(codes2({0x00fc}), u"\u0107"); + font.cmap.map_single(codes2({0x0105}), u"\u0105"); + const std::string bytes = codes2({0x00fc, 0x0105}) + '\0'; + + for (const auto *encoding : {"Identity-H", "Identity-V"}) { + font.cid_encoding_name = encoding; + EXPECT_EQ(font.to_unicode(bytes), "ćą"); + std::vector codes; + for (const std::uint32_t code : font.codes(bytes)) { + codes.push_back(code); + } + EXPECT_EQ(codes, (std::vector{0x00fc, 0x0105})); + } +} + TEST(PdfFont, simple_font_glyph_for_code_via_cmap) { // A simple (1-byte) TrueType font: the code's Unicode reaches the glyph // through the embedded (3,1) cmap. From e3a580038f583e827df1d663e630c819ab265580 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Tue, 6 Oct 2026 21:40:43 +0200 Subject: [PATCH 2/2] docs: wrap the Identity font changelog entry at 80 columns Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01MxyTMutqSUJRGfxA8CyzMc --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4ef61e4c..518855d3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ The release run heads these entries with the version and opens a fresh ## Unreleased - PDF Identity-H/V fonts retain two-byte character codes even when their - text-extraction map declares a conflicting codespace, preserving glyphs and spacing. + text-extraction map declares a conflicting codespace, preserving glyphs and + spacing. - PDF annotation snapshots no longer expose mutable pending state. Pointer options are copied, and unknown option keys leave the settings unchanged.