fix(pdf): preserve Identity font character widths

CHANGELOG.md

@@ -16,6 +16,10 @@ The release run heads these entries with the version and opens a fresh
 
 ## Unreleased
 
+- PDF Identity-H/V fonts retain two-byte character codes even when their
+  text-extraction map declares a conflicting codespace, preserving glyphs and
+  spacing.
+
 - PDF annotation snapshots no longer expose mutable pending state. Pointer
   options are copied, and unknown option keys leave the settings unchanged.
 

src/odr/internal/pdf/AGENTS.md

@@ -40,7 +40,8 @@ output.
   `to_unicode` imposes that width instead of reading the `/ToUnicode`
   codespace, because producers write `<0000> <FFFF>` there regardless. The
   entries are keyed either way, so a one-byte code is also looked up
-  zero-padded. Composite codes split by the codespace, as `Font::codes` does.
+  zero-padded. Identity-H/V fixes two-byte codes regardless of `/ToUnicode`;
+  other composite codes split by the codespace, as `Font::codes` does.
 - **`std::any` object model.** `Object` holds its value in `std::any` with
   typed `is_*` and `as_*` accessors, like `oldms/`'s `Entry`. The cost: no
   exhaustive matching, and accidental copies are easy. `resolve_object_copy`

src/odr/internal/pdf/pdf_cmap.cpp

@@ -107,13 +107,17 @@ std::size_t CMap::code_width(const std::string_view bytes) const {
 }
 
 std::string CMap::translate_string(const std::string &codes,
-                                   const bool single_byte_codes) const {
+                                   const std::size_t fixed_width) const {
   std::u16string result;
 
   std::size_t pos = 0;
   while (pos < codes.size()) {
     const std::size_t width =
-        single_byte_codes ? 1 : code_width(std::string_view(codes).substr(pos));
+        fixed_width != 0 ? fixed_width
+                         : code_width(std::string_view(codes).substr(pos));
+    if (width > codes.size() - pos) {
+      break;
+    }
     const std::string code = codes.substr(pos, width);
     pos += width;
 
@@ -124,7 +128,7 @@ std::string CMap::translate_string(const std::string &codes,
 
     // Only for an imposed width — a declared mixed codespace keeps `<20>` and
     // `<0020>` distinct.
-    if (single_byte_codes) {
+    if (fixed_width == 1) {
       if (const std::optional<std::u16string> unicode =
               unicode_for_code(std::string(1, '\0') + code)) {
         result += *unicode;

src/odr/internal/pdf/pdf_cmap.hpp

@@ -66,12 +66,10 @@ class CMap {
   /// so the glyph paths split codes exactly as `translate_string` does.
   [[nodiscard]] std::size_t code_width(std::string_view bytes) const;
 
-  /// `single_byte_codes` overrides the codespace ranges. An imposed
-  /// single-byte code is also looked up zero-padded to two bytes, producers
-  /// keying the entries either way.
-  [[nodiscard]] std::string
-  translate_string(const std::string &codes,
-                   bool single_byte_codes = false) const;
+  /// A nonzero `fixed_width` overrides the codespace. One-byte codes are also
+  /// looked up zero-padded to two bytes; a trailing partial code is ignored.
+  [[nodiscard]] std::string translate_string(const std::string &codes,
+                                             std::size_t fixed_width = 0) const;
 
   /// True when at least one `cidchar`/`cidrange` mapping was parsed (an
   /// embedded CID `/Encoding` CMap). When false the composite code -> CID is

src/odr/internal/pdf/pdf_document.cpp

@@ -169,11 +169,10 @@ std::uint16_t Font::glyph_for_code(const std::uint32_t code) const {
 }
 
 std::string Font::to_unicode(const std::string &codes) const {
-  // A simple font's codes are one byte each (ISO 32000-1 9.10.3); its
-  // `ToUnicode` codespace is not to be trusted, producers writing the
-  // two-byte `<0000> <FFFF>` boilerplate there regardless.
+  // Encoding fixes the width for simple and Identity fonts (9.10.3, 9.7.5.2).
   if (!cmap.empty()) {
-    return cmap.translate_string(codes, !composite);
+    const std::size_t width = !composite ? 1 : has_identity_encoding() ? 2 : 0;
+    return cmap.translate_string(codes, width);
   }
   if (composite) {
     // A composite (Type0) font with no `ToUnicode` CMap. A predefined
@@ -194,9 +193,8 @@ std::string Font::to_unicode(const std::string &codes) const {
     // descendant CIDFont's `/CIDSystemInfo`. Only take this when the codes
     // really are CIDs (identity or an embedded stream); a named CMap we lack
     // tables for must not be misread as identity CIDs.
-    const bool identity_cids = cid_encoding_name.empty() ||
-                               cid_encoding_name == "Identity-H" ||
-                               cid_encoding_name == "Identity-V";
+    const bool identity_cids =
+        cid_encoding_name.empty() || has_identity_encoding();
     if (identity_cids && !cid_registry.empty() && !cid_ordering.empty()) {
       std::string result;
       for (const std::uint32_t cid : this->codes(codes)) {
@@ -221,7 +219,7 @@ std::string Font::to_unicode(const std::string &codes) const {
       !unicode.empty()) {
     return unicode;
   }
-  return cmap.translate_string(codes, true);
+  return cmap.translate_string(codes, 1);
 }
 
 } // namespace odr::internal::pdf

src/odr/internal/pdf/pdf_document_element.hpp

@@ -216,10 +216,8 @@ struct Pattern final : Element {
 /// big-endian codes on iteration; must not outlive the bytes. A trailing
 /// partial code is dropped, as the text-showing operators do.
 ///
-/// A `codespace` CMap picks each code's width from its first byte (ISO 32000-1
-/// 9.7.6.2), so a mixed 1-/2-byte encoding (a 1-byte space among 2-byte CIDs,
-/// which PDFClown et al. emit) stays aligned; without one the width is a fixed
-/// `Font::code_byte_width()`.
+/// A codespace matches every byte (ISO 32000-1 9.7.6.2); without one,
+/// `Font::code_byte_width()` supplies the fixed width.
 class CodeRange {
 public:
   class Iterator {
@@ -362,15 +360,20 @@ struct Font final : Element {
     return composite ? 2 : 1;
   }
 
+  [[nodiscard]] bool has_identity_encoding() const {
+    return cid_encoding_name == "Identity-H" ||
+           cid_encoding_name == "Identity-V";
+  }
+
   /// View `codes` as character codes, each yielded as the CID it selects. The
   /// result borrows `codes`.
   ///
   /// The split is by the embedded CID `/Encoding` CMap's codespace if there is
-  /// one, else the `/ToUnicode` codespace (which keeps the split identical to
-  /// `to_unicode`'s), else the fixed `code_byte_width`.
+  /// one, else the `/ToUnicode` codespace. Identity-H/V always use two bytes
+  /// (ISO 32000-1 9.7.5.2).
   [[nodiscard]] CodeRange codes(std::string_view codes) const {
     const std::size_t width = code_byte_width();
-    if (!composite) {
+    if (!composite || has_identity_encoding()) {
       return {codes, width, nullptr, nullptr};
     }
     const CMap *cid_map = cid_encoding.has_cid_map() ? &cid_encoding : nullptr;

test/src/internal/pdf/pdf_cmap.cpp

@@ -241,7 +241,7 @@ TEST(PdfCMap, imposed_code_width_overrides_codespace) {
                     "endbfchar\n");
 
   EXPECT_EQ(cmap.translate_string("\x41\x42"), "\xe4\x85\x82"); // U+4142
-  EXPECT_EQ(cmap.translate_string("\x41\x42", true), "AB");
+  EXPECT_EQ(cmap.translate_string("\x41\x42", 1), "AB");
 }
 
 TEST(PdfCMap, imposed_code_width_falls_back_to_a_padded_entry) {
@@ -253,7 +253,7 @@ TEST(PdfCMap, imposed_code_width_falls_back_to_a_padded_entry) {
                     "<0042> <0042>\n"
                     "endbfchar\n");
 
-  EXPECT_EQ(cmap.translate_string("\x41\x42", true), "AB");
+  EXPECT_EQ(cmap.translate_string("\x41\x42", 1), "AB");
 }
 
 TEST(PdfCMap, imposed_code_width_keeps_a_mixed_codespace_distinct) {

test/src/internal/pdf/pdf_font.cpp

@@ -113,6 +113,25 @@ TEST(PdfFont, composite_to_unicode_splits_by_cmap_codespace) {
   EXPECT_EQ(font.to_unicode("\x41\x42"), "Z");
 }
 
+TEST(PdfFont, identity_encoding_ignores_tounicode_codespace) {
+  Font font;
+  font.composite = true;
+  font.cmap.add_codespace_range(codes2({0x00e0}), codes2({0x028c}));
+  font.cmap.map_single(codes2({0x00fc}), u"\u0107");
+  font.cmap.map_single(codes2({0x0105}), u"\u0105");
+  const std::string bytes = codes2({0x00fc, 0x0105}) + '\0';
+
+  for (const auto *encoding : {"Identity-H", "Identity-V"}) {
+    font.cid_encoding_name = encoding;
+    EXPECT_EQ(font.to_unicode(bytes), "ćą");
+    std::vector<std::uint32_t> codes;
+    for (const std::uint32_t code : font.codes(bytes)) {
+      codes.push_back(code);
+    }
+    EXPECT_EQ(codes, (std::vector<std::uint32_t>{0x00fc, 0x0105}));
+  }
+}
+
 TEST(PdfFont, simple_font_glyph_for_code_via_cmap) {
   // A simple (1-byte) TrueType font: the code's Unicode reaches the glyph
   // through the embedded (3,1) cmap.