Cut a roster line between graphemes, not between code points

CHANGELOG.md

@@ -8,6 +8,16 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
 
 ## Unreleased
 
+### A flag or a family emoji in a channel preview is no longer cut in half
+
+The one line a roster draws is cut to a cap, and the cut walked code points -- right for a plain
+emoji, wrong for every emoji built out of more than one. A flag is two regional indicators, a family
+is three people joined by zero-width joiners, a thumbs-up with a skin tone is the thumb plus a
+modifier, and a keycap is a digit plus a variation selector plus an enclosing mark. Landing the cut
+inside any of those left a boxed letter, a dangling joiner or a bare digit in the sidebar, in the
+generated channel title, and in the excerpt the titler is shown. The cut is now taken between
+grapheme clusters, so what a person sees as one character is kept or dropped whole. Plain text is
+cut in exactly the same place as before.
 ### A scroll with an unusable `deltaY` is refused, rather than scrolling some other distance
 
 `POST /computers/:botId/scroll` and `POST /computers/:botId/human/scroll` accepted any JSON number

server/src/channels/routes.ts

@@ -212,11 +212,11 @@ export type ChannelStore = {
 
 const PRIVATE_AGENT_CHANNEL_DESCRIPTION = "Private agent channel.";
 const MAX_CHANNEL_NAME_CODE_POINTS = 120;
-const MAX_ACTIVITY_CODE_POINTS = 200;
+const MAX_ACTIVITY_GRAPHEMES = 200;
 
 /** Reduce a message to the one line a roster draws. See `oneLine` for why it is shared. */
 function previewOf(text: string) {
-  return oneLine(text, MAX_ACTIVITY_CODE_POINTS);
+  return oneLine(text, MAX_ACTIVITY_GRAPHEMES);
 }
 
 function channelName(names: string[]) {

server/src/channels/summary.ts

@@ -22,10 +22,10 @@ import { oneLine } from "./text";
 export const CHANNEL_SUMMARY_KIND = "channel.summary";
 
 /** A title long enough to truncate says no more than the preview it replaced. */
-const MAX_SUMMARY_CODE_POINTS = 60;
+const MAX_SUMMARY_GRAPHEMES = 60;
 
 /** How much of the opening exchange the model is shown. Enough to see the topic, not the whole run. */
-const MAX_EXCERPT_CODE_POINTS = 600;
+const MAX_EXCERPT_GRAPHEMES = 600;
 
 /** A seam, so a test drives every path with no key and no network. Null means nothing worth writing. */
 export type ChannelTitler = (excerpt: string) => Promise<string | null>;
@@ -224,7 +224,7 @@ async function summariseOne(
   const answer = await options.title(excerpt);
   if (!answer) return "nothing to name it with";
 
-  const title = oneLine(stripWrappingQuotes(answer), MAX_SUMMARY_CODE_POINTS);
+  const title = oneLine(stripWrappingQuotes(answer), MAX_SUMMARY_GRAPHEMES);
   if (!title) return "nothing to name it with";
 
   return await options.database.transaction<Attempt>(
@@ -288,7 +288,7 @@ async function openingOf(
 
   return oneLine(
     replied ? `Asked: ${asked}\nAnswered: ${replied}` : `Asked: ${asked}`,
-    MAX_EXCERPT_CODE_POINTS,
+    MAX_EXCERPT_GRAPHEMES,
   );
 }
 

server/src/channels/text.ts

@@ -1,12 +1,31 @@
 /**
- * One line a roster can draw: control characters stripped, whitespace collapsed, cut on code points
- * so an emoji is never split. The caller supplies the cap; a preview and a title want different ones.
+ * The unit a person sees as one character.
+ *
+ * `Array.from` splits on code points, which is right for a plain emoji and wrong for every emoji
+ * built out of more than one. A flag is two regional indicators, a family is three people joined by
+ * zero-width joiners, a thumbs-up with a skin tone is the thumb plus a modifier, and a keycap is a
+ * digit plus a variation selector plus an enclosing mark. Cut between any of those parts and what is
+ * left is not a shorter emoji: it is a boxed letter, a dangling joiner, or a bare digit.
  */
-export function oneLine(text: string, maxCodePoints: number): string {
+const GRAPHEMES = new Intl.Segmenter(undefined, { granularity: "grapheme" });
+
+/**
+ * One line a roster can draw: control characters stripped, whitespace collapsed, cut on grapheme
+ * clusters so an emoji is never split. The caller supplies the cap; a preview and a title want
+ * different ones.
+ */
+export function oneLine(text: string, maxGraphemes: number): string {
   // biome-ignore lint/suspicious/noControlCharactersInRegex: stripping them is the point.
   const flattened = text.replace(/[\u0000-\u001f\u007f-\u009f]+/g, " ").trim();
   const collapsed = flattened.replace(/\s+/g, " ");
-  const codePoints = Array.from(collapsed);
-  if (codePoints.length <= maxCodePoints) return collapsed;
-  return `${codePoints.slice(0, maxCodePoints - 1).join("")}…`;
+  // A string can never hold more graphemes than it holds UTF-16 units, so a line this short is
+  // already under the cap and needs no segmenting. Nearly everything a roster draws is that short,
+  // and segmenting is the expensive part of this function.
+  if (collapsed.length <= maxGraphemes) return collapsed;
+  const graphemes = Array.from(
+    GRAPHEMES.segment(collapsed),
+    (each) => each.segment,
+  );
+  if (graphemes.length <= maxGraphemes) return collapsed;
+  return `${graphemes.slice(0, maxGraphemes - 1).join("")}…`;
 }

server/tests/channel-one-line.test.ts

@@ -0,0 +1,77 @@
+import { describe, expect, test } from "bun:test";
+import { oneLine } from "../src/channels/text";
+
+const SEGMENTER = new Intl.Segmenter(undefined, { granularity: "grapheme" });
+const graphemesOf = (value: string): string[] =>
+  Array.from(SEGMENTER.segment(value), (each) => each.segment);
+
+/** South Korea: two regional indicators. Cut between them and one letter is left in a box. */
+const FLAG = "\u{1F1F0}\u{1F1F7}";
+/** Man, woman, girl, joined. Cut anywhere and a zero-width joiner dangles off the end. */
+const FAMILY = "\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}";
+/** A thumbs-up plus a skin-tone modifier. Cut and the modifier is dropped. */
+const THUMB = "\u{1F44D}\u{1F3FD}";
+/** Digit, variation selector, enclosing keycap. Cut and it is a bare 1. */
+const KEYCAP = "1\u{FE0F}\u{20E3}";
+
+describe("oneLine", () => {
+  /*
+   * The guard against over-correcting. Every one of these passes on the previous implementation
+   * too: the cut lands in the same place, the ellipsis replaces the same last unit, and control
+   * characters and runs of whitespace are still flattened. Only the definition of "one unit" moved.
+   */
+  test("leaves a short line alone", () => {
+    expect(oneLine("Deploy the staging build", 200)).toBe(
+      "Deploy the staging build",
+    );
+  });
+
+  test("flattens control characters and collapses whitespace", () => {
+    expect(oneLine("one\ttwo  three\nfour", 200)).toBe("one two three four");
+  });
+
+  test("cuts plain text to the cap, ellipsis included", () => {
+    expect(oneLine("abcdefghij", 5)).toBe("abcd…");
+    expect(graphemesOf(oneLine("abcdefghij", 5))).toHaveLength(5);
+  });
+
+  test("keeps a single-code-point emoji whole", () => {
+    expect(oneLine(`aaaa\u{1F600}bbbb`, 5)).toBe("aaaa…");
+  });
+
+  /*
+   * The bug. `Array.from` walks code points, and every emoji below is made of more than one, so the
+   * cut landed inside the emoji the docstring promised never to split.
+   */
+  test.each([
+    ["a flag", FLAG],
+    ["a joined family", FAMILY],
+    ["a skin-tone thumb", THUMB],
+    ["a keycap", KEYCAP],
+  ])("never cuts %s in half", (_name, emoji) => {
+    const line = `${emoji} report`;
+    const whole = graphemesOf(line);
+
+    // Every cap from 1 up past the whole line, so no single lucky boundary can carry the test.
+    for (let cap = 1; cap <= whole.length + 2; cap += 1) {
+      const cut = oneLine(line, cap);
+      const kept = cut.endsWith("…") ? cut.slice(0, -1) : cut;
+
+      expect(graphemesOf(cut).length).toBeLessThanOrEqual(cap);
+      // What survives is a whole number of the line's own clusters, taken from the front. Anything
+      // split in half fails here, because its pieces are not clusters of the original.
+      expect(graphemesOf(kept)).toEqual(
+        whole.slice(0, graphemesOf(kept).length),
+      );
+    }
+  });
+
+  test("cuts between emoji rather than inside one", () => {
+    // Three clusters, sixteen UTF-16 units, five code points' worth of joiners and modifiers
+    // between them. A cap of three is the whole line; a cap of two keeps one emoji and the ellipsis.
+    expect(oneLine(`${FLAG}${FAMILY}${THUMB}`, 3)).toBe(
+      `${FLAG}${FAMILY}${THUMB}`,
+    );
+    expect(oneLine(`${FLAG}${FAMILY}${THUMB}`, 2)).toBe(`${FLAG}…`);
+  });
+});