Preserve signed child JSON during typed compound serialization

CHANGES.md

@@ -29,6 +29,29 @@ To be released.
         the new *Clearing legacy cache entries* section of the
         [key–value store guide] to clear them proactively instead of waiting.
 
+ -  Fixed `signObject()` so that a signed object keeps verifying after it is
+    assigned to a typed parent and the parent is serialized.  `signObject()`
+    now captures the secured JSON document its proof covers, and nested
+    serialization embeds that document verbatim instead of rebuilding the
+    child under the parent's JSON-LD context.  Producing a [FEP-ef61] compound
+    document with `signObject()` no longer requires assembling the JSON by
+    hand.  [[#288], [#1041], [#1044], [#1051]]
+
+     -  The captured document is a snapshot: `clone()` does not carry it, and
+        mutating a signed object in place does not change it.  Sign a clone
+        again when it has to be embedded as a secured child.
+     -  An object parsed with `fromJsonLd()`, an object that already carried a
+        proof, and a `toJsonLd()` `context` option that could hide the
+        internal placeholder all fall back to the previous behavior.
+     -  Outgoing JSON-LD compatibility normalization now leaves a nested
+        self-contained secured document untouched while signing and sending,
+        so it cannot rewrite bytes that the child's own proof covers.
+        Inbound verification is unaffected.
+     -  Fanout delivery now reuses the document the activity was already
+        serialized into instead of reparsing and reserializing it, which
+        previously invalidated an embedded signed child and the outer proof
+        that covered it.
+
  -  Fixed `verifyProof()` so Ed25519 JCS proofs authenticate every received
     proof option except `proofValue`, including `expires`, `domain`,
     `challenge`, `nonce`, and extension options.  It now rejects expired or
@@ -166,6 +189,8 @@ To be released.
 [#1017]: https://github.com/fedify-dev/fedify/issues/1017
 [#1027]: https://github.com/fedify-dev/fedify/pull/1027
 [#1041]: https://github.com/fedify-dev/fedify/pull/1041
+[#1044]: https://github.com/fedify-dev/fedify/issues/1044
+[#1051]: https://github.com/fedify-dev/fedify/pull/1051
 
 ### @fedify/adonisjs
 
@@ -436,6 +461,13 @@ To be released.
         `Note`, and other object types, for per-language translation
         metadata without creating separate posts.
 
+ -  Changed nested serialization so that an object carrying a signed JSON-LD
+    representation retained by `signObject()` is embedded with that exact
+    representation, including its own `@context`, rather than being rebuilt
+    under the parent's context.  `clone()` never carries the retained
+    representation, because a clone may differ from the document the proof
+    covers.  [[#288], [#1044], [#1051]]
+
 [FEP-22cd]: https://w3id.org/fep/22cd
 [#810]: https://github.com/fedify-dev/fedify/issues/810
 [#826]: https://github.com/fedify-dev/fedify/issues/826

changes.d/fedify/signed-child-representation.md

@@ -0,0 +1,31 @@
+---
+links:
+  '#1041': https://github.com/fedify-dev/fedify/pull/1041
+  '#1044': https://github.com/fedify-dev/fedify/issues/1044
+  '#1051': https://github.com/fedify-dev/fedify/pull/1051
+  '#288': https://github.com/fedify-dev/fedify/issues/288
+---
+ -  Fixed `signObject()` so that a signed object keeps verifying after it is
+    assigned to a typed parent and the parent is serialized.  `signObject()`
+    now captures the secured JSON document its proof covers, and nested
+    serialization embeds that document verbatim instead of rebuilding the
+    child under the parent's JSON-LD context.  Producing a [FEP-ef61] compound
+    document with `signObject()` no longer requires assembling the JSON by
+    hand.  [[#288], [#1041], [#1044], [#1051]]
+
+     -  The captured document is a snapshot: `clone()` does not carry it, and
+        mutating a signed object in place does not change it.  Sign a clone
+        again when it has to be embedded as a secured child.
+     -  An object parsed with `fromJsonLd()`, an object that already carried a
+        proof, and a `toJsonLd()` `context` option that could hide the
+        internal placeholder all fall back to the previous behavior.
+     -  Outgoing JSON-LD compatibility normalization now leaves a nested
+        self-contained secured document untouched while signing and sending,
+        so it cannot rewrite bytes that the child's own proof covers.
+        Inbound verification is unaffected.
+     -  Fanout delivery now reuses the document the activity was already
+        serialized into instead of reparsing and reserializing it, which
+        previously invalidated an embedded signed child and the outer proof
+        that covered it.
+
+[FEP-ef61]: https://w3id.org/fep/ef61

changes.d/vocab/signed-child-representation.md

@@ -0,0 +1,12 @@
+---
+links:
+  '#1044': https://github.com/fedify-dev/fedify/issues/1044
+  '#1051': https://github.com/fedify-dev/fedify/pull/1051
+  '#288': https://github.com/fedify-dev/fedify/issues/288
+---
+ -  Changed nested serialization so that an object carrying a signed JSON-LD
+    representation retained by `signObject()` is embedded with that exact
+    representation, including its own `@context`, rather than being rebuilt
+    under the parent's context.  `clone()` never carries the retained
+    representation, because a clone may differ from the document the proof
+    covers.  [[#288], [#1044], [#1051]]

docs/manual/send.md

@@ -1170,19 +1170,87 @@ verified in isolation even if the parent's active context causes the embedded
 JSON-LD to expand differently.  Proof success does not establish that the
 isolated and embedded expansions are equivalent.
 
+#### Producing a compound document
+
+`signObject()` captures the secured JSON document that the proof it creates
+covers, and keeps it on the object it returns.  Assigning that object to a
+typed parent and serializing the parent embeds the captured document verbatim,
+so the child keeps its own `@context` and its own proof context even when they
+differ from the parent's:
+
+~~~~ typescript twoslash
+import { signObject } from "@fedify/fedify";
+import { Create, Note } from "@fedify/vocab";
+const noteId = new URL("ap://did:key:z6Mkabc/objects/1");
+const activityId = new URL("ap://did:key:z6Mkabc/activities/1");
+const actorId = new URL("ap://did:key:z6Mkabc/actor");
+const key = null as unknown as CryptoKey;
+const keyId = new URL("did:key:z6Mkabc#z6Mkabc");
+const portableContext = [
+  "https://www.w3.org/ns/activitystreams",
+  "https://w3id.org/security/data-integrity/v1",
+  "https://w3id.org/fep/ef61",
+];
+// ---cut-before---
+const note = await signObject(
+  new Note({ id: noteId, attribution: actorId, content: "Hello" }),
+  key,
+  keyId,
+  { context: portableContext },
+);
+const create = await signObject(
+  new Create({ id: activityId, actor: actorId, object: note }),
+  key,
+  keyId,
+  { context: portableContext },
+);
+const compound = await create.toJsonLd({
+  format: "compact",
+  context: portableContext,
+});
+~~~~
+
+Extracting `compound`'s `object` gives back exactly the bytes `note`'s proof
+covers, and the outer proof covers that same secured child.
+
+Serialize the parent with the same `context` it was signed with.  A proof
+covers one serialization, and a document emitted under a different context is
+a document the proof was never computed over.  This matters most when the
+activity goes out through
+[`sendActivity()`](#sending-an-activity), which
+serializes with the type's default context: omit the `context` option on the
+outer `signObject()` call in that case, so the proof covers the bytes Fedify
+actually sends.  The child keeps its own context either way, which is what
+makes a mismatch on the parent easy to misread as a working document.
+
+The captured document is a snapshot.  It is independent of anything done to
+the returned object afterwards, and `clone()` never carries it, because a
+clone may differ from the document the proof covers.  Sign the clone again
+when it has to be embedded as a secured child.
+
 > [!WARNING]
-> Typed vocabulary serialization can change the secured representation of a
-> signed embedded object.  Signing a typed child, assigning it to a typed
-> parent, and serializing the parent can remove the child's document and proof
-> contexts while retaining its `proofValue`.  Do not serialize a signed typed
-> child through a typed parent when producing a map-local compound document.
-> The profile also accepts exactly one direct proof per map, while Fedify's
-> ordinary activity signer creates one proof for each Ed25519 key.  A sender
-> producing a portable compound document must arrange for exactly one direct
-> proof on each map.
-> When forwarding an already signed payload, use
-> [`forwardActivity()`](./outbox.md#federating-posted-activities) to avoid a
-> vocabulary-object round trip.
+> Several things take a signed child outside this supported path, and each
+> one falls back to ordinary serialization, which rebuilds the child under the
+> parent's context and leaves it unable to verify on its own:
+>
+>  -  An object parsed with `fromJsonLd()`.  Parsing does not establish which
+>     representation was signed, so nothing is captured.  When forwarding an
+>     already signed payload, use
+>     [`forwardActivity()`](./outbox.md#federating-posted-activities) to avoid
+>     a vocabulary-object round trip.
+>  -  An object that already carried a proof.  The profile accepts exactly one
+>     direct proof per map, while Fedify's ordinary activity signer creates one
+>     proof for each Ed25519 key, so a sender producing a portable compound
+>     document must arrange for exactly one direct proof on each map.
+>  -  A `toJsonLd()` call whose `context` option could hide the marker Fedify
+>     uses to place the captured document, for example a context that aliases
+>     `@id` under a term other than `id`, declares `@nest`, uses an `@id` or
+>     `@type` container, or cannot be resolved from Fedify's preloaded
+>     contexts.
+>  -  A mutation applied to the returned object in place, through a plural
+>     accessor's array or a `proofValue` byte.  The snapshot still holds what
+>     was signed, so the embedded child keeps verifying while the typed object
+>     no longer matches it.
 
 > [!TIP]
 > HTTPS Signatures, Linked Data Signatures, and Object Integrity Proofs can

packages/fedify/src/compat/outgoing-jsonld.test.ts

@@ -285,3 +285,114 @@ test("normalizeOutgoingActivityJsonLd() applies outgoing JSON-LD workarounds", a
   const normalizedObject = normalized.object as Record<string, unknown>;
   assertEquals(Array.isArray(normalizedObject.attachment), true);
 });
+
+const securedChild = {
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://w3id.org/security/data-integrity/v1",
+  ],
+  id: "ap://did:key:z6Mkabc/objects/1",
+  type: "Note",
+  content: "Hello",
+  to: "as:Public",
+  attachment: {
+    type: "Document",
+    mediaType: "image/png",
+    url: "https://example.com/image.png",
+  },
+  proof: {
+    "@context": [
+      "https://www.w3.org/ns/activitystreams",
+      "https://w3id.org/security/data-integrity/v1",
+    ],
+    type: "DataIntegrityProof",
+    cryptosuite: "eddsa-jcs-2022",
+    created: "2023-02-24T23:36:38Z",
+    verificationMethod: "did:key:z6Mkabc#z6Mkabc",
+    proofPurpose: "assertionMethod",
+    proofValue: "z3FXQ",
+  },
+};
+
+function compoundWithSecuredChild(): Record<string, unknown> {
+  return {
+    "@context": [
+      "https://www.w3.org/ns/activitystreams",
+      "https://w3id.org/security/data-integrity/v1",
+    ],
+    id: "ap://did:key:z6Mkdef/activities/1",
+    type: "Create",
+    actor: "ap://did:key:z6Mkdef/actor",
+    to: "as:Public",
+    object: structuredClone(securedChild),
+  };
+}
+
+test("normalizeOutgoingActivityJsonLd() can preserve a nested secured child", async () => {
+  const input = compoundWithSecuredChild();
+  const preserved = await normalizeOutgoingActivityJsonLd(
+    input,
+    mockDocumentLoader,
+    { preserveNestedSecuredDocuments: true },
+  ) as Record<string, unknown>;
+
+  // The parent is still normalized …
+  assertEquals(preserved.to, PUBLIC_COLLECTION.href);
+  // … while every byte the child's own proof covers is left alone.
+  assertEquals(preserved.object, securedChild);
+});
+
+test("normalizeOutgoingActivityJsonLd() rewrites a nested secured child by default", async () => {
+  const input = compoundWithSecuredChild();
+  const normalized = await normalizeOutgoingActivityJsonLd(
+    input,
+    mockDocumentLoader,
+  ) as Record<string, unknown>;
+  const child = normalized.object as Record<string, unknown>;
+
+  assertEquals(normalized.to, PUBLIC_COLLECTION.href);
+  assertEquals(child.to, PUBLIC_COLLECTION.href);
+  assertEquals(child.attachment, [securedChild.attachment]);
+});
+
+test("normalizeOutgoingActivityJsonLd() only preserves a complete secured child", async () => {
+  const cases: Record<string, (child: Record<string, unknown>) => void> = {
+    "no proof": (child) => delete child.proof,
+    "null proof": (child) => child.proof = null,
+    "proof set": (child) => child.proof = [securedChild.proof],
+    "proof without a verification method": (child) =>
+      delete (child.proof as Record<string, unknown>).verificationMethod,
+    "incomplete proof": (child) =>
+      delete (child.proof as Record<string, unknown>).cryptosuite,
+    "empty proof value": (child) =>
+      (child.proof as Record<string, unknown>).proofValue = "",
+  };
+  for (const [name, mutate] of Object.entries(cases)) {
+    const input = compoundWithSecuredChild();
+    mutate(input.object as Record<string, unknown>);
+    const normalized = await normalizeOutgoingActivityJsonLd(
+      input,
+      mockDocumentLoader,
+      { preserveNestedSecuredDocuments: true },
+    ) as Record<string, unknown>;
+    const child = normalized.object as Record<string, unknown>;
+
+    assertEquals(child.to, PUBLIC_COLLECTION.href, name);
+    assertEquals(child.attachment, [securedChild.attachment], name);
+  }
+});
+
+test("normalizeOutgoingActivityJsonLd() still normalizes a top-level secured document", async () => {
+  const input = {
+    ...structuredClone(securedChild),
+    id: "ap://did:key:z6Mkabc/objects/top-level",
+  };
+  const normalized = await normalizeOutgoingActivityJsonLd(
+    input,
+    mockDocumentLoader,
+    { preserveNestedSecuredDocuments: true },
+  ) as Record<string, unknown>;
+
+  assertEquals(normalized.to, PUBLIC_COLLECTION.href);
+  assertEquals(normalized.attachment, [securedChild.attachment]);
+});

packages/fedify/src/compat/outgoing-jsonld.ts

@@ -3,6 +3,10 @@ import jsonld from "@fedify/vocab-runtime/jsonld";
 import { getLogger } from "@logtape/logtape";
 import { preloadedOnlyDocumentLoader } from "./preloaded-context-loader.ts";
 import { normalizePublicAudience } from "./public-audience.ts";
+import {
+  isSelfContainedSecuredDocument,
+  type OutgoingNormalizationOptions,
+} from "./secured-document.ts";
 
 const logger = getLogger(["fedify", "compat", "outgoing-jsonld"]);
 
@@ -92,15 +96,23 @@ function getKnownSafeContextUrls(): ReadonlySet<string> {
  */
 function wrapScalarAttachments(
   jsonLd: unknown,
+  preserveSecured: boolean,
   depth: number = 0,
 ): unknown {
   if (depth >= MAX_TRAVERSAL_DEPTH) return jsonLd;
+  // A nested self-contained secured document is signed as it stands; its
+  // bytes belong to its own proof, not to this document's wire format.
+  if (
+    preserveSecured && depth > 0 && isSelfContainedSecuredDocument(jsonLd)
+  ) {
+    return jsonLd;
+  }
 
   if (Array.isArray(jsonLd)) {
     let normalized: unknown[] | null = null;
     for (let i = 0; i < jsonLd.length; i++) {
       const item = jsonLd[i];
-      const next = wrapScalarAttachments(item, depth + 1);
+      const next = wrapScalarAttachments(item, preserveSecured, depth + 1);
       if (normalized == null && next !== item) {
         normalized = jsonLd.slice(0, i);
       }
@@ -122,7 +134,7 @@ function wrapScalarAttachments(
     const next = key === "@context" ||
         (key === "@value" && isJsonLdValueObject(jsonLd))
       ? value
-      : wrapScalarAttachments(value, depth + 1);
+      : wrapScalarAttachments(value, preserveSecured, depth + 1);
     const shouldWrap = ATTACHMENT_FIELDS.has(key) &&
       next != null &&
       !Array.isArray(next) &&
@@ -247,8 +259,12 @@ function getLogSafeJsonLdMetadata(jsonLd: unknown): Record<string, unknown> {
 export async function normalizeAttachmentArrays(
   jsonLd: unknown,
   contextLoader?: DocumentLoader,
+  options: OutgoingNormalizationOptions = {},
 ): Promise<unknown> {
-  const normalized = wrapScalarAttachments(jsonLd);
+  const normalized = wrapScalarAttachments(
+    jsonLd,
+    options.preserveNestedSecuredDocuments ?? false,
+  );
   if (normalized === jsonLd) return jsonLd;
   if (exceedsTraversalDepth(jsonLd)) {
     logger.debug(
@@ -298,7 +314,8 @@ export async function normalizeAttachmentArrays(
 export async function normalizeOutgoingActivityJsonLd(
   jsonLd: unknown,
   contextLoader?: DocumentLoader,
+  options: OutgoingNormalizationOptions = {},
 ): Promise<unknown> {
-  jsonLd = await normalizePublicAudience(jsonLd, contextLoader);
-  return await normalizeAttachmentArrays(jsonLd, contextLoader);
+  jsonLd = await normalizePublicAudience(jsonLd, contextLoader, options);
+  return await normalizeAttachmentArrays(jsonLd, contextLoader, options);
 }