From 936f665a93aaabe3774f61ba2aa594a4834a7657 Mon Sep 17 00:00:00 2001
From: George <georgian@dusk.network>
Date: Wed, 30 Sep 2026 18:17:45 +0400
Subject: [PATCH] Allow AI use signals in docs robots.txt

---
 public/robots.txt           |  1 +
 test/machine-files.test.mjs | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/public/robots.txt b/public/robots.txt
index 4b753bd..ac39801 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,4 +1,5 @@
 User-agent: *
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 Allow: /
 
 Sitemap: https://docs.dusk.network/sitemap-index.xml
diff --git a/test/machine-files.test.mjs b/test/machine-files.test.mjs
index b22b4d5..cc822cc 100644
--- a/test/machine-files.test.mjs
+++ b/test/machine-files.test.mjs
@@ -17,6 +17,36 @@ test("robots.txt permits crawling and advertises the generated sitemap index", a
   assert.doesNotMatch(robots, /^Disallow: \/$/m);
 });
 
+test("robots.txt explicitly permits search, AI input, and AI training", async () => {
+  const sourceRobots = await readPublicFile("robots.txt");
+  const builtRobots = await readFile(
+    new URL("../dist/robots.txt", import.meta.url),
+    "utf8",
+  );
+  const contentSignal = builtRobots
+    .split(/\r?\n/)
+    .find((line) => line.toLowerCase().startsWith("content-signal:"));
+
+  assert.equal(builtRobots, sourceRobots);
+  assert.ok(contentSignal, "robots.txt must include a Content-Signal directive");
+
+  const preferences = Object.fromEntries(
+    contentSignal
+      .slice(contentSignal.indexOf(":") + 1)
+      .split(",")
+      .map((preference) => {
+        const [name, value] = preference.trim().split("=");
+        return [name, value];
+      }),
+  );
+
+  assert.deepEqual(preferences, {
+    search: "yes",
+    "ai-input": "yes",
+    "ai-train": "yes",
+  });
+});
+
 test("the conventional sitemap URL mirrors Astro's generated sitemap index", async () => {
   const generatedIndex = await readFile(
     new URL("../dist/sitemap-index.xml", import.meta.url),