Identify crawlers from the shared generated registry (#86)

.github/workflows/ci.yml

@@ -73,3 +73,21 @@ jobs:
           --extensions=php
           --ignore=vendor/,sdk/vendor/,build/,dist/
           .
+
+  sdk-tests:
+    name: SDK behaviour and registry parity
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '7.4'
+
+      # Dependency-free: the pure SDK classes, the AI crawler policy, and the
+      # generated crawler registry replayed against the shared parity vectors.
+      # A hand edit to the generated table, or a regeneration whose vectors
+      # were not committed with it, fails here.
+      - name: Run tests
+        run: php tests/run.php

changelog.txt

@@ -1,5 +1,11 @@
 *** WebDecoy Bot Detection Changelog ***
 
+= Unreleased =
+* Changed: known crawlers are now identified from the same crawler registry the WebDecoy dashboard and the other WebDecoy sensors use, instead of a list kept only in this plugin. A crawler is named the same thing here and in your reports. The registry knows 182 crawlers where the old list knew 54, so more search engines (Naver, Seznam, Ecosia, Mojeek and others), more AI crawlers (ByteSpider, Mistral, YouBot and others) and more link-preview, monitoring, SEO and feed services are recognised as what they are.
+* Changed: with Block AI crawlers on, AI agents and assistants that browse on a person's behalf (for example ChatGPT-User, Claude-User, Operator, Siri) are refused along with training crawlers, and so are the newly recognised AI crawlers above. ChatGPT-User and OAI-SearchBot are now classified as OpenAI documents them (an assistant and a search crawler, neither used for training), which matters once per-path crawler policy arrives; under this setting they are refused either way. The setting is the one instruction about AI traffic; if you want a specific one through, add it to the custom allowlist by name.
+* Changed: the W3C markup and Validator.nu checkers are no longer listed as known bots. They were never treated as good bots, so requests from them are scored exactly as before; they simply no longer carry a bot name.
+* Internal: the crawler table is generated from WebDecoy/app and replayed against shared parity vectors in CI, so it cannot drift from the other sensors unnoticed. PHP 7.4 compatibility and the SDK behaviour tests now run in CI.
+
 = 2.8.3 - 2026-09-21 =
 * Fixed: Block AI crawlers is now an explicit decision rather than a scoring hint. The setting worked by withdrawing a recognised AI crawler's good-bot pass and leaving the rest to heuristic scoring. Scoring adds no points for a recognised bot, so a well-behaved AI crawler sending ordinary headers could stay under the block threshold. With the setting on, a recognised AI crawler is now refused whatever it scores, and the Detections page records the reason as ai_crawler_blocked. The custom allowlist still wins, search engines and social bots are unaffected, and in monitor mode the block is counted rather than applied, like every other action. If you rely on this setting, please update.
 * Internal: added behaviour tests for the AI crawler setting.

sdk/src/AgentRegistry.php

@@ -0,0 +1,114 @@
+<?php
+
+declare(strict_types=1);
+
+namespace WebDecoy;
+
+if (!defined('ABSPATH')) {
+    exit;
+}
+
+/**
+ * The canonical crawler registry, generated from WebDecoy/app.
+ *
+ * The plugin used to carry its own table of known bots. That table and the
+ * one the dashboard, the edge sensor and the Node SDK use drifted apart, so a
+ * crawler could be named one thing in this plugin and another in the report
+ * about it. `registry/agents.generated.php` is the shared table in PHP form,
+ * and this class is the only reader of it.
+ *
+ * Matching is by lower-cased User-Agent containing an agent's pattern, first
+ * hit in file order. The order is the same one every other consumer uses,
+ * which is what keeps the answers equal.
+ */
+final class AgentRegistry
+{
+    /**
+     * The artifact shape this class was written against. A regenerated file
+     * with a different shape is refused rather than misread.
+     */
+    public const SCHEMA = 2;
+
+    /** @var array<string,mixed>|null */
+    private static $table = null;
+
+    /**
+     * Load the generated table once per process.
+     *
+     * @return array{schema:int,agent_count:int,categories:list<string>,agents:list<array<string,mixed>>}
+     */
+    private static function table(): array
+    {
+        if (self::$table === null) {
+            $loaded = require __DIR__ . '/registry/agents.generated.php';
+            if (!is_array($loaded) || (int) ($loaded['schema'] ?? 0) !== self::SCHEMA) {
+                throw new \RuntimeException(
+                    'WebDecoy agent registry schema mismatch: expected ' . self::SCHEMA
+                    . ', found ' . (string) ($loaded['schema'] ?? 'none')
+                );
+            }
+            self::$table = $loaded;
+        }
+        return self::$table;
+    }
+
+    /**
+     * Identify the agent a User-Agent claims to be.
+     *
+     * A claim, not a verification: the User-Agent header is whatever the
+     * client chose to send. Callers that need more ask GoodBotList to verify
+     * the source address.
+     *
+     * @return array{id:string,name:string,category:string,behavior:string,organization:string,pattern:string,website:string,robots_name:string}|null
+     */
+    public static function match(string $userAgent): ?array
+    {
+        if ($userAgent === '') {
+            return null;
+        }
+        $ua = strtolower($userAgent);
+        foreach (self::table()['agents'] as $agent) {
+            foreach ($agent['patterns'] as $pattern) {
+                if ($pattern !== '' && strpos($ua, $pattern) !== false) {
+                    return [
+                        'id' => (string) $agent['id'],
+                        'name' => (string) $agent['name'],
+                        'category' => (string) $agent['category'],
+                        'behavior' => (string) ($agent['behavior'] ?? ''),
+                        'organization' => (string) $agent['organization'],
+                        'pattern' => (string) $pattern,
+                        'website' => (string) $agent['website'],
+                        'robots_name' => (string) $agent['robots_name'],
+                    ];
+                }
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Every agent, in match order.
+     *
+     * @return list<array<string,mixed>>
+     */
+    public static function all(): array
+    {
+        return self::table()['agents'];
+    }
+
+    /**
+     * The category vocabulary the table uses (the customer-facing names).
+     *
+     * @return list<string>
+     */
+    public static function categories(): array
+    {
+        return self::table()['categories'];
+    }
+
+    /** How many agents the table carries, for drift checks. */
+    public static function count(): int
+    {
+        return (int) self::table()['agent_count'];
+    }
+}