fix: make the Block AI crawlers setting an explicit decision

sdk/src/BotDetector.php

@@ -28,6 +28,9 @@ class BotDetector
     private const SCORE_RATE_EXCEEDED = 25;
     private const SCORE_HONEYPOT = 60;
     private const SCORE_FAKE_BOT = 80; // Claiming to be a bot but IP doesn't verify
+    // The owner's own instruction to block. The top of the scale, so it clears
+    // min_score_to_block whatever the site has set it to.
+    private const SCORE_POLICY_DENIED = 100;
 
     // Path-based scoring (MITRE ATT&CK aligned)
     private const SCORE_PATH_CONFIG_FILE = 30;      // TA0006 Credential Access - config files
@@ -194,6 +197,29 @@ public function analyze(?array $signals = null, ?string $clientIP = null): Detec
             $result->setBotName($botInfo['name']);
             $result->setBotCategory($botInfo['category']);
 
+            // A crawler the owner has chosen to block is refused here, as an
+            // explicit decision, before any heuristic scoring.
+            //
+            // The setting used to work by withdrawing the crawler's good-bot
+            // pass and letting scoring decide. Scoring does not add points for a
+            // recognised bot, so a well-behaved AI crawler could finish below
+            // the block threshold. An owner's instruction should not depend on
+            // how the request happens to score.
+            //
+            // The User-Agent claim is enough to act on: refusing a request
+            // because it says it is GPTBot affects nobody who is not claiming
+            // to be GPTBot.
+            if ($this->deniedByPolicy($botInfo)) {
+                $result->setIsGoodBot(false);
+                $result->setScore(self::SCORE_POLICY_DENIED);
+                $result->setScoreBreakdown(['ai_crawler_blocked' => self::SCORE_POLICY_DENIED]);
+                $result->setConfidence(1.0);
+                $result->addFlag('ai_crawler_blocked');
+                $result->addMetadata('bot_info', $botInfo);
+                $result->addMetadata('denied_by', 'block_ai_crawlers');
+                return $result;
+            }
+
             // Check if this bot should be allowed
             if ($this->shouldAllowBot($botInfo)) {
                 // Verify the bot's IP if verification is enabled
@@ -516,6 +542,26 @@ public function identifyBot(string $userAgent): ?array
         return $this->goodBotList->identify($userAgent);
     }
 
+    /**
+     * Whether the owner's settings say to block this recognised bot outright.
+     *
+     * Only the AI crawler category has a "block" setting. Turning OFF "allow
+     * search engines" or "allow social bots" withdraws a free pass and lets the
+     * heuristics judge the request; it is not an instruction to block. The
+     * custom allowlist wins, as it does in shouldAllowBot().
+     *
+     * @param array $botInfo Bot information
+     * @return bool
+     */
+    private function deniedByPolicy(array $botInfo): bool
+    {
+        if (in_array($botInfo['name'] ?? '', $this->options['custom_allowlist'], true)) {
+            return false;
+        }
+        return ($botInfo['category'] ?? '') === GoodBotList::CATEGORY_AI_CRAWLER
+            && !empty($this->options['block_ai_crawlers']);
+    }
+
     /**
      * Check if a bot should be allowed based on policies
      *

tests/AiCrawlerPolicyTest.php

@@ -0,0 +1,88 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * The "Block AI crawlers" setting is an instruction, not a scoring hint.
+ *
+ * It used to withdraw the crawler's good-bot pass and leave the outcome to
+ * heuristic scoring, where a recognised bot earns no user-agent points. A
+ * well-behaved AI crawler could therefore finish under the block threshold.
+ * These tests pin the behaviour: on means refused, at any threshold; off means
+ * a good bot; other categories and the custom allowlist are unaffected.
+ *
+ * Run: php tests/run.php
+ */
+
+if (!defined('ABSPATH')) {
+    define('ABSPATH', '/tmp/');
+}
+if (!function_exists('get_transient')) {
+    function get_transient($key) { return false; }
+}
+if (!function_exists('set_transient')) {
+    function set_transient($key, $value, $ttl = 0) { return true; }
+}
+foreach (['DetectionResult', 'GoodBotList', 'SignalCollector', 'MitreMapping', 'BotDetector'] as $class) {
+    $file = dirname(__DIR__) . '/sdk/src/' . $class . '.php';
+    if (is_file($file)) {
+        require_once $file;
+    }
+}
+
+$t = ['TestRunner', 'test'];
+$same = ['TestRunner', 'assertSame'];
+$true = ['TestRunner', 'assertTrue'];
+
+const AI_POLICY_DEFAULT_THRESHOLD = 75;
+
+function ai_policy_analyze(string $userAgent, array $options, bool $browserHeaders = true): \WebDecoy\DetectionResult
+{
+    $_SERVER = [
+        'HTTP_USER_AGENT' => $userAgent,
+        'REMOTE_ADDR' => '203.0.113.9',
+        'REQUEST_URI' => '/premium/article',
+        'REQUEST_METHOD' => 'GET',
+    ];
+    if ($browserHeaders) {
+        $_SERVER += ['HTTP_ACCEPT' => '*/*', 'HTTP_ACCEPT_ENCODING' => 'gzip', 'HTTP_ACCEPT_LANGUAGE' => 'en'];
+    }
+    return (new \WebDecoy\BotDetector($options))->analyze();
+}
+
+const AI_POLICY_GPTBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)';
+const AI_POLICY_CLAUDEBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)';
+const AI_POLICY_GOOGLEBOT = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)';
+
+echo "\nAI crawler policy: the block setting blocks\n";
+
+$t('with the setting on, an AI crawler is blocked at the default threshold', function () use ($true, $same) {
+    foreach ([AI_POLICY_GPTBOT, AI_POLICY_CLAUDEBOT] as $ua) {
+        $result = ai_policy_analyze($ua, ['block_ai_crawlers' => true]);
+        $true($result->shouldBlock(AI_POLICY_DEFAULT_THRESHOLD), 'served despite the setting: ' . $ua);
+        $true(in_array('ai_crawler_blocked', $result->getFlags(), true), 'the reason must be visible in the log');
+        $same(false, $result->isGoodBot());
+    }
+});
+
+$t('it clears even the strictest threshold a site can set', function () use ($true) {
+    $true(ai_policy_analyze(AI_POLICY_GPTBOT, ['block_ai_crawlers' => true])->shouldBlock(100));
+});
+
+$t('with the setting off, an AI crawler is still a good bot', function () use ($same) {
+    $result = ai_policy_analyze(AI_POLICY_GPTBOT, ['block_ai_crawlers' => false]);
+    $same(0, $result->getScore());
+    $same(true, $result->isGoodBot());
+});
+
+$t('the setting does not touch search engines', function () use ($same) {
+    $result = ai_policy_analyze(AI_POLICY_GOOGLEBOT, ['block_ai_crawlers' => true, 'verify_bot_ips' => false]);
+    $same(false, $result->shouldBlock(AI_POLICY_DEFAULT_THRESHOLD));
+    $same(false, in_array('ai_crawler_blocked', $result->getFlags(), true));
+});
+
+$t('a crawler on the custom allowlist is not blocked by the category setting', function () use ($same) {
+    $result = ai_policy_analyze(AI_POLICY_GPTBOT, ['block_ai_crawlers' => true, 'custom_allowlist' => ['GPTBot']]);
+    $same(false, $result->shouldBlock(AI_POLICY_DEFAULT_THRESHOLD));
+    $same(true, $result->isGoodBot());
+});