From c43b327dedbd5c0174dd9e1a20aba547ca252c9e Mon Sep 17 00:00:00 2001 From: Chris Portscheller Date: Mon, 21 Sep 2026 15:01:46 -0500 Subject: [PATCH] fix: make the Block AI crawlers setting an explicit decision The setting withdrew a recognised AI crawler's good-bot pass and left the outcome to heuristic scoring. Scoring adds no user-agent points for a recognised bot, so a well-behaved AI crawler could finish below the block threshold. A crawler the owner has chosen to block is now refused before scoring, with the flag ai_crawler_blocked so the Detections page says why. The custom allowlist still wins, search engines and social bots are unaffected, and monitor mode suppresses the block like every other enforcement path. Adds behaviour tests for the setting. --- sdk/src/BotDetector.php | 46 ++++++++++++++++++ tests/AiCrawlerPolicyTest.php | 88 +++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 tests/AiCrawlerPolicyTest.php diff --git a/sdk/src/BotDetector.php b/sdk/src/BotDetector.php index 424c4aa..2e6af21 100644 --- a/sdk/src/BotDetector.php +++ b/sdk/src/BotDetector.php @@ -28,6 +28,9 @@ class BotDetector private const SCORE_RATE_EXCEEDED = 25; private const SCORE_HONEYPOT = 60; private const SCORE_FAKE_BOT = 80; // Claiming to be a bot but IP doesn't verify + // The owner's own instruction to block. The top of the scale, so it clears + // min_score_to_block whatever the site has set it to. + private const SCORE_POLICY_DENIED = 100; // Path-based scoring (MITRE ATT&CK aligned) private const SCORE_PATH_CONFIG_FILE = 30; // TA0006 Credential Access - config files @@ -194,6 +197,29 @@ public function analyze(?array $signals = null, ?string $clientIP = null): Detec $result->setBotName($botInfo['name']); $result->setBotCategory($botInfo['category']); + // A crawler the owner has chosen to block is refused here, as an + // explicit decision, before any heuristic scoring. + // + // The setting used to work by withdrawing the crawler's good-bot + // pass and letting scoring decide. Scoring does not add points for a + // recognised bot, so a well-behaved AI crawler could finish below + // the block threshold. An owner's instruction should not depend on + // how the request happens to score. + // + // The User-Agent claim is enough to act on: refusing a request + // because it says it is GPTBot affects nobody who is not claiming + // to be GPTBot. + if ($this->deniedByPolicy($botInfo)) { + $result->setIsGoodBot(false); + $result->setScore(self::SCORE_POLICY_DENIED); + $result->setScoreBreakdown(['ai_crawler_blocked' => self::SCORE_POLICY_DENIED]); + $result->setConfidence(1.0); + $result->addFlag('ai_crawler_blocked'); + $result->addMetadata('bot_info', $botInfo); + $result->addMetadata('denied_by', 'block_ai_crawlers'); + return $result; + } + // Check if this bot should be allowed if ($this->shouldAllowBot($botInfo)) { // Verify the bot's IP if verification is enabled @@ -516,6 +542,26 @@ public function identifyBot(string $userAgent): ?array return $this->goodBotList->identify($userAgent); } + /** + * Whether the owner's settings say to block this recognised bot outright. + * + * Only the AI crawler category has a "block" setting. Turning OFF "allow + * search engines" or "allow social bots" withdraws a free pass and lets the + * heuristics judge the request; it is not an instruction to block. The + * custom allowlist wins, as it does in shouldAllowBot(). + * + * @param array $botInfo Bot information + * @return bool + */ + private function deniedByPolicy(array $botInfo): bool + { + if (in_array($botInfo['name'] ?? '', $this->options['custom_allowlist'], true)) { + return false; + } + return ($botInfo['category'] ?? '') === GoodBotList::CATEGORY_AI_CRAWLER + && !empty($this->options['block_ai_crawlers']); + } + /** * Check if a bot should be allowed based on policies * diff --git a/tests/AiCrawlerPolicyTest.php b/tests/AiCrawlerPolicyTest.php new file mode 100644 index 0000000..6c0e571 --- /dev/null +++ b/tests/AiCrawlerPolicyTest.php @@ -0,0 +1,88 @@ + $userAgent, + 'REMOTE_ADDR' => '203.0.113.9', + 'REQUEST_URI' => '/premium/article', + 'REQUEST_METHOD' => 'GET', + ]; + if ($browserHeaders) { + $_SERVER += ['HTTP_ACCEPT' => '*/*', 'HTTP_ACCEPT_ENCODING' => 'gzip', 'HTTP_ACCEPT_LANGUAGE' => 'en']; + } + return (new \WebDecoy\BotDetector($options))->analyze(); +} + +const AI_POLICY_GPTBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)'; +const AI_POLICY_CLAUDEBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)'; +const AI_POLICY_GOOGLEBOT = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'; + +echo "\nAI crawler policy: the block setting blocks\n"; + +$t('with the setting on, an AI crawler is blocked at the default threshold', function () use ($true, $same) { + foreach ([AI_POLICY_GPTBOT, AI_POLICY_CLAUDEBOT] as $ua) { + $result = ai_policy_analyze($ua, ['block_ai_crawlers' => true]); + $true($result->shouldBlock(AI_POLICY_DEFAULT_THRESHOLD), 'served despite the setting: ' . $ua); + $true(in_array('ai_crawler_blocked', $result->getFlags(), true), 'the reason must be visible in the log'); + $same(false, $result->isGoodBot()); + } +}); + +$t('it clears even the strictest threshold a site can set', function () use ($true) { + $true(ai_policy_analyze(AI_POLICY_GPTBOT, ['block_ai_crawlers' => true])->shouldBlock(100)); +}); + +$t('with the setting off, an AI crawler is still a good bot', function () use ($same) { + $result = ai_policy_analyze(AI_POLICY_GPTBOT, ['block_ai_crawlers' => false]); + $same(0, $result->getScore()); + $same(true, $result->isGoodBot()); +}); + +$t('the setting does not touch search engines', function () use ($same) { + $result = ai_policy_analyze(AI_POLICY_GOOGLEBOT, ['block_ai_crawlers' => true, 'verify_bot_ips' => false]); + $same(false, $result->shouldBlock(AI_POLICY_DEFAULT_THRESHOLD)); + $same(false, in_array('ai_crawler_blocked', $result->getFlags(), true)); +}); + +$t('a crawler on the custom allowlist is not blocked by the category setting', function () use ($same) { + $result = ai_policy_analyze(AI_POLICY_GPTBOT, ['block_ai_crawlers' => true, 'custom_allowlist' => ['GPTBot']]); + $same(false, $result->shouldBlock(AI_POLICY_DEFAULT_THRESHOLD)); + $same(true, $result->isGoodBot()); +});