diff --git a/sdk/src/BotDetector.php b/sdk/src/BotDetector.php index 424c4aa..2e6af21 100644 --- a/sdk/src/BotDetector.php +++ b/sdk/src/BotDetector.php @@ -28,6 +28,9 @@ class BotDetector private const SCORE_RATE_EXCEEDED = 25; private const SCORE_HONEYPOT = 60; private const SCORE_FAKE_BOT = 80; // Claiming to be a bot but IP doesn't verify + // The owner's own instruction to block. The top of the scale, so it clears + // min_score_to_block whatever the site has set it to. + private const SCORE_POLICY_DENIED = 100; // Path-based scoring (MITRE ATT&CK aligned) private const SCORE_PATH_CONFIG_FILE = 30; // TA0006 Credential Access - config files @@ -194,6 +197,29 @@ public function analyze(?array $signals = null, ?string $clientIP = null): Detec $result->setBotName($botInfo['name']); $result->setBotCategory($botInfo['category']); + // A crawler the owner has chosen to block is refused here, as an + // explicit decision, before any heuristic scoring. + // + // The setting used to work by withdrawing the crawler's good-bot + // pass and letting scoring decide. Scoring does not add points for a + // recognised bot, so a well-behaved AI crawler could finish below + // the block threshold. An owner's instruction should not depend on + // how the request happens to score. + // + // The User-Agent claim is enough to act on: refusing a request + // because it says it is GPTBot affects nobody who is not claiming + // to be GPTBot. + if ($this->deniedByPolicy($botInfo)) { + $result->setIsGoodBot(false); + $result->setScore(self::SCORE_POLICY_DENIED); + $result->setScoreBreakdown(['ai_crawler_blocked' => self::SCORE_POLICY_DENIED]); + $result->setConfidence(1.0); + $result->addFlag('ai_crawler_blocked'); + $result->addMetadata('bot_info', $botInfo); + $result->addMetadata('denied_by', 'block_ai_crawlers'); + return $result; + } + // Check if this bot should be allowed if ($this->shouldAllowBot($botInfo)) { // Verify the bot's IP if verification is enabled @@ -516,6 +542,26 @@ public function identifyBot(string $userAgent): ?array return $this->goodBotList->identify($userAgent); } + /** + * Whether the owner's settings say to block this recognised bot outright. + * + * Only the AI crawler category has a "block" setting. Turning OFF "allow + * search engines" or "allow social bots" withdraws a free pass and lets the + * heuristics judge the request; it is not an instruction to block. The + * custom allowlist wins, as it does in shouldAllowBot(). + * + * @param array $botInfo Bot information + * @return bool + */ + private function deniedByPolicy(array $botInfo): bool + { + if (in_array($botInfo['name'] ?? '', $this->options['custom_allowlist'], true)) { + return false; + } + return ($botInfo['category'] ?? '') === GoodBotList::CATEGORY_AI_CRAWLER + && !empty($this->options['block_ai_crawlers']); + } + /** * Check if a bot should be allowed based on policies * diff --git a/tests/AiCrawlerPolicyTest.php b/tests/AiCrawlerPolicyTest.php new file mode 100644 index 0000000..6c0e571 --- /dev/null +++ b/tests/AiCrawlerPolicyTest.php @@ -0,0 +1,88 @@ + $userAgent, + 'REMOTE_ADDR' => '203.0.113.9', + 'REQUEST_URI' => '/premium/article', + 'REQUEST_METHOD' => 'GET', + ]; + if ($browserHeaders) { + $_SERVER += ['HTTP_ACCEPT' => '*/*', 'HTTP_ACCEPT_ENCODING' => 'gzip', 'HTTP_ACCEPT_LANGUAGE' => 'en']; + } + return (new \WebDecoy\BotDetector($options))->analyze(); +} + +const AI_POLICY_GPTBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)'; +const AI_POLICY_CLAUDEBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)'; +const AI_POLICY_GOOGLEBOT = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'; + +echo "\nAI crawler policy: the block setting blocks\n"; + +$t('with the setting on, an AI crawler is blocked at the default threshold', function () use ($true, $same) { + foreach ([AI_POLICY_GPTBOT, AI_POLICY_CLAUDEBOT] as $ua) { + $result = ai_policy_analyze($ua, ['block_ai_crawlers' => true]); + $true($result->shouldBlock(AI_POLICY_DEFAULT_THRESHOLD), 'served despite the setting: ' . $ua); + $true(in_array('ai_crawler_blocked', $result->getFlags(), true), 'the reason must be visible in the log'); + $same(false, $result->isGoodBot()); + } +}); + +$t('it clears even the strictest threshold a site can set', function () use ($true) { + $true(ai_policy_analyze(AI_POLICY_GPTBOT, ['block_ai_crawlers' => true])->shouldBlock(100)); +}); + +$t('with the setting off, an AI crawler is still a good bot', function () use ($same) { + $result = ai_policy_analyze(AI_POLICY_GPTBOT, ['block_ai_crawlers' => false]); + $same(0, $result->getScore()); + $same(true, $result->isGoodBot()); +}); + +$t('the setting does not touch search engines', function () use ($same) { + $result = ai_policy_analyze(AI_POLICY_GOOGLEBOT, ['block_ai_crawlers' => true, 'verify_bot_ips' => false]); + $same(false, $result->shouldBlock(AI_POLICY_DEFAULT_THRESHOLD)); + $same(false, in_array('ai_crawler_blocked', $result->getFlags(), true)); +}); + +$t('a crawler on the custom allowlist is not blocked by the category setting', function () use ($same) { + $result = ai_policy_analyze(AI_POLICY_GPTBOT, ['block_ai_crawlers' => true, 'custom_allowlist' => ['GPTBot']]); + $same(false, $result->shouldBlock(AI_POLICY_DEFAULT_THRESHOLD)); + $same(true, $result->isGoodBot()); +});