diff --git a/sdk/src/registry/agents.generated.php b/sdk/src/registry/agents.generated.php index 0456cf7..3b28205 100644 --- a/sdk/src/registry/agents.generated.php +++ b/sdk/src/registry/agents.generated.php @@ -14,7 +14,7 @@ return [ 'schema' => 2, - 'agent_count' => 182, + 'agent_count' => 186, 'categories' => ['training_crawler', 'ai_search_crawler', 'ai_agent', 'ai_assistant', 'search_crawler', 'seo_crawler', 'security_scanner', 'generic_scraper', 'archiver', 'fetcher', 'monitoring', 'headless_browser', 'feed_reader', 'commercial_scraper', 'none'], 'agents' => [ [ @@ -62,7 +62,7 @@ 'category' => 'training_crawler', 'behavior' => 'training', 'organization' => 'Anthropic', - 'patterns' => ['anthropic'], + 'patterns' => ['anthropic-ai'], 'robots_name' => 'anthropic', 'website' => 'https://www.anthropic.com', 'base_score' => 85, @@ -173,19 +173,6 @@ 'respects_robots' => true, 'http_client' => false, ], - [ - 'id' => 'perplexitybot', - 'name' => 'PerplexityBot', - 'category' => 'training_crawler', - 'behavior' => 'training', - 'organization' => 'Perplexity AI', - 'patterns' => ['perplexitybot'], - 'robots_name' => 'PerplexityBot', - 'website' => 'https://perplexity.ai', - 'base_score' => 80, - 'respects_robots' => true, - 'http_client' => false, - ], [ 'id' => 'applebot-extended', 'name' => 'Applebot-Extended', @@ -218,7 +205,7 @@ 'category' => 'training_crawler', 'behavior' => 'training', 'organization' => 'Mistral AI', - 'patterns' => ['mistral'], + 'patterns' => ['mistralbot'], 'robots_name' => 'MistralBot', 'website' => 'https://mistral.ai', 'base_score' => 80, @@ -433,6 +420,32 @@ 'respects_robots' => true, 'http_client' => false, ], + [ + 'id' => 'perplexitybot', + 'name' => 'PerplexityBot', + 'category' => 'ai_search_crawler', + 'behavior' => 'search', + 'organization' => 'Perplexity AI', + 'patterns' => ['perplexitybot'], + 'robots_name' => 'PerplexityBot', + 'website' => 'https://docs.perplexity.ai/guides/bots', + 'base_score' => 80, + 'respects_robots' => true, + 'http_client' => false, + ], + [ + 'id' => 'claude-searchbot', + 'name' => 'Claude-SearchBot', + 'category' => 'ai_search_crawler', + 'behavior' => 'search', + 'organization' => 'Anthropic', + 'patterns' => ['claude-searchbot'], + 'robots_name' => 'Claude-SearchBot', + 'website' => 'https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler', + 'base_score' => 75, + 'respects_robots' => true, + 'http_client' => false, + ], [ 'id' => 'oai-searchbot', 'name' => 'OAI-SearchBot', @@ -2324,13 +2337,52 @@ 'category' => 'ai_assistant', 'behavior' => 'agent', 'organization' => 'OpenAI', - 'patterns' => ['chatgpt-user', 'chatgpt'], + 'patterns' => ['chatgpt-user'], 'robots_name' => 'ChatGPT-User', 'website' => 'https://openai.com', 'base_score' => 85, 'respects_robots' => true, 'http_client' => false, ], + [ + 'id' => 'perplexity-user', + 'name' => 'Perplexity-User', + 'category' => 'ai_assistant', + 'behavior' => 'agent', + 'organization' => 'Perplexity AI', + 'patterns' => ['perplexity-user'], + 'robots_name' => 'Perplexity-User', + 'website' => 'https://docs.perplexity.ai/guides/bots', + 'base_score' => 75, + 'respects_robots' => false, + 'http_client' => false, + ], + [ + 'id' => 'mistralai-user', + 'name' => 'MistralAI-User', + 'category' => 'ai_assistant', + 'behavior' => 'agent', + 'organization' => 'Mistral AI', + 'patterns' => ['mistralai-user'], + 'robots_name' => 'MistralAI-User', + 'website' => 'https://docs.mistral.ai/robots', + 'base_score' => 75, + 'respects_robots' => true, + 'http_client' => false, + ], + [ + 'id' => 'meta-externalfetcher', + 'name' => 'Meta-ExternalFetcher', + 'category' => 'ai_assistant', + 'behavior' => 'agent', + 'organization' => 'Meta', + 'patterns' => ['meta-externalfetcher'], + 'robots_name' => 'Meta-ExternalFetcher', + 'website' => 'https://developers.facebook.com/docs/sharing/webmasters/web-crawlers', + 'base_score' => 70, + 'respects_robots' => false, + 'http_client' => false, + ], [ 'id' => 'gemini-deep-research', 'name' => 'Gemini-Deep-Research', diff --git a/sdk/src/registry/parity-vectors.generated.json b/sdk/src/registry/parity-vectors.generated.json index 8332f95..1182ef5 100644 --- a/sdk/src/registry/parity-vectors.generated.json +++ b/sdk/src/registry/parity-vectors.generated.json @@ -84,13 +84,13 @@ "matched": true }, { - "userAgent": "anthropic", + "userAgent": "anthropic-ai", "id": "anthropic", "category": "training_crawler", "matched": true }, { - "userAgent": "Mozilla/5.0 (compatible; anthropic/1.0; +http://example.com/bot)", + "userAgent": "Mozilla/5.0 (compatible; anthropic-ai/1.0; +http://example.com/bot)", "id": "anthropic", "category": "training_crawler", "matched": true @@ -372,15 +372,15 @@ "matched": true }, { - "userAgent": "chatgpt", - "id": "chatgpt-user", - "category": "ai_assistant", + "userAgent": "claude-searchbot", + "id": "claude-searchbot", + "category": "ai_search_crawler", "matched": true }, { - "userAgent": "Mozilla/5.0 (compatible; chatgpt/1.0; +http://example.com/bot)", - "id": "chatgpt-user", - "category": "ai_assistant", + "userAgent": "Mozilla/5.0 (compatible; claude-searchbot/1.0; +http://example.com/bot)", + "id": "claude-searchbot", + "category": "ai_search_crawler", "matched": true }, { @@ -1223,6 +1223,18 @@ "category": "training_crawler", "matched": true }, + { + "userAgent": "meta-externalfetcher", + "id": "meta-externalfetcher", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; meta-externalfetcher/1.0; +http://example.com/bot)", + "id": "meta-externalfetcher", + "category": "ai_assistant", + "matched": true + }, { "userAgent": "metaphor", "id": "metaphor", @@ -1260,13 +1272,25 @@ "matched": true }, { - "userAgent": "mistral", + "userAgent": "mistralai-user", + "id": "mistralai-user", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; mistralai-user/1.0; +http://example.com/bot)", + "id": "mistralai-user", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "mistralbot", "id": "mistral", "category": "training_crawler", "matched": true }, { - "userAgent": "Mozilla/5.0 (compatible; mistral/1.0; +http://example.com/bot)", + "userAgent": "Mozilla/5.0 (compatible; mistralbot/1.0; +http://example.com/bot)", "id": "mistral", "category": "training_crawler", "matched": true @@ -1523,16 +1547,28 @@ "category": "generic_scraper", "matched": true }, + { + "userAgent": "perplexity-user", + "id": "perplexity-user", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; perplexity-user/1.0; +http://example.com/bot)", + "id": "perplexity-user", + "category": "ai_assistant", + "matched": true + }, { "userAgent": "perplexitybot", "id": "perplexitybot", - "category": "training_crawler", + "category": "ai_search_crawler", "matched": true }, { "userAgent": "Mozilla/5.0 (compatible; perplexitybot/1.0; +http://example.com/bot)", "id": "perplexitybot", - "category": "training_crawler", + "category": "ai_search_crawler", "matched": true }, {