Skip to content

chore(bots): regenerate registry (182 agents) for app#1275 #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cport1 merged 2 commits into from
Sep 23, 2026
Merged

chore(bots): regenerate registry (182 agents) for app#1275 #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
53617ad
chore(bots): regenerate registry with seven plugin-only agents (182)
cport1 Sep 23, 2026
233ebaa
chore(bots): ChatGPT-User is ai_assistant, OAI-SearchBot ai_search_cr…
cport1 Sep 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/webdecoy/src/bots/bots.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ describe('matchUserAgent', () => {

describe('registry integrity', () => {
it('carries the full generated table', () => {
expect(BOT_REGISTRY.length).toBe(175);
expect(BOT_REGISTRY.length).toBe(182);
expect(BOT_CATEGORIES).toContain('training_crawler');
expect(BOT_CATEGORIES).toContain('search_crawler');
});
Expand Down
108 changes: 102 additions & 6 deletions packages/webdecoy/src/bots/parity-vectors.generated.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,25 +362,25 @@
{
"userAgent": "chatgpt-user",
"id": "chatgpt-user",
"category": "training_crawler",
"category": "ai_assistant",
"matched": true
},
{
"userAgent": "Mozilla/5.0 (compatible; chatgpt-user/1.0; +http://example.com/bot)",
"id": "chatgpt-user",
"category": "training_crawler",
"category": "ai_assistant",
"matched": true
},
{
"userAgent": "chatgpt",
"id": "chatgpt-user",
"category": "training_crawler",
"category": "ai_assistant",
"matched": true
},
{
"userAgent": "Mozilla/5.0 (compatible; chatgpt/1.0; +http://example.com/bot)",
"id": "chatgpt-user",
"category": "training_crawler",
"category": "ai_assistant",
"matched": true
},
{
Expand All @@ -395,6 +395,18 @@
"category": "ai_agent",
"matched": true
},
{
"userAgent": "claude-web",
"id": "claude-web",
"category": "training_crawler",
"matched": true
},
{
"userAgent": "Mozilla/5.0 (compatible; claude-web/1.0; +http://example.com/bot)",
"id": "claude-web",
"category": "training_crawler",
"matched": true
},
{
"userAgent": "claudebot",
"id": "claudebot",
Expand Down Expand Up @@ -671,6 +683,18 @@
"category": "training_crawler",
"matched": true
},
{
"userAgent": "facebot",
"id": "facebot",
"category": "fetcher",
"matched": true
},
{
"userAgent": "Mozilla/5.0 (compatible; facebot/1.0; +http://example.com/bot)",
"id": "facebot",
"category": "fetcher",
"matched": true
},
{
"userAgent": "faraday",
"id": "faraday",
Expand All @@ -695,6 +719,18 @@
"category": "feed_reader",
"matched": true
},
{
"userAgent": "feedfetcher",
"id": "feedfetcher-google",
"category": "feed_reader",
"matched": true
},
{
"userAgent": "Mozilla/5.0 (compatible; feedfetcher/1.0; +http://example.com/bot)",
"id": "feedfetcher-google",
"category": "feed_reader",
"matched": true
},
{
"userAgent": "feedly",
"id": "feedly",
Expand Down Expand Up @@ -755,6 +791,18 @@
"category": "training_crawler",
"matched": true
},
{
"userAgent": "gtmetrix",
"id": "gtmetrix",
"category": "monitoring",
"matched": true
},
{
"userAgent": "Mozilla/5.0 (compatible; gtmetrix/1.0; +http://example.com/bot)",
"id": "gtmetrix",
"category": "monitoring",
"matched": true
},
{
"userAgent": "gemini",
"id": "gemini",
Expand Down Expand Up @@ -815,6 +863,18 @@
"category": "training_crawler",
"matched": true
},
{
"userAgent": "google-inspectiontool",
"id": "google-inspectiontool",
"category": "search_crawler",
"matched": true
},
{
"userAgent": "Mozilla/5.0 (compatible; google-inspectiontool/1.0; +http://example.com/bot)",
"id": "google-inspectiontool",
"category": "search_crawler",
"matched": true
},
{
"userAgent": "googleagent-mariner",
"id": "googleagent-mariner",
Expand Down Expand Up @@ -1067,6 +1127,30 @@
"category": "fetcher",
"matched": true
},
{
"userAgent": "chrome-lighthouse",
"id": "chrome-lighthouse",
"category": "monitoring",
"matched": true
},
{
"userAgent": "Mozilla/5.0 (compatible; chrome-lighthouse/1.0; +http://example.com/bot)",
"id": "chrome-lighthouse",
"category": "monitoring",
"matched": true
},
{
"userAgent": "google page speed",
"id": "chrome-lighthouse",
"category": "monitoring",
"matched": true
},
{
"userAgent": "Mozilla/5.0 (compatible; google page speed/1.0; +http://example.com/bot)",
"id": "chrome-lighthouse",
"category": "monitoring",
"matched": true
},
{
"userAgent": "linkedinbot",
"id": "linkedinbot",
Expand Down Expand Up @@ -1103,6 +1187,18 @@
"category": "seo_crawler",
"matched": true
},
{
"userAgent": "msnbot",
"id": "msnbot",
"category": "search_crawler",
"matched": true
},
{
"userAgent": "Mozilla/5.0 (compatible; msnbot/1.0; +http://example.com/bot)",
"id": "msnbot",
"category": "search_crawler",
"matched": true
},
{
"userAgent": "mechanize",
"id": "mechanize",
Expand Down Expand Up @@ -1334,13 +1430,13 @@
{
"userAgent": "oai-searchbot",
"id": "oai-searchbot",
"category": "training_crawler",
"category": "ai_search_crawler",
"matched": true
},
{
"userAgent": "Mozilla/5.0 (compatible; oai-searchbot/1.0; +http://example.com/bot)",
"id": "oai-searchbot",
"category": "training_crawler",
"category": "ai_search_crawler",
"matched": true
},
{
Expand Down
13 changes: 10 additions & 3 deletions packages/webdecoy/src/bots/registry.generated.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
* agents share User-Agent substrings, so sorting this array changes how real
* traffic is classified.
*
* 175 agents across 15 categories.
* 182 agents across 15 categories.
*/

/**
Expand Down Expand Up @@ -77,10 +77,9 @@ export const BOT_CATEGORIES: readonly BotCategory[] = [
export const BOT_REGISTRY: readonly BotAgent[] = [
{ id: "reflectionbot", name: "Reflectionbot", category: "training_crawler", organization: "Reflection AI", baseScore: 70, respectsRobots: true, uaPatterns: ["reflectionbot"] },
{ id: "gptbot", name: "GPTBot", category: "training_crawler", organization: "OpenAI", baseScore: 85, respectsRobots: true, uaPatterns: ["gptbot"] },
{ id: "chatgpt-user", name: "ChatGPT-User", category: "training_crawler", organization: "OpenAI", baseScore: 85, respectsRobots: true, uaPatterns: ["chatgpt-user", "chatgpt"] },
{ id: "oai-searchbot", name: "OAI-SearchBot", category: "training_crawler", organization: "OpenAI", baseScore: 80, respectsRobots: true, uaPatterns: ["oai-searchbot"] },
{ id: "claudebot", name: "ClaudeBot", category: "training_crawler", organization: "Anthropic", baseScore: 85, respectsRobots: true, uaPatterns: ["claudebot"] },
{ id: "anthropic", name: "Anthropic", category: "training_crawler", organization: "Anthropic", baseScore: 85, respectsRobots: true, uaPatterns: ["anthropic"] },
{ id: "claude-web", name: "Claude-Web", category: "training_crawler", organization: "Anthropic", baseScore: 85, respectsRobots: true, uaPatterns: ["claude-web"] },
{ id: "ccbot", name: "CCBot", category: "training_crawler", organization: "Common Crawl", baseScore: 80, respectsRobots: true, uaPatterns: ["ccbot"] },
{ id: "google-extended", name: "Google-Extended", category: "training_crawler", organization: "Google", baseScore: 80, respectsRobots: true, uaPatterns: ["google-extended"] },
{ id: "bytespider", name: "ByteSpider", category: "training_crawler", organization: "ByteDance", baseScore: 75, respectsRobots: false, uaPatterns: ["bytespider"] },
Expand Down Expand Up @@ -108,6 +107,7 @@ export const BOT_REGISTRY: readonly BotAgent[] = [
{ id: "sofyabot", name: "SofyaBot", category: "ai_search_crawler", organization: "Sofya", baseScore: 65, respectsRobots: true, uaPatterns: ["sofyabot"] },
{ id: "xai-searchbot", name: "xAI-SearchBot", category: "ai_search_crawler", organization: "xAI", baseScore: 70, respectsRobots: true, uaPatterns: ["xai-searchbot"] },
{ id: "linkupbot", name: "LinkupBot", category: "ai_search_crawler", organization: "Linkup", baseScore: 65, respectsRobots: true, uaPatterns: ["linkupbot"] },
{ id: "oai-searchbot", name: "OAI-SearchBot", category: "ai_search_crawler", organization: "OpenAI", baseScore: 80, respectsRobots: true, uaPatterns: ["oai-searchbot"] },
{ id: "searchgpt", name: "SearchGPT", category: "ai_search_crawler", organization: "OpenAI", baseScore: 75, respectsRobots: true, uaPatterns: ["searchgpt"] },
{ id: "phind", name: "Phind", category: "ai_search_crawler", organization: "Phind", baseScore: 70, respectsRobots: true, uaPatterns: ["phind"] },
{ id: "kagi", name: "Kagi", category: "ai_search_crawler", organization: "Kagi", baseScore: 65, respectsRobots: true, uaPatterns: ["kagi"] },
Expand All @@ -126,7 +126,9 @@ export const BOT_REGISTRY: readonly BotAgent[] = [
{ id: "multion", name: "MultiOn", category: "ai_agent", organization: "MultiOn", baseScore: 70, respectsRobots: false, uaPatterns: ["multion"] },
{ id: "iboubot", name: "IbouBot", category: "search_crawler", organization: "Ibou", baseScore: 30, respectsRobots: true, uaPatterns: ["iboubot"] },
{ id: "googlebot", name: "Googlebot", category: "search_crawler", organization: "Google", baseScore: 30, respectsRobots: true, uaPatterns: ["googlebot"] },
{ id: "google-inspectiontool", name: "Google-InspectionTool", category: "search_crawler", organization: "Google", baseScore: 30, respectsRobots: true, uaPatterns: ["google-inspectiontool"] },
{ id: "bingbot", name: "Bingbot", category: "search_crawler", organization: "Microsoft", baseScore: 30, respectsRobots: true, uaPatterns: ["bingbot"] },
{ id: "msnbot", name: "MSNBot", category: "search_crawler", organization: "Microsoft", baseScore: 30, respectsRobots: true, uaPatterns: ["msnbot"] },
{ id: "yandexbot", name: "YandexBot", category: "search_crawler", organization: "Yandex", baseScore: 35, respectsRobots: true, uaPatterns: ["yandexbot"] },
{ id: "baiduspider", name: "Baiduspider", category: "search_crawler", organization: "Baidu", baseScore: 40, respectsRobots: true, uaPatterns: ["baiduspider"] },
{ id: "duckduckbot", name: "DuckDuckBot", category: "search_crawler", organization: "DuckDuckGo", baseScore: 30, respectsRobots: true, uaPatterns: ["duckduckbot"] },
Expand Down Expand Up @@ -201,6 +203,7 @@ export const BOT_REGISTRY: readonly BotAgent[] = [
{ id: "heritrix", name: "Heritrix", category: "archiver", organization: "Internet Archive", baseScore: 30, respectsRobots: true, uaPatterns: ["heritrix"] },
{ id: "brozzler", name: "Brozzler", category: "archiver", organization: "Internet Archive", baseScore: 30, respectsRobots: true, uaPatterns: ["brozzler"] },
{ id: "facebookexternalhit", name: "Facebook", category: "fetcher", organization: "Meta", baseScore: 25, respectsRobots: true, uaPatterns: ["facebookexternalhit"] },
{ id: "facebot", name: "Facebot", category: "fetcher", organization: "Meta", baseScore: 25, respectsRobots: true, uaPatterns: ["facebot"] },
{ id: "twitterbot", name: "Twitterbot", category: "fetcher", organization: "X Corp", baseScore: 25, respectsRobots: true, uaPatterns: ["twitterbot"] },
{ id: "linkedinbot", name: "LinkedInBot", category: "fetcher", organization: "LinkedIn", baseScore: 25, respectsRobots: true, uaPatterns: ["linkedinbot"] },
{ id: "slackbot", name: "Slackbot", category: "fetcher", organization: "Slack", baseScore: 25, respectsRobots: true, uaPatterns: ["slackbot"] },
Expand Down Expand Up @@ -228,7 +231,10 @@ export const BOT_REGISTRY: readonly BotAgent[] = [
{ id: "freshping", name: "Freshping", category: "monitoring", organization: "Freshworks", baseScore: 20, respectsRobots: true, uaPatterns: ["freshping"] },
{ id: "hetrixtools", name: "HetrixTools", category: "monitoring", organization: "HetrixTools", baseScore: 20, respectsRobots: true, uaPatterns: ["hetrixtools"] },
{ id: "nodeping", name: "NodePing", category: "monitoring", organization: "NodePing", baseScore: 20, respectsRobots: true, uaPatterns: ["nodeping"] },
{ id: "gtmetrix", name: "GTmetrix", category: "monitoring", organization: "GTmetrix", baseScore: 20, respectsRobots: true, uaPatterns: ["gtmetrix"] },
{ id: "chrome-lighthouse", name: "Lighthouse", category: "monitoring", organization: "Google", baseScore: 20, respectsRobots: true, uaPatterns: ["chrome-lighthouse", "google page speed"] },
{ id: "feedly", name: "Feedly", category: "feed_reader", organization: "Feedly", baseScore: 25, respectsRobots: true, uaPatterns: ["feedly"] },
{ id: "feedfetcher-google", name: "Feedfetcher-Google", category: "feed_reader", organization: "Google", baseScore: 25, respectsRobots: true, uaPatterns: ["feedfetcher"] },
{ id: "newsblur", name: "NewsBlur", category: "feed_reader", organization: "NewsBlur", baseScore: 25, respectsRobots: true, uaPatterns: ["newsblur"] },
{ id: "inoreader", name: "Inoreader", category: "feed_reader", organization: "Inoreader", baseScore: 25, respectsRobots: true, uaPatterns: ["inoreader"] },
{ id: "theoldreader", name: "The Old Reader", category: "feed_reader", organization: "The Old Reader", baseScore: 25, respectsRobots: true, uaPatterns: ["theoldreader"] },
Expand All @@ -246,6 +252,7 @@ export const BOT_REGISTRY: readonly BotAgent[] = [
{ id: "apify", name: "Apify", category: "commercial_scraper", organization: "Apify", baseScore: 65, respectsRobots: true, uaPatterns: ["apify"] },
{ id: "crawlbase", name: "Crawlbase", category: "commercial_scraper", organization: "Crawlbase", baseScore: 65, respectsRobots: true, uaPatterns: ["crawlbase"] },
{ id: "webscrapingapi", name: "WebScrapingAPI", category: "commercial_scraper", organization: "WebScrapingAPI", baseScore: 65, respectsRobots: true, uaPatterns: ["webscrapingapi"] },
{ id: "chatgpt-user", name: "ChatGPT-User", category: "ai_assistant", organization: "OpenAI", baseScore: 85, respectsRobots: true, uaPatterns: ["chatgpt-user", "chatgpt"] },
{ id: "gemini-deep-research", name: "Gemini-Deep-Research", category: "ai_assistant", organization: "Google", baseScore: 65, respectsRobots: true, uaPatterns: ["gemini-deep-research"] },
{ id: "copilot", name: "Microsoft Copilot", category: "ai_assistant", organization: "Microsoft", baseScore: 65, respectsRobots: true, uaPatterns: ["copilot"] },
{ id: "cortana", name: "Cortana", category: "ai_assistant", organization: "Microsoft", baseScore: 60, respectsRobots: true, uaPatterns: ["cortana"] },
Expand Down
Loading