From 69d1c8f77b29681987239cc33a1ab041faa1cfad Mon Sep 17 00:00:00 2001
From: Hugo Gruson <git@hugogruson.fr>
Date: Wed, 25 Mar 2026 16:06:51 +0100
Subject: [PATCH 1/4] Decrease cpu request for git index update

---
 kubernetes/cronjob-index.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kubernetes/cronjob-index.yaml b/kubernetes/cronjob-index.yaml
index b378097..a90c4fb 100644
--- a/kubernetes/cronjob-index.yaml
+++ b/kubernetes/cronjob-index.yaml
@@ -26,7 +26,7 @@ spec:
                 cpu: 1000m
                 memory: 2Gi
               requests:
-                cpu: 700m
+                cpu: 100m
                 memory: 1Gi
             imagePullPolicy: Always
             image: grimbough/code.bioc-mirror-updater:0.1.6

From 4b74082af72c68292a995fb765386055a64a892d Mon Sep 17 00:00:00 2001
From: Hugo Gruson <git@hugogruson.fr>
Date: Wed, 25 Mar 2026 16:17:39 +0100
Subject: [PATCH 2/4] Make sure cronjobs don't get stuck or spawn thousands of
 times

---
 kubernetes/cronjob-index.yaml     | 3 +++
 kubernetes/cronjob-logrotate.yaml | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/kubernetes/cronjob-index.yaml b/kubernetes/cronjob-index.yaml
index a90c4fb..038c545 100644
--- a/kubernetes/cronjob-index.yaml
+++ b/kubernetes/cronjob-index.yaml
@@ -5,9 +5,12 @@ metadata:
   namespace: bioc-code-explorer
 spec:
   schedule: "*/15 * * * *"
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 60
   jobTemplate:
     spec:
       ttlSecondsAfterFinished: 100
+      activeDeadlineSeconds: 780
       template:
         spec:
           securityContext:
diff --git a/kubernetes/cronjob-logrotate.yaml b/kubernetes/cronjob-logrotate.yaml
index 3e649b8..fd72567 100644
--- a/kubernetes/cronjob-logrotate.yaml
+++ b/kubernetes/cronjob-logrotate.yaml
@@ -5,9 +5,12 @@ metadata:
   namespace: bioc-code-explorer
 spec:
   schedule: "0 0 * * *"  # Daily at midnight
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 300
   jobTemplate:
     spec:
       ttlSecondsAfterFinished: 600
+      activeDeadlineSeconds: 3600
       template:
         spec:
           securityContext:

From 5e2e1e359885f86e72295254a36509e43aaf4514 Mon Sep 17 00:00:00 2001
From: Hugo Gruson <git@hugogruson.fr>
Date: Wed, 25 Mar 2026 17:36:59 +0100
Subject: [PATCH 3/4] Exclude more LLM bots

---
 mirror-updater/utils.R | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/mirror-updater/utils.R b/mirror-updater/utils.R
index c5c2b69..fceaf20 100644
--- a/mirror-updater/utils.R
+++ b/mirror-updater/utils.R
@@ -158,7 +158,9 @@ write_robots_txt <- function(pkgs, output_file = "/var/shared/robots.txt") {
                        "GPTBot", "AhrefsBot", "PetalBot", "ClaudeBot",
                        "SemrushBot", "meta-externalagent", "SEOkicks",
                        "AwarioRssBot", "AwarioSmartBot", "ImagesiftBot",
-                       "AliyunSecBot", "Aliyun", "Bytespider")
+                       "AliyunSecBot", "Aliyun", "Bytespider", "YandexBot",
+                       "AcademicBotRTU", "Claude-SearchBot", "dotbot", 
+                       "ChatGPT-User")
 
     for(bot in excluded_bots) {
         writeLines(paste0("User-agent: ", bot), con = con)

From efc7f7310d388df9e01707b6ec67214d231d2754 Mon Sep 17 00:00:00 2001
From: Hugo Gruson <git@hugogruson.fr>
Date: Wed, 25 Mar 2026 17:45:37 +0100
Subject: [PATCH 4/4] Switch to a "disallow all but..." approach for robots.txt

---
 mirror-updater/utils.R | 25 ++++++-------------------
 1 file changed, 6 insertions(+), 19 deletions(-)

diff --git a/mirror-updater/utils.R b/mirror-updater/utils.R
index fceaf20..22ed1da 100644
--- a/mirror-updater/utils.R
+++ b/mirror-updater/utils.R
@@ -168,25 +168,12 @@ write_robots_txt <- function(pkgs, output_file = "/var/shared/robots.txt") {
     }
 
     writeLines("User-agent: *", con = con)
-
-    writeLines(paste0("Allow: /browse/*/"), con = con)
-    writeLines(paste0("Disallow: /browse/*/*/"), con = con)
-    writeLines(paste0("Disallow: /browse/*/treegraph"), con = con)
-    writeLines(paste0("Disallow: /browse/themes"), con = con)
-    writeLines(paste0("Disallow: *tree/"), con = con)
-    writeLines(paste0("Disallow: *blob/"), con = con)
-    writeLines(paste0("Disallow: *commit"), con = con)
-    writeLines(paste0("Disallow: *stats/"), con = con)
-    writeLines(paste0("Disallow: *network/"), con = con)
-    writeLines(paste0("Disallow: *RELEASE_"), con = con)
-    writeLines(paste0("Disallow: *raw/"), con = con)
-    writeLines(paste0("Disallow: *logpatch/"), con = con)
-    writeLines(paste0("Disallow: *zipball/"), con = con)
-    writeLines(paste0("Disallow: *tarball/"), con = con)
-    writeLines(paste0("Disallow: *blame/"), con = con)
-    writeLines(paste0("Disallow: *rss/"), con = con)
-
-    writeLines(paste0("Disallow: /search/search?q"), con = con)
+    writeLines("Disallow: /", con = con)
+    writeLines("Allow: /browse/", con = con)
+    writeLines("Allow: /search/", con = con)
+    writeLines("Allow: /index.html", con = con)
+    writeLines("Allow: /about.html", con = con)
+    writeLines(sprintf("Allow: /browse/%s/", basename(pkgs)), con = con)
 
     writeLines("\nSitemap: https://code.bioconductor.org/sitemap.txt", con = con)
 }