Serve randomized decoy sites with robots.txt

.github/workflows/integration.yml

@@ -44,7 +44,6 @@ jobs:
 
   two-host-jump:
     name: Debian 12 two-host Jump
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     timeout-minutes: 35
     steps:

docs/en/README.md

@@ -149,6 +149,11 @@ that wait for `407` before sending credentials. Keep it disabled unless tested w
 clients. A route can override the entry setting; `knock` allows selected hostnames to receive the
 normal challenge.
 
+Masked routes serve a static site using GOST's built-in loopback file server. A standard-library
+Python script generates a random heading and abstract SVG once. Repeated `apply` runs keep the
+site; remove `/opt/megaproxy/decoy/index.html` to regenerate it on the next `apply`.
+The page includes `noindex,nofollow`; `/robots.txt` contains `User-agent: *` and `Disallow: /`.
+
 ## HTTPS certificates and chains
 
 Domain certificates are issued with Certbot standalone ACME. An entry certificate contains its

docs/ru/README.md

@@ -155,6 +155,12 @@ Probe resistance подавляет обычный запрос аутентиф
 нужных клиентов. Маршрут может переопределить настройку entry; `knock` разрешает обычный запрос
 аутентификации для выбранных hostname-ов.
 
+При включённом маскировании GOST отдаёт статический сайт через встроенный файловый сервер
+на loopback. Python без дополнительных библиотек один раз создаёт случайный заголовок и
+абстрактную SVG-картинку. Повторный `apply` сохраняет сайт; удаление
+`/opt/megaproxy/decoy/index.html` приводит к новой генерации при следующем `apply`.
+Страница содержит `noindex,nofollow`, а `/robots.txt` — `User-agent: *` и `Disallow: /`.
+
 ## HTTPS-сертификаты и chains
 
 Доменные сертификаты выпускаются Certbot в standalone-режиме. Сертификат entry-сервера содержит его

playbooks/verify.yml

@@ -87,17 +87,16 @@
             ['curl', '--silent', '--show-error', '--fail', '--max-time', '15',
              '--write-out', '\n%{http_code}']
             + (['--insecure'] if megaproxy_services.https.certificate == 'self-signed' else [])
-            + ['https://' ~ item.hostname ~ ':' ~ (item.port | string) ~ '/']
+            + ['https://' ~ item.hostname ~ ':' ~ (megaproxy_services.https.port | string) ~ '/']
           }}
       register: decoy_response
       changed_when: false
       delegate_to: localhost
       become: false
-      loop: "{{ megaproxy_https_public_routes | default([]) }}"
+      loop: "{{ megaproxy_https_public_routes | default([]) | selectattr('probe_resistance_enabled') | list }}"
       when:
         - megaproxy_services.https is defined
         - megaproxy_services.https.enabled | bool
-        - megaproxy_services.https.probe_resistance.enabled | bool
 
     - name: Ensure decoy does not disclose proxy software
       ansible.builtin.assert:
@@ -107,9 +106,33 @@
           - "'proxy-authenticate' not in (item.stdout | lower)"
           - "'gost' not in (item.stdout | lower)"
           - "'megaproxy' not in (item.stdout | lower)"
+          - "'content=\"noindex,nofollow\"' in item.stdout"
       loop: "{{ decoy_response.results | default([]) }}"
       when: megaproxy_services.https is defined and megaproxy_services.https.enabled | bool
 
+    - name: Read robots.txt from masked routes
+      ansible.builtin.command:
+        argv: >-
+          {{
+            ['curl', '--silent', '--show-error', '--fail', '--max-time', '15']
+            + (['--insecure'] if megaproxy_services.https.certificate == 'self-signed' else [])
+            + ['https://' ~ item.hostname ~ ':' ~ (megaproxy_services.https.port | string) ~ '/robots.txt']
+          }}
+      loop: "{{ megaproxy_https_public_routes | default([]) | selectattr('probe_resistance_enabled') | list }}"
+      register: robots_responses
+      changed_when: false
+      delegate_to: localhost
+      become: false
+      when: megaproxy_services.https is defined and megaproxy_services.https.enabled | bool
+
+    - name: Require robots.txt to disallow crawling
+      ansible.builtin.assert:
+        that:
+          - >-
+            item.stdout == 'User-agent: *\nDisallow: /'
+      loop: "{{ robots_responses.results | default([]) }}"
+      when: item is not skipped
+
     - name: Probe CONNECT without credentials
       ansible.builtin.command:
         argv:

roles/admin/handlers/main.yml

@@ -1,7 +1,6 @@
 ---
 - name: Validate and reload administrative ssh policy
   ansible.builtin.command: /usr/sbin/sshd -t
-  changed_when: false
   notify: Reload ssh after administrative policy
 
 - name: Reload ssh after administrative policy

roles/https_proxy/files/generate-decoy.py

@@ -0,0 +1,47 @@
+"""Generate a small static site using only the Python standard library."""
+
+import random
+import sys
+from pathlib import Path
+
+
+def generate(directory):
+    directory = Path(directory)
+    directory.mkdir(parents=True, exist_ok=True)
+    title = f"{random.choice(('Quiet', 'Soft', 'Distant', 'Golden', 'Hidden', 'Open'))} {random.choice(('Horizons', 'Shapes', 'Reflections', 'Gardens', 'Waves', 'Spaces'))}"
+    hue = random.randrange(360)
+    shapes = []
+    for _ in range(18):
+        x, y, radius = random.randrange(800), random.randrange(480), random.randrange(30, 180)
+        color = f"hsl({(hue + random.randrange(90)) % 360},55%,65%)"
+        shapes.append(f'<circle cx="{x}" cy="{y}" r="{radius}" fill="{color}" opacity="0.45"/>')
+    image = (
+        '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 480">'
+        f'<rect width="800" height="480" fill="hsl({hue},25%,94%)"/>'
+        + ''.join(shapes) + '</svg>\n'
+    )
+    (directory / 'art.svg').write_text(image, encoding='utf-8')
+    (directory / 'robots.txt').write_text('User-agent: *\nDisallow: /\n', encoding='utf-8')
+    html = f'''<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <meta name="robots" content="noindex,nofollow">
+  <title>{title}</title>
+  <style>body{{max-width:50rem;margin:8vh auto;padding:0 1.5rem;font:17px/1.6 system-ui,sans-serif;color:#28323c}}img{{width:100%;height:auto;border-radius:12px}}</style>
+</head>
+<body>
+  <main>
+    <h1>{title}</h1>
+    <p>A small study of colour, form, and light.</p>
+    <img src="/art.svg" alt="An abstract composition of overlapping colourful circles" width="800" height="480">
+  </main>
+</body>
+</html>
+'''
+    (directory / 'index.html').write_text(html, encoding='utf-8')
+
+
+if __name__ == '__main__':
+    generate(sys.argv[1])

roles/https_proxy/tasks/main.yml

@@ -96,14 +96,20 @@
     - megaproxy_services.https.certificate == "ip-acme"
     - not megaproxy_certificate.stat.exists
 
-- name: Install active-probe decoy page
-  ansible.builtin.template:
-    src: decoy.html.j2
-    dest: /opt/megaproxy/decoy.html
+- name: Create decoy directory
+  ansible.builtin.file:
+    path: /opt/megaproxy/decoy
+    state: directory
     owner: root
     group: root
-    mode: "0644"
-  notify: Restart MegaProxy GOST
+    mode: "0755"
+
+- name: Generate a random static decoy site
+  ansible.builtin.script:
+    cmd: generate-decoy.py /opt/megaproxy/decoy
+    executable: /usr/bin/python3
+    creates: /opt/megaproxy/decoy/index.html
+  when: megaproxy_services.https.routes | selectattr('probe_resistance.enabled') | list | length > 0
 
 - name: Install secret GOST configuration
   ansible.builtin.template:

roles/https_proxy/templates/gost.yml.j2

@@ -10,7 +10,7 @@ services:
 {% endif %}
       metadata:
 {% if route.probe_resistance.enabled %}
-        probeResist: "file:/opt/megaproxy/decoy.html"
+        probeResist: "host:127.0.0.1:18080"
 {% if route.probe_resistance.knock %}
         knock: {{ route.probe_resistance.knock | join(',') | to_json }}
 {% endif %}
@@ -28,6 +28,16 @@ services:
           maxVersion: VersionTLS13
           alpn: [h2, http/1.1]
 {% endfor %}
+{% if megaproxy_services.https.routes | selectattr('probe_resistance.enabled') | list %}
+  - name: decoy
+    addr: "127.0.0.1:18080"
+    handler:
+      type: file
+      metadata:
+        dir: /opt/megaproxy/decoy
+    listener:
+      type: tcp
+{% endif %}
 
 authers:
   - name: megaproxy-users

roles/https_proxy/templates/megaproxy-gost.service.j2

@@ -7,7 +7,7 @@ Wants=network-online.target
 [Service]
 Type=simple
 ExecStartPre=-/usr/bin/docker rm -f megaproxy-gost
-ExecStart=/usr/bin/docker run --name megaproxy-gost --network host --read-only --cap-drop ALL --security-opt no-new-privileges --memory 256m --pids-limit 128 -v /opt/megaproxy/gost.yml:/etc/gost/gost.yml:ro -v /opt/megaproxy/decoy.html:/opt/megaproxy/decoy.html:ro -v {{ megaproxy_certificate_volume }} gogost/gost:{{ megaproxy_services.https.gost_version }} -C /etc/gost/gost.yml
+ExecStart=/usr/bin/docker run --name megaproxy-gost --network host --read-only --cap-drop ALL --security-opt no-new-privileges --memory 256m --pids-limit 128 -v /opt/megaproxy/gost.yml:/etc/gost/gost.yml:ro -v /opt/megaproxy/decoy:/opt/megaproxy/decoy:ro -v {{ megaproxy_certificate_volume }} gogost/gost:{{ megaproxy_services.https.gost_version }} -C /etc/gost/gost.yml
 ExecStop=/usr/bin/docker stop -t 10 megaproxy-gost
 Restart=on-failure
 RestartSec=5

roles/ssh_proxy/handlers/main.yml

@@ -1,7 +1,6 @@
 ---
 - name: Validate and reload ssh
   ansible.builtin.command: sshd -t
-  changed_when: false
   notify: Reload ssh
 
 - name: Reload ssh

src/megaproxy_server/inventory.py

@@ -202,7 +202,7 @@ def ansible_inventory(inventory: Inventory) -> dict[str, Any]:
             routes = https_routes(inventory, name)
             services["https"]["routes"] = routes
             variables_public_routes = [
-                {"name": route["name"], "hostname": route["hostname"], "port": https.port, "backend_port": route["port"]}
+                {"name": route["name"], "hostname": route["hostname"], "port": https.port, "backend_port": route["port"], "probe_resistance_enabled": route["probe_resistance"]["enabled"]}
                 for route in routes
             ]
             services["https"]["machine_auth"] = (

tests/integration/lxd.sh

@@ -69,6 +69,16 @@ done
 export ANSIBLE_HOST_KEY_CHECKING=True
 export ANSIBLE_SSH_ARGS="-o UserKnownHostsFile=$work_dir/known_hosts"
 
+# ProxyJump's child ssh reads -F too, but does not inherit command-line -i/-o options.
+cat > "$work_dir/ssh_config" <<EOF
+Host *
+  IdentityFile $key_file
+  IdentitiesOnly yes
+  UserKnownHostsFile $work_dir/known_hosts
+  StrictHostKeyChecking yes
+  BatchMode yes
+EOF
+
 {
   printf '%s\n' 'version: 1' 'settings:' '  manage_firewall: true' '  unattended_upgrades: true' 'hosts:'
   printf '%s\n' '  one:' "    address: $ip_one" '    admin:' '      user: ci-admin' '      bootstrap_user: root' '      port: 22' "      private_key_file: $key_file" "      public_key: \"$public_key\"" '    services:'
@@ -97,8 +107,6 @@ if grep -Eq 'changed=[1-9][0-9]*' "$second_run"; then
 fi
 
 control_socket="$work_dir/direct-control"
-printf '%s\n' 'LogLevel DEBUG1' | sudo lxc exec megaproxy-ci-one -- tee /etc/ssh/sshd_config.d/00-ci-debug.conf >/dev/null
-sudo lxc exec megaproxy-ci-one -- systemctl reload ssh
 if ! ssh -vvv -fNT -M -S "$control_socket" -i "$key_file" -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o UserKnownHostsFile="$work_dir/known_hosts" -L 18443:example.com:443 "mp-ci@$ip_one"; then
   sudo lxc exec megaproxy-ci-one -- getent passwd mp-ci || true
   sudo lxc exec megaproxy-ci-one -- passwd -S mp-ci || true
@@ -114,7 +122,11 @@ ssh -S "$control_socket" -O exit "mp-ci@$ip_one"
 
 if [[ "$host_count" -ge 2 ]]; then
   jump_socket="$work_dir/jump-control"
-  ssh -fNT -M -S "$jump_socket" -i "$key_file" -o IdentitiesOnly=yes -o ExitOnForwardFailure=yes -o UserKnownHostsFile="$work_dir/known_hosts" -o "ProxyJump=mp-ci@$ip_one" -L 19443:example.com:443 "mp-ci@$ip_two"
+  if ! ssh -vvv -F "$work_dir/ssh_config" -fNT -M -S "$jump_socket" -o ExitOnForwardFailure=yes -o "ProxyJump=mp-ci@$ip_one" -L 19443:example.com:443 "mp-ci@$ip_two"; then
+    sudo lxc exec megaproxy-ci-two -- journalctl -u ssh --since=-2min --no-pager
+    sudo lxc exec megaproxy-ci-two -- namei -l /etc/ssh/megaproxy_authorized_keys/mp-ci
+    exit 1
+  fi
   echo | openssl s_client -connect 127.0.0.1:19443 -servername example.com -verify_return_error >/dev/null
   ssh -S "$jump_socket" -O exit "mp-ci@$ip_two"
 fi

tests/test_decoy.py

@@ -0,0 +1,15 @@
+import runpy
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+
+def test_generated_decoy(tmp_path):
+    script = Path(__file__).resolve().parents[1] / 'roles/https_proxy/files/generate-decoy.py'
+    generate = runpy.run_path(str(script))['generate']
+    generate(tmp_path)
+    html = (tmp_path / 'index.html').read_text()
+    assert 'content="noindex,nofollow"' in html
+    assert '<h1>' in html and 'src="/art.svg"' in html
+    assert (tmp_path / 'robots.txt').read_text() == 'User-agent: *\nDisallow: /\n'
+    image = ET.parse(tmp_path / 'art.svg').getroot()
+    assert len(image.findall('{http://www.w3.org/2000/svg}circle')) == 18

tests/test_inventory.py

@@ -167,6 +167,9 @@ def test_route_probe_override_is_independent() -> None:
     route = https_routes(inventory, "entry")[0]
     assert route["probe_resistance"]["enabled"] is True
     assert route["probe_resistance"]["knock"] == ["private.example"]
+    public_route = ansible_inventory(inventory)["all"]["hosts"]["entry"]["megaproxy_https_public_routes"][0]
+    assert public_route["probe_resistance_enabled"] is True
+    assert "chain" not in public_route
 
 
 def test_https_chain_title_falls_back_to_entry_title() -> None:

tests/test_ssh_handlers.py

@@ -0,0 +1,33 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+from ruamel.yaml import YAML
+
+
+def test_ssh_validation_notifies_reload(tmp_path):
+    root = Path(__file__).resolve().parents[1]
+    yaml = YAML(typ="safe")
+    tasks, handlers, markers = [], [], []
+    for role in ("admin", "ssh_proxy"):
+        validation, reload = yaml.load((root / f"roles/{role}/handlers/main.yml").read_text())[:2]
+        validation["ansible.builtin.command"] = "true"
+        marker = tmp_path / role
+        markers.append(marker)
+        reload.pop("ansible.builtin.systemd_service")
+        reload["ansible.builtin.copy"] = {"dest": str(marker), "content": "reloaded"}
+        tasks.append({"ansible.builtin.debug": {"msg": role}, "changed_when": True,
+                      "notify": validation["name"]})
+        handlers.extend((validation, reload))
+    playbook = tmp_path / "handlers.yml"
+    with playbook.open("w") as stream:
+        yaml.dump([{"hosts": "all", "gather_facts": False, "tasks": tasks,
+                    "handlers": handlers}], stream)
+    environment = os.environ.copy()
+    environment["ANSIBLE_REMOTE_TEMP"] = str(tmp_path / "remote-tmp")
+    environment["PATH"] = str(Path(sys.executable).parent) + os.pathsep + environment["PATH"]
+    result = subprocess.run(["ansible-playbook", "-i", "localhost,", "-c", "local", str(playbook)],
+                            cwd=root, env=environment, capture_output=True, text=True)
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert all(marker.read_text() == "reloaded" for marker in markers)