|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Fail if any internal link in the built site points at a page that doesn't exist. |
| 3 | +
|
| 4 | +Usage: hugo --minify && python3 tools/check-internal-links.py [public] |
| 5 | +
|
| 6 | +Checks every href in the generated HTML plus the <loc> URLs in the sitemap |
| 7 | +(relative and absolute oursqlfoundation.org links alike). |
| 8 | +""" |
| 9 | +import collections |
| 10 | +import os |
| 11 | +import re |
| 12 | +import sys |
| 13 | +import urllib.parse |
| 14 | + |
| 15 | +ROOT = sys.argv[1] if len(sys.argv) > 1 else "public" |
| 16 | +HOSTS = ("oursqlfoundation.org",) |
| 17 | + |
| 18 | + |
| 19 | +def exists(path): |
| 20 | + cand = os.path.join(ROOT, urllib.parse.unquote(path).lstrip("/")) |
| 21 | + return os.path.isfile(cand) or os.path.isfile(os.path.join(cand, "index.html")) |
| 22 | + |
| 23 | + |
| 24 | +def normalize(href): |
| 25 | + m = re.match(r"^https?://(?:www\.)?([^/]+)(/.*)?$", href) |
| 26 | + if m: |
| 27 | + return (m.group(2) or "/") if m.group(1) in HOSTS else None |
| 28 | + return href if href.startswith("/") and not href.startswith("//") else None |
| 29 | + |
| 30 | + |
| 31 | +broken = collections.defaultdict(set) |
| 32 | +checked = 0 |
| 33 | +for dirpath, _, files in os.walk(ROOT): |
| 34 | + for name in files: |
| 35 | + if not name.endswith((".html", ".xml")): |
| 36 | + continue |
| 37 | + path = os.path.join(dirpath, name) |
| 38 | + text = open(path, encoding="utf-8", errors="ignore").read() |
| 39 | + refs = re.findall(r'href="?([^"\s>]+)', text) |
| 40 | + if name.endswith(".xml"): |
| 41 | + refs += re.findall(r"<loc>([^<]+)</loc>", text) |
| 42 | + for raw in refs: |
| 43 | + href = normalize(raw.split("#")[0].split("?")[0]) |
| 44 | + if href: |
| 45 | + checked += 1 |
| 46 | + if not exists(href): |
| 47 | + broken[href].add(os.path.relpath(path, ROOT)) |
| 48 | + |
| 49 | +print(f"links checked: {checked} | distinct broken: {len(broken)}") |
| 50 | +for href, pages in sorted(broken.items()): |
| 51 | + print(f" {href} <- {len(pages)} page(s), e.g. {sorted(pages)[0]}") |
| 52 | +sys.exit(1 if broken else 0) |
0 commit comments