| 1 | #!/usr/bin/env python3 |
| 2 | """Generate a static browsing site for a directory of Mercurial repositories. |
| 3 | |
| 4 | For each repo: an index page (metadata, rendered README, file list of tip) |
| 5 | and one syntax-highlighted HTML page per file at tip. Plus a top-level |
| 6 | index of all repos. |
| 7 | |
| 8 | Requires hg (obviously). Optional dependencies: |
| 9 | |
| 10 | - pygments (highlighting) |
| 11 | - markdown-it-py, or markdown (the former tends to do a better job) for README.md rendering |
| 12 | - docutils for README.rst rendering |
| 13 | """ |
| 14 | |
| 15 | import argparse |
| 16 | import configparser |
| 17 | import html |
| 18 | import posixpath |
| 19 | import re |
| 20 | import shutil |
| 21 | import subprocess |
| 22 | import sys |
| 23 | import tempfile |
| 24 | from datetime import datetime, timezone |
| 25 | from pathlib import Path |
| 26 | from urllib.parse import quote, unquote, urlsplit |
| 27 | |
| 28 | try: |
| 29 | import pygments |
| 30 | from pygments import lexers, formatters |
| 31 | from pygments.util import ClassNotFound |
| 32 | except ImportError: |
| 33 | pygments = None |
| 34 | |
| 35 | try: |
| 36 | from markdown_it import MarkdownIt |
| 37 | except ImportError: |
| 38 | MarkdownIt = None |
| 39 | |
| 40 | try: |
| 41 | import markdown |
| 42 | except ImportError: |
| 43 | markdown = None |
| 44 | |
| 45 | try: |
| 46 | from docutils.core import publish_parts |
| 47 | except ImportError: |
| 48 | publish_parts = None |
| 49 | |
| 50 | MAX_RENDER_BYTES = 512 * 1024 |
| 51 | README_NAMES = ("readme.md", "readme.rst", "readme.txt", "readme") |
| 52 | |
| 53 | PAGE = """<!DOCTYPE html> |
| 54 | <html lang="en"> |
| 55 | <head> |
| 56 | <meta charset="utf-8"> |
| 57 | <meta name="viewport" content="width=device-width, initial-scale=1"> |
| 58 | <title>{title}</title> |
| 59 | <link rel="stylesheet" href="{root}style.css"> |
| 60 | </head> |
| 61 | <body> |
| 62 | <nav>{nav}</nav> |
| 63 | {body} |
| 64 | <footer>generated {generated}</footer> |
| 65 | </body> |
| 66 | </html> |
| 67 | """ |
| 68 | |
| 69 | CSS = """\ |
| 70 | :root { --fg: #1a1a1a; --bg: #ffffff; --dim: #6a737d; --line: #e1e4e8; |
| 71 | --accent: #0550ae; --code-bg: #f6f8fa; } |
| 72 | @media (prefers-color-scheme: dark) { |
| 73 | :root { --fg: #d0d4da; --bg: #16181c; --dim: #8b949e; --line: #30363d; |
| 74 | --accent: #6cb2ff; --code-bg: #1f2329; } |
| 75 | } |
| 76 | body { color: var(--fg); background: var(--bg); margin: 0 auto; padding: 0 1rem 2rem; |
| 77 | max-width: 60rem; font: 16px/1.5 system-ui, sans-serif; } |
| 78 | a { color: var(--accent); text-decoration: none; } |
| 79 | a:hover { text-decoration: underline; } |
| 80 | nav { padding: 0.8rem 0; border-bottom: 1px solid var(--line); color: var(--dim); } |
| 81 | nav .rev { font-family: ui-monospace, monospace; font-size: 0.85em; } |
| 82 | footer { margin-top: 2rem; border-top: 1px solid var(--line); padding-top: 0.5rem; |
| 83 | color: var(--dim); font-size: 0.8rem; } |
| 84 | table { border-collapse: collapse; width: 100%; } |
| 85 | td, th { padding: 0.3rem 1rem 0.3rem 0; text-align: left; vertical-align: baseline; } |
| 86 | .dim { color: var(--dim); font-size: 0.9rem; } |
| 87 | pre, code, .filelist { font: 13px/1.45 ui-monospace, monospace; } |
| 88 | pre { background: var(--code-bg); padding: 0.8rem; overflow-x: auto; |
| 89 | border-radius: 6px; } |
| 90 | .clone { user-select: all; } |
| 91 | .readme { border: 1px solid var(--line); border-radius: 6px; padding: 0 1.5rem; |
| 92 | margin: 1.5rem 0; } |
| 93 | .readme img { max-width: 100%; } |
| 94 | .source pre { background: none; padding: 0; margin: 0; overflow: visible; } |
| 95 | .source { background: var(--code-bg); padding: 0.8rem; border-radius: 6px; |
| 96 | overflow-x: auto; } |
| 97 | .source td { padding: 0; } |
| 98 | .source td.linenos { padding-right: 1rem; user-select: none; text-align: right; } |
| 99 | .source td.linenos a { color: var(--dim); display: block; } |
| 100 | .source tr:target td, .source tr:has(td.linenos:hover) td { |
| 101 | background: color-mix(in srgb, var(--accent) 15%, transparent); } |
| 102 | """ |
| 103 | |
| 104 | |
| 105 | def run(cmd, cwd=None): |
| 106 | return subprocess.run(cmd, cwd=cwd, check=True, capture_output=True, |
| 107 | text=True).stdout |
| 108 | |
| 109 | |
| 110 | def find_repos(root): |
| 111 | """Yield paths of repos under root, without descending into them.""" |
| 112 | repos = [] |
| 113 | def walk(d): |
| 114 | if (d / ".hg").is_dir(): |
| 115 | repos.append(d) |
| 116 | return |
| 117 | for child in sorted(d.iterdir()): |
| 118 | if child.is_dir() and not child.is_symlink(): |
| 119 | walk(child) |
| 120 | for child in sorted(root.iterdir()): |
| 121 | if child.is_dir() and not child.is_symlink(): |
| 122 | walk(child) |
| 123 | return repos |
| 124 | |
| 125 | |
| 126 | def repo_meta(repo): |
| 127 | tip = run(["hg", "-R", str(repo), "log", "-r", "tip", |
| 128 | "-T", "{node|short}\t{date|rfc3339date}\t{date|shortdate}"]) |
| 129 | node, date, shortdate = (tip.split("\t") if tip else ("", "", "")) |
| 130 | if set(node) == {"0"}: # null revision: empty repository |
| 131 | node, date, shortdate = "", "", "" |
| 132 | desc = subprocess.run(["hg", "-R", str(repo), "config", "web.description"], |
| 133 | capture_output=True, text=True).stdout.strip() |
| 134 | return {"node": node, "date": date, "shortdate": shortdate, "desc": desc} |
| 135 | |
| 136 | |
| 137 | def render_readme(name, data, paths): |
| 138 | """Render a README to HTML; relative links to files in the repo (given as |
| 139 | a set of posix relative paths) are pointed at their rendered pages.""" |
| 140 | text = data.decode("utf-8", errors="replace") |
| 141 | lower = name.lower() |
| 142 | if lower.endswith(".md") and MarkdownIt: |
| 143 | out = MarkdownIt("commonmark").enable(["table", "strikethrough"]) \ |
| 144 | .render(text) |
| 145 | elif lower.endswith(".md") and markdown: |
| 146 | out = markdown.markdown(text, extensions=["fenced_code", "tables"]) |
| 147 | elif lower.endswith(".rst") and publish_parts: |
| 148 | out = publish_parts(text, writer_name="html5", |
| 149 | settings_overrides={"report_level": 5, |
| 150 | "file_insertion_enabled": False, |
| 151 | "raw_enabled": False})["body"] |
| 152 | else: |
| 153 | return "<pre>%s</pre>" % html.escape(text) |
| 154 | |
| 155 | def fix(m): |
| 156 | url = urlsplit(html.unescape(m.group(1))) |
| 157 | if url.scheme or url.netloc or not url.path or url.path.startswith("/"): |
| 158 | return m.group(0) |
| 159 | rel = posixpath.normpath(unquote(url.path)) |
| 160 | if rel not in paths: |
| 161 | return m.group(0) |
| 162 | return 'href="file/%s.html"' % quote(rel) |
| 163 | return re.sub(r'href="([^"]*)"', fix, out) |
| 164 | |
| 165 | |
| 166 | def highlight(relpath, data): |
| 167 | text = data.decode("utf-8", errors="replace") |
| 168 | if pygments: |
| 169 | try: |
| 170 | lexer = lexers.get_lexer_for_filename(relpath, text) |
| 171 | except ClassNotFound: |
| 172 | lexer = lexers.TextLexer() |
| 173 | fmt = formatters.HtmlFormatter(nowrap=True) |
| 174 | lines = pygments.highlight(text, lexer, fmt).split("\n") |
| 175 | if lines and lines[-1] == "": |
| 176 | lines.pop() |
| 177 | else: |
| 178 | lines = [html.escape(l) for l in text.splitlines()] |
| 179 | rows = "\n".join( |
| 180 | '<tr id="l-%d"><td class="linenos"><a href="#l-%d">%d</a></td>' |
| 181 | '<td><pre>%s</pre></td></tr>' % (i, i, i, l or " ") |
| 182 | for i, l in enumerate(lines or [""], 1)) |
| 183 | return '<div class="source"><table>%s</table></div>' % rows |
| 184 | |
| 185 | |
| 186 | def pygments_css(): |
| 187 | if not pygments: |
| 188 | return "" |
| 189 | light = formatters.HtmlFormatter(cssclass="source").get_style_defs(".source") |
| 190 | try: |
| 191 | dark = formatters.HtmlFormatter( |
| 192 | style="github-dark", cssclass="source").get_style_defs(".source") |
| 193 | return ("%s\n@media (prefers-color-scheme: dark) {\n%s\n}\n" |
| 194 | % (light, dark)) |
| 195 | except ClassNotFound: |
| 196 | return light |
| 197 | |
| 198 | |
| 199 | def page(out_path, title, nav, body, depth): |
| 200 | root = "../" * depth |
| 201 | out_path.parent.mkdir(parents=True, exist_ok=True) |
| 202 | out_path.write_text(PAGE.format( |
| 203 | title=html.escape(title), nav=nav, body=body, root=root, |
| 204 | generated=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"))) |
| 205 | |
| 206 | |
| 207 | def is_binary(data): |
| 208 | return b"\0" in data[:8192] |
| 209 | |
| 210 | |
| 211 | def fmt_size(n): |
| 212 | for unit in ("B", "KiB", "MiB", "GiB"): |
| 213 | if n < 1024 or unit == "GiB": |
| 214 | return "%d %s" % (n, unit) if unit == "B" else "%.1f %s" % (n, unit) |
| 215 | n /= 1024 |
| 216 | |
| 217 | |
| 218 | def generate_repo(repo, name, out, clone_base): |
| 219 | """Build the static pages for one repo into out/<name>.""" |
| 220 | meta = repo_meta(repo) |
| 221 | depth = len(Path(name).parts) |
| 222 | dest = out / name |
| 223 | tmp = out / (name + ".new") |
| 224 | shutil.rmtree(tmp, ignore_errors=True) |
| 225 | tmp.mkdir(parents=True) |
| 226 | |
| 227 | tip = ' <span class="rev">@ %s</span>' % meta["node"] if meta["node"] else "" |
| 228 | nav = '<a href="%s">repos</a> / %s%s' % ("../" * depth, |
| 229 | html.escape(name), tip) |
| 230 | body = ["<h1>%s</h1>" % html.escape(name)] |
| 231 | if meta["desc"]: |
| 232 | body.append("<p>%s</p>" % html.escape(meta["desc"])) |
| 233 | |
| 234 | if not meta["node"]: |
| 235 | body.append('<p class="dim">empty repository</p>') |
| 236 | else: |
| 237 | clone_url = ("%s/%s" % (clone_base.rstrip("/"), name) |
| 238 | if clone_base else name) |
| 239 | body.append('<pre class="clone">hg clone %s</pre>' |
| 240 | % html.escape(clone_url)) |
| 241 | body.append('<p class="dim">tip: %s (%s)</p>' |
| 242 | % (meta["node"], meta["shortdate"])) |
| 243 | |
| 244 | with tempfile.TemporaryDirectory() as td: |
| 245 | arch = Path(td) / "tip" |
| 246 | run(["hg", "-R", str(repo), "archive", "-r", "tip", |
| 247 | "-t", "files", str(arch)]) |
| 248 | files = sorted(p for p in arch.rglob("*") |
| 249 | if p.is_file() and not p.is_symlink() |
| 250 | and p.name != ".hg_archival.txt") |
| 251 | |
| 252 | readme = next((p for p in files |
| 253 | if p.parent == arch |
| 254 | and p.name.lower() in README_NAMES), None) |
| 255 | if readme: |
| 256 | body.append('<div class="readme">%s</div>' |
| 257 | % render_readme(readme.name, readme.read_bytes(), |
| 258 | {f.relative_to(arch).as_posix() |
| 259 | for f in files})) |
| 260 | |
| 261 | rows = [] |
| 262 | for f in files: |
| 263 | rel = f.relative_to(arch).as_posix() |
| 264 | data = f.read_bytes() |
| 265 | label = html.escape(rel) |
| 266 | size = fmt_size(len(data)) |
| 267 | if is_binary(data): |
| 268 | rows.append("<tr><td>%s</td><td>%s</td>" |
| 269 | '<td class="dim">binary</td></tr>' |
| 270 | % (label, size)) |
| 271 | continue |
| 272 | href = "file/%s.html" % quote(rel) |
| 273 | rows.append('<tr><td><a href="%s">%s</a></td>' |
| 274 | "<td>%s</td><td></td></tr>" % (href, label, size)) |
| 275 | if len(data) > MAX_RENDER_BYTES: |
| 276 | content = ('<p class="dim">file too large to render ' |
| 277 | "(%s)</p>" % size) |
| 278 | else: |
| 279 | content = highlight(rel, data) |
| 280 | fdepth = depth + 1 + rel.count("/") |
| 281 | fnav = ('<a href="%s">repos</a> / <a href="%s">%s</a> / %s%s' |
| 282 | % ("../" * fdepth, |
| 283 | "../" * (1 + rel.count("/")), |
| 284 | html.escape(name), label, tip)) |
| 285 | page(tmp / "file" / (rel + ".html"), "%s: %s" % (name, rel), |
| 286 | fnav, "<h1>%s</h1>%s" % (label, content), fdepth) |
| 287 | body.append("<h2>Files</h2>" |
| 288 | '<table class="filelist">%s</table>' % "".join(rows)) |
| 289 | |
| 290 | page(tmp / "index.html", name, nav, "\n".join(body), depth) |
| 291 | shutil.rmtree(dest, ignore_errors=True) |
| 292 | tmp.rename(dest) |
| 293 | return meta |
| 294 | |
| 295 | |
| 296 | def discover_config(repo): |
| 297 | """Find hgstatic.ini in an ancestor of repo (directly or in config/).""" |
| 298 | for anc in (repo, *repo.parents): |
| 299 | for cand in (anc / "hgstatic.ini", anc / "config" / "hgstatic.ini"): |
| 300 | if cand.is_file(): |
| 301 | return cand |
| 302 | return None |
| 303 | |
| 304 | |
| 305 | def main(): |
| 306 | ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) |
| 307 | ap.add_argument("-c", "--config", type=Path, |
| 308 | help="ini file with an [hgstatic] section supplying " |
| 309 | "root, out, clone-base and title; command-line " |
| 310 | "options override it") |
| 311 | ap.add_argument("--root", type=Path, |
| 312 | help="directory containing the hg repositories") |
| 313 | ap.add_argument("--out", type=Path, |
| 314 | help="output directory for the static site") |
| 315 | ap.add_argument("--clone-base", |
| 316 | help="base URL shown in clone commands, " |
| 317 | "e.g. https://code.example.org") |
| 318 | ap.add_argument("--only", type=Path, |
| 319 | help="regenerate only this repo (path or name relative " |
| 320 | "to root); the index is always regenerated. " |
| 321 | "Without -c/--root, hgstatic.ini is searched for " |
| 322 | "in the repo's ancestors (also under config/); if " |
| 323 | "none is found, exit quietly -- this makes a " |
| 324 | "global changegroup hook safe for every repo") |
| 325 | ap.add_argument("--title", help="title of the top-level index page") |
| 326 | args = ap.parse_args() |
| 327 | |
| 328 | cfg_path = args.config |
| 329 | if cfg_path is None and args.root is None: |
| 330 | if args.only is None: |
| 331 | ap.error("one of --config, --root or --only is required") |
| 332 | cfg_path = discover_config(args.only.resolve()) |
| 333 | if cfg_path is None: |
| 334 | return # repo outside any configured area: nothing to do |
| 335 | cfg = {} |
| 336 | if cfg_path is not None: |
| 337 | cp = configparser.ConfigParser() |
| 338 | if not cp.read(cfg_path) or "hgstatic" not in cp: |
| 339 | sys.exit("%s: unreadable or missing [hgstatic] section" |
| 340 | % cfg_path) |
| 341 | cfg = cp["hgstatic"] |
| 342 | cfg_dir = cfg_path.parent.resolve() |
| 343 | # relative paths in the ini are relative to the ini's directory |
| 344 | for key in ("root", "out"): |
| 345 | if key in cfg: |
| 346 | cfg[key] = str((cfg_dir / cfg[key]).resolve()) |
| 347 | |
| 348 | root = args.root or (Path(cfg["root"]) if "root" in cfg |
| 349 | else cfg_path and cfg_path.parent) |
| 350 | out = args.out or ("out" in cfg and Path(cfg["out"])) |
| 351 | if not root or not out: |
| 352 | sys.exit("no repository root/output directory given " |
| 353 | "(--root/--out or ini keys root/out)") |
| 354 | clone_base = args.clone_base if args.clone_base is not None \ |
| 355 | else cfg.get("clone-base", "") |
| 356 | title = args.title if args.title is not None \ |
| 357 | else cfg.get("title", "repositories") |
| 358 | |
| 359 | root = root.resolve() |
| 360 | repos = find_repos(root) |
| 361 | if not repos: |
| 362 | sys.exit("no repositories found under %s" % root) |
| 363 | only = None |
| 364 | if args.only: |
| 365 | only = args.only.resolve() if args.only.is_absolute() \ |
| 366 | else (root / args.only).resolve() |
| 367 | if only not in repos: |
| 368 | sys.exit("%s is not a repository under %s" % (only, root)) |
| 369 | |
| 370 | out.mkdir(parents=True, exist_ok=True) |
| 371 | entries = [] |
| 372 | for repo in repos: |
| 373 | name = repo.relative_to(root).as_posix() |
| 374 | if only is None or repo == only: |
| 375 | meta = generate_repo(repo, name, out, clone_base) |
| 376 | else: |
| 377 | meta = repo_meta(repo) |
| 378 | entries.append((name, meta)) |
| 379 | |
| 380 | entries.sort(key=lambda e: e[1]["date"], reverse=True) |
| 381 | rows = "".join( |
| 382 | '<tr><td><a href="%s/">%s</a></td><td>%s</td>' |
| 383 | '<td class="dim">%s</td></tr>' |
| 384 | % (quote(name), html.escape(name), html.escape(m["desc"]), |
| 385 | m["shortdate"] or "empty") |
| 386 | for name, m in entries) |
| 387 | page(out / "index.html", title, |
| 388 | html.escape(title), |
| 389 | "<h1>%s</h1><table>%s</table>" % (html.escape(title), rows), 0) |
| 390 | (out / "style.css").write_text(CSS + pygments_css()) |
| 391 | |
| 392 | |
| 393 | if __name__ == "__main__": |
| 394 | main() |