GitHub file history

#!/usr/bin/env python3
"""Build the list collect.sh reads: one owner/repo/PATTERN per line.

Pick one source of repos, then say which paths you want with --path (repeatable; a literal
path or a glob such as '.cursor/rules/*.mdc'). Every repo is crossed with every --path.

Sources:
  file FILE              one owner/repo per line (a GitHub URL or an owner/repo/path line is
                         cut down to owner/repo). Lines starting with # are skipped.
  search FILE            a GitHub search export: the JSON of `gh search repos ... --json fullName`,
                         a saved REST response from /search/repositories ({"items": [...]}), a
                         JSON list of names, or a CSV with a full_name, fullName, nameWithOwner,
                         repo_name or repo column.
  census FILE --n N      a random draw of N repos from any census list (CSV with a repo_name
                         column, gzipped or not, or one owner/repo per line), with
                         random.Random(--seed).sample. --sampled writes the drawn repos, so the
                         denominator is on record.
  catalog                the agentleFS public catalog sitemap (https://agentlefs.com/context/sitemap.xml).
                         It lists agent instruction files (AGENTS.md, CLAUDE.md, SKILL.md, llms.txt)
                         in public repos with 1,000+ stars whose licence allows redistribution.
                         Without --path it lists the catalog's own files (root files only, unless
                         --any-depth; --names filters by file name). With --path it is just a
                         source of repos.

Options:
  --path PATTERN         a path or glob to collect in every repo (repeatable)
  --probe                keep only (repo, path) pairs whose file exists today on the default
                         branch, by a HEAD request to raw.githubusercontent.com (no token, no API
                         rate limit). Literal paths only; globs are kept as they are.
  --jobs 16              probes at once. Non-404 errors are retried 4 times, then reported on
                         stderr as "ERR <code> <repo> <path>".
  --limit N              stop after N output lines

Examples:
  python3 make_list.py file repos.txt --path README.md --path package.json > list.txt
  python3 make_list.py census census.csv.gz --n 30000 --seed 1 --path CLAUDE.md --path AGENTS.md \\
      --probe --sampled drawn.txt > random.txt
  python3 make_list.py catalog --names CLAUDE.md AGENTS.md > popular.txt
  python3 make_list.py search repos.json --path '.cursor/rules/*.mdc' > list.txt
"""
import argparse
import concurrent.futures as cf
import csv
import gzip
import io
import json
import random
import re
import sys
import time
import urllib.error
import urllib.request

SITEMAP = "https://agentlefs.com/context/sitemap.xml"
UA = {"User-Agent": "github-file-history/1.0"}
NAME = re.compile(r"^[A-Za-z0-9-]+/[A-Za-z0-9._-]+$")
NAME_COLS = ("full_name", "fullName", "nameWithOwner", "repo_name", "repo")


def opener(path):
    return gzip.open(path, "rt", encoding="utf-8", errors="replace") if path.endswith(".gz") else open(path, encoding="utf-8", errors="replace")


def as_repo(s):
    s = s.strip()
    s = re.sub(r"^(https?://)?(www\.)?github\.com/", "", s)
    s = re.sub(r"\.git$", "", s)
    parts = [p for p in s.split("/") if p]
    name = "/".join(parts[:2]) if len(parts) >= 2 and not s.startswith("#") else None
    return name if name and NAME.match(name) else None


def from_file(path):
    with opener(path) as f:
        return [r for r in (as_repo(l) for l in f) if r]


def from_search(path):
    with opener(path) as f:
        text = f.read()
    try:
        data = json.loads(text)
    except ValueError:
        start = min((text.find(c) for c in NAME_COLS if c in text), default=0)
        start = text.rfind("\n", 0, start) + 1
        rows = csv.DictReader(io.StringIO(text[start:]))
        col = next(c for c in NAME_COLS if c in (rows.fieldnames or []))
        return [r for r in (as_repo(row[col] or "") for row in rows) if r]
    if isinstance(data, dict):
        data = data.get("items", data.get("repositories", []))
    out = []
    for item in data:
        name = item if isinstance(item, str) else next((item[c] for c in NAME_COLS if c in item), None)
        if name is None and isinstance(item, dict) and "owner" in item and "name" in item:
            owner = item["owner"]["login"] if isinstance(item["owner"], dict) else item["owner"]
            name = f"{owner}/{item['name']}"
        r = as_repo(name or "")
        if r:
            out.append(r)
    return out


def read_census(path):
    """Repo names from a census: a CSV with a repo_name column (the header may follow junk lines),
    or one owner/repo per line. Names that are not owner/repo are dropped and counted."""
    with opener(path) as f:
        lines = f.read().splitlines()
    header = next((i for i, l in enumerate(lines[:100]) if "repo_name" in next(csv.reader([l]), [])), None)
    if header is not None:
        names = [(r.get("repo_name") or "").strip() for r in csv.DictReader(io.StringIO("\n".join(lines[header:])))]
    else:
        names = [l.strip() for l in lines if l.strip() and not l.startswith("#")]
    good = [n for n in names if NAME.match(n)]
    if len(good) < len(names):
        bad = [n for n in names if not NAME.match(n)]
        print(f"census: dropped {len(bad)} lines that are not owner/repo, e.g. {bad[0][:60]!r}", file=sys.stderr)
    if not good:
        sys.exit(f"{path}: no owner/repo names found. Use a CSV with a repo_name column, or one owner/repo per line.")
    return good


def from_catalog(src, names, any_depth):
    if src.startswith("http"):
        xml = urllib.request.urlopen(urllib.request.Request(src, headers=UA), timeout=60).read().decode("utf-8")
    else:
        xml = open(src, encoding="utf-8").read()
    locs = re.findall(r"<loc>https://agentlefs\.com/context/([^<]+)</loc>", xml)
    total = xml.count("<loc>")
    out, seen = [], set()
    for p in locs:
        parts = p.split("/")
        if len(parts) < 3 or (not any_depth and len(parts) != 3):
            continue
        if names and parts[-1] not in names:
            continue
        if p not in seen:
            seen.add(p)
            out.append(p)
    print(f"sitemap urls {total}, files kept {len(out)}", file=sys.stderr)
    if total >= 20000:
        print("note: the sitemap is at its 20,000-URL cap, so the lowest-starred files may be missing", file=sys.stderr)
    return out


def probe(repo, path, tries=4):
    url = f"https://raw.githubusercontent.com/{repo}/HEAD/{path}"
    code = None
    for k in range(tries):
        try:
            with urllib.request.urlopen(urllib.request.Request(url, method="HEAD", headers=UA), timeout=15) as r:
                return r.status
        except urllib.error.HTTPError as e:
            if e.code == 404:
                return 404
            code = e.code
        except Exception as e:  # timeouts, resets
            code = type(e).__name__
        time.sleep(2 ** k)
    return code


def is_glob(p):
    return any(c in p for c in "*?[")


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("source", choices=["file", "search", "census", "catalog"])
    ap.add_argument("input", nargs="?", help="the file to read (catalog: optional local sitemap)")
    ap.add_argument("--path", action="append", default=[], help="path or glob to collect in every repo")
    ap.add_argument("--n", type=int, help="census: repos to draw")
    ap.add_argument("--seed", type=int, default=1)
    ap.add_argument("--sampled", help="census: also write the drawn repos here")
    ap.add_argument("--names", nargs="+", help="catalog: keep these file names")
    ap.add_argument("--any-depth", action="store_true", help="catalog: also files below the repo root")
    ap.add_argument("--probe", action="store_true")
    ap.add_argument("--jobs", type=int, default=16)
    ap.add_argument("--limit", type=int, default=0)
    a = ap.parse_args()

    if a.source == "catalog":
        files = from_catalog(a.input or SITEMAP, a.names, a.any_depth)
        if not a.path:
            pairs = [("/".join(p.split("/")[:2]), "/".join(p.split("/")[2:])) for p in files]
            repos = None
        else:
            repos = list(dict.fromkeys("/".join(p.split("/")[:2]) for p in files))
    else:
        if not a.input:
            sys.exit(f"{a.source} needs an input file")
        if a.source == "file":
            repos = from_file(a.input)
        elif a.source == "search":
            repos = from_search(a.input)
        else:
            if not a.n:
                sys.exit("census needs --n")
            names = read_census(a.input)
            repos = random.Random(a.seed).sample(names, min(a.n, len(names)))
            print(f"census {len(names)} repos, drew {len(repos)} (seed {a.seed})", file=sys.stderr)
            if a.sampled:
                open(a.sampled, "w").write("".join(r + "\n" for r in repos))
        repos = list(dict.fromkeys(repos)) if a.source != "census" else repos
    if repos is not None:
        if not a.path:
            sys.exit("say which files to collect with --path")
        pairs = [(r, p) for r in repos for p in a.path]

    if a.probe:
        todo = [(r, p) for r, p in pairs if not is_glob(p)]
        ok, errs = set(), 0
        with cf.ThreadPoolExecutor(a.jobs) as ex:
            for (r, p), code in zip(todo, ex.map(lambda j: probe(*j), todo)):
                if code == 200:
                    ok.add((r, p))
                elif code != 404:
                    print(f"ERR {code} {r} {p}", file=sys.stderr)
                    errs += 1
        print(f"probed {len(todo)}, found {len(ok)}, errors {errs}", file=sys.stderr)
        pairs = [(r, p) for r, p in pairs if is_glob(p) or (r, p) in ok]
    if a.limit:
        pairs = pairs[: a.limit]
    sys.stdout.write("".join(f"{r}/{p}\n" for r, p in pairs))


if __name__ == "__main__":
    main()