GitHub file history

#!/usr/bin/env python3
"""Build a census of active public repos from GH Archive's free hourly files.

GH Archive (https://www.gharchive.org) publishes every public GitHub event, one gzipped JSON file
per hour, at https://data.gharchive.org/YYYY-MM-DD-H.json.gz. No account or token is needed.
This keeps the names of repos with at least one event of the chosen types in the chosen hours.

Usage:
  python3 gharchive_census.py --start 2026-10-01T00 --hours 24 [--every 1]
                              [--types PushEvent] > census.txt

  --start   first hour, UTC, as YYYY-MM-DDTHH
  --hours   how many hours to read from --start
  --every   read one hour in N (e.g. --hours 720 --every 24: one hour a day for 30 days)
  --types   event types to count (default PushEvent: repos someone pushed to)

Output: one owner/repo per line, sorted, unique. make_list.py census reads it.
One hour is about 18 MB to download and gives about 40,000 repos (2026-10-01 12:00 UTC gave 39,397).
A failed hour is reported on stderr and skipped.
"""
import argparse
import gzip
import io
import json
import re
import sys
import urllib.request
from datetime import datetime, timedelta

NAME = re.compile(r"^[A-Za-z0-9-]+/[A-Za-z0-9._-]+$")


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--start", required=True)
    ap.add_argument("--hours", type=int, default=1)
    ap.add_argument("--every", type=int, default=1)
    ap.add_argument("--types", nargs="+", default=["PushEvent"])
    a = ap.parse_args()
    t0 = datetime.strptime(a.start, "%Y-%m-%dT%H")
    types, repos, read = set(a.types), set(), 0
    for h in range(0, a.hours, a.every):
        t = t0 + timedelta(hours=h)
        url = f"https://data.gharchive.org/{t:%Y-%m-%d}-{t.hour}.json.gz"
        try:
            raw = urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "github-file-history/1.0"}), timeout=120).read()
        except Exception as e:  # missing hours exist in the archive
            print(f"ERR {url} {e}", file=sys.stderr)
            continue
        with gzip.open(io.BytesIO(raw), "rt", encoding="utf-8", errors="replace") as f:
            for line in f:
                try:
                    e = json.loads(line)
                except ValueError:
                    continue
                if e.get("type") in types:
                    name = (e.get("repo") or {}).get("name", "")
                    if NAME.match(name):
                        repos.add(name)
        read += 1
        print(f"{url}: {len(repos)} repos so far", file=sys.stderr)
    sys.stdout.write("".join(r + "\n" for r in sorted(repos)))
    print(f"read {read} hours, {len(repos)} repos", file=sys.stderr)


if __name__ == "__main__":
    main()