GitHub file history

#!/usr/bin/env bash
# history.sh OWNER/REPO OUTDIR PATTERN [PATTERN ...]
#
# One bare, blob-less clone of a public repo, then the full history of every file that matches
# a PATTERN. A pattern is a literal path (README.md, docs/intro.md) or a glob:
#   *   any characters except /      ?  one character except /
#   **  any characters including /   [abc]  one of a set
# A glob is matched against every path that ever existed in the history, so files that were
# deleted or renamed away are found too.
#
# For each file it writes OUTDIR/vers/<owner>_<repo>__<path with / as _>.tsv.gz, one row per
# commit that touched the file, in git log order (newest first):
#   sha <TAB> author_date (ISO 8601) <TAB> claude_coauthor (0/1) <TAB> base64(file body)
# A commit that deleted the file has an empty body. Set HISTORY_FORMAT=tsv for plain .tsv files.
#
# It prints one manifest line per file written or already present: key <TAB> owner/repo <TAB> path.
# Errors go to stderr: "ERR clone <repo>", "ERR nohistory <repo> <pattern>", or "ERR blob <repo>
# <path> <sha>" when a version's content could not be fetched (that file is not written).
# Each file's rows come from its own single-path git log, so a file's history is the same
# whether you asked for it by name or through a glob.
set -uo pipefail
repo=${1:?owner/repo}; out=${2:?output dir}; shift 2
[ $# -gt 0 ] || { echo "usage: history.sh OWNER/REPO OUTDIR PATTERN..." >&2; exit 2; }
mkdir -p "$out/vers"
out=$(cd "$out" && pwd)
ext=tsv.gz; [ "${HISTORY_FORMAT:-gz}" = tsv ] && ext=tsv
keyof() { local p=$1; echo "${repo//\//_}__${p//\//_}"; }
isglob() { case "$1" in *[\*\?\[]*) return 0;; *) return 1;; esac; }

# Literal paths that are already collected need no clone.
need=0
for pat in "$@"; do
  if isglob "$pat"; then need=1
  elif [ -s "$out/vers/$(keyof "$pat").$ext" ]; then printf '%s\t%s\t%s\n' "$(keyof "$pat")" "$repo" "$pat"
  else need=1; fi
done
[ $need = 1 ] || exit 0

d=$(mktemp -d)
trap 'rm -rf "$d" 2>/dev/null' EXIT
if ! timeout "${CLONE_TIMEOUT:-300}" git clone -q --bare --filter=blob:none "https://github.com/$repo.git" "$d/g" 2>/dev/null; then
  echo "ERR clone $repo" >&2
  exit 0
fi
cd "$d/g" || exit 0

# glob -> anchored ERE
glob_re() {
  local g=$1 r="" c i
  for ((i = 0; i < ${#g}; i++)); do
    c=${g:i:1}
    if [ "$c" = "*" ] && [ "${g:i+1:1}" = "*" ]; then
      if [ "${g:i+2:1}" = "/" ]; then r+="(.*/)?"; i=$((i + 2)); else r+=".*"; i=$((i + 1)); fi
    elif [ "$c" = "*" ]; then r+="[^/]*"
    elif [ "$c" = "?" ]; then r+="[^/]"
    elif [ "$c" = "[" ]; then r+="["
    elif [ "$c" = "]" ]; then r+="]"
    elif [[ "$c" == [.^\$+\(\)\{\}\|\\] ]]; then r+="\\$c"
    else r+="$c"; fi
  done
  echo "^$r\$"
}

paths=()
for pat in "$@"; do
  if isglob "$pat"; then
    re=$(glob_re "$pat")
    mapfile -t found < <(git log --format= --name-only --no-renames 2>/dev/null | grep -E "$re" | sort -u)
    [ ${#found[@]} -gt 0 ] || { echo "ERR nohistory $repo $pat" >&2; continue; }
    paths+=("${found[@]}")
  else
    paths+=("$pat")
  fi
done

zero=0000000000000000000000000000000000000000
[ ${#paths[@]} -gt 0 ] || exit 0          # every pattern was a glob that matched nothing (already logged)
printf '%s\n' "${paths[@]}" | awk '!seen[$0]++' > "$d/paths.txt"

# Pass 1: the commits that touched each file, and the blob each one left.
n=0
: > "$d/todo.tsv"
while IFS= read -r path <&3; do
  key=$(keyof "$path")
  dest="$out/vers/$key.$ext"
  if [ -s "$dest" ]; then printf '%s\t%s\t%s\n' "$key" "$repo" "$path"; continue; fi
  n=$((n + 1))
  # One line per (commit, blob of PATH after the commit). --raw gives the new blob id.
  # Default history simplification first (git log's usual view of one path). Only if that finds
  # nothing, e.g. a file that lived only on a merged side branch, retry with --full-history.
  for mode in "" "--full-history"; do
    git log $mode --format='C%x09%H%x09%aI%x09%(trailers:key=Co-authored-by,valueonly,separator=;)' \
        --raw --no-abbrev --no-renames -- "$path" 2>/dev/null \
      | awk -F'\t' -v f="$path" '
          /^C\t/ { sha=$2; date=$3; co=(tolower($4) ~ /claude/) ? 1 : 0; next }
          /^:/   { split($1, a, " "); if ($2 == f) print sha "\t" date "\t" co "\t" a[4] }' > "$d/c$n.tsv"
    [ -s "$d/c$n.tsv" ] && break
  done
  if [ ! -s "$d/c$n.tsv" ]; then
    echo "ERR nohistory $repo $path" >&2
    continue
  fi
  printf '%s\t%s\n' "$n" "$path" >> "$d/todo.tsv"
done 3< "$d/paths.txt"

# Pass 2: fetch every needed blob in a few batches instead of one round trip per blob.
# If the server refuses a batch, git falls back to fetching each blob when it is read.
cat "$d"/c[0-9]*.tsv 2>/dev/null | cut -f4 | grep -v "^$zero\$" | sort -u > "$d/oids.txt"
if [ -s "$d/oids.txt" ]; then
  xargs -n 500 git -c fetch.negotiationAlgorithm=noop fetch -q --no-tags --no-write-fetch-head \
      --recurse-submodules=no --filter=blob:none origin < "$d/oids.txt" >/dev/null 2>&1 || true
fi

# Pass 3: write each file's rows. A blob that cannot be read is an error, never a deletion:
# the file is left out and logged as "ERR blob", so a re-run tries it again.
while IFS=$'\t' read -r i path <&3; do
  key=$(keyof "$path")
  dest="$out/vers/$key.$ext"
  ok=1
  while IFS=$'\t' read -r sha date co oid; do
    if [ "$oid" = "$zero" ]; then b=""
    elif ! b=$(git cat-file blob "$oid" 2>/dev/null | base64 -w0; exit "${PIPESTATUS[0]}"); then
      echo "ERR blob $repo $path $sha" >&2; ok=0; break
    fi
    printf '%s\t%s\t%s\t%s\n' "$sha" "$date" "$co" "$b"
  done < "$d/c$i.tsv" > "$d/rows.tsv"
  [ $ok = 1 ] || continue
  if [ "$ext" = tsv ]; then mv "$d/rows.tsv" "$dest"; else gzip -n < "$d/rows.tsv" > "$dest.tmp" && mv "$dest.tmp" "$dest"; fi
  printf '%s\t%s\t%s\n' "$key" "$repo" "$path"
done 3< "$d/todo.tsv"