GitHub file history

#!/usr/bin/env bash
# collect.sh LIST OUTDIR [JOBS]
#
# LIST: one line per wanted file as owner/repo/PATTERN, where PATTERN is a path or a glob
# (README.md, package.json, .cursor/rules/*.mdc, docs/**/*.md). Blank lines and lines starting
# with # are ignored. Paths with spaces are not supported. make_list.py writes this file.
#
# Lines are grouped by repo, so each repo is cloned once however many patterns it has.
# history.sh runs JOBS repos at a time (default 8).
# Writes OUTDIR/manifest.tsv (key, repo, path: one line per file collected) and OUTDIR/err.log.
# Safe to re-run: finished files are skipped, and literal paths already on disk need no clone.
set -uo pipefail
list=${1:?list of owner/repo/pattern}; out=${2:?output dir}; jobs=${3:-8}
here=$(cd "$(dirname "$0")" && pwd)
mkdir -p "$out/vers"
out=$(cd "$out" && pwd)
: > "$out/err.log"

# one line per repo: owner/repo pattern1 pattern2 ...
grep -v -E '^[[:space:]]*(#|$)' "$list" | awk '{
    n = split($0, a, "/"); repo = a[1] "/" a[2]; pat = substr($0, length(repo) + 2)
    if (!(repo in seen)) { seen[repo] = 1; order[++k] = repo }
    pats[repo] = pats[repo] " " pat
  } END { for (i = 1; i <= k; i++) print order[i] pats[order[i]] }' > "$out/.repos"

export HISTORY_OUT="$out" HISTORY_SH="$here/history.sh"
# xargs -L1 passes each repo line as arguments; history.sh prints manifest lines on stdout.
xargs -P "$jobs" -L 1 bash -c 'bash "$HISTORY_SH" "$0" "$HISTORY_OUT" "$@"' < "$out/.repos" \
  > "$out/.manifest" 2>>"$out/err.log"
sort -u "$out/.manifest" > "$out/manifest.tsv"
rm -f "$out/.manifest" "$out/.repos"

listed=$(grep -c -v -E '^[[:space:]]*(#|$)' "$list")
echo "listed $listed  files collected $(wc -l < "$out/manifest.tsv")  errors $(grep -c "^ERR" "$out/err.log")"