#!/usr/bin/env python3
"""Offline secret scan over bulk-cloned repos — working trees + FULL git history.

Two layers:
  A) working tree: regex over all text files (fast, current state)
  B) git history: `git grep <pattern> $(git rev-list --all)` per repo → secrets that
     were committed then deleted (never visible in HEAD). This is the payoff layer.

Patterns: high-signal secret shapes (not generic words). Dedup by (value) — we want
distinct secrets, then map to repos/paths/commits. No-masking per repo rules.
Output: offline_secrets.json + offline_secrets.md in the dossier root.
"""
import json
import re
import subprocess
import sys
import time
from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path

HERE = Path(__file__).resolve().parent
REPOS = HERE / "repos"
OUT_JSON = HERE / "offline_secrets.json"
OUT_MD = HERE / "offline_secrets.md"

# high-signal secret patterns (value-capturing). Avoid generic "password"-word scans here —
# that's layer-A via key=value. History layer uses fixed-string grep on distinctive markers.
VALUE_PATTERNS = [
    ("aws_aki",        re.compile(r"\b(AKIA[0-9A-Z]{16})\b")),
    ("gcp_api_key",    re.compile(r"\b(AIza[0-9A-Za-z_\-]{35})\b")),
    ("gitlab_pat",     re.compile(r"\b(glpat-[0-9A-Za-z_\-]{20,})\b")),
    ("gitlab_runner",  re.compile(r"\b(glrt-[0-9A-Za-z_\-]{20,})\b")),
    ("github_pat",     re.compile(r"\b((?:ghp|gho|ghu|ghs|ghr)_[0-9A-Za-z]{36,})\b")),
    ("slack_token",    re.compile(r"\b(xox[baprs]-[0-9A-Za-z\-]{10,})\b")),
    ("openai",         re.compile(r"\b(sk-(?:proj-)?[0-9A-Za-z_\-]{20,})\b")),
    ("stripe",         re.compile(r"\b([sr]k_(?:live|test)_[0-9A-Za-z]{16,})\b")),
    ("jwt",            re.compile(r"\b(eyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{8,})\b")),
    ("priv_key",       re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY")),
    ("azure_conn",     re.compile(r"(DefaultEndpointsProtocol=https;AccountName=[^;\"']+;AccountKey=[^;\"']+)")),
    ("basic_auth_url", re.compile(r"https?://([^/\s:@]{3,}:[^/\s@]{6,})@[0-9A-Za-z.\-]+")),
]
# key=value secrets (working tree + history): PASSWORD=..., api_key: "..."
KV_RE = re.compile(
    r"(?i)\b(pass(?:word|wd)?|secret|api[_-]?key|apikey|token|private[_-]?key|access[_-]?key|"
    r"client[_-]?secret|jwt|salt|auth[_-]?key|db[_-]?pass|smtp[_-]?pass)\b"
    r"\s*[:=]\s*[\"']?([^\s\"',}]{8,})")
NOISE = re.compile(r"(?ix)(^\.data-api$|publickeytoken|^b03f5f7f11d50a3a$|process\.env|^env\(|"
                   r"^isset\(|^config\.|tobemodified|placeholder|your[-_]|\$\{|<%|<\?|\{\{)")
HISTORY_MARKERS = ["PRIVATE KEY-----", "AKIA", "AIza", "glpat-", "xoxb-", "sk_live",
                   "BEGIN RSA", "AccountKey=", "PASSWORD=", "api_key=", "secret="]

def is_real(v):
    if not v or len(v) < 8 or len(v) > 400 or NOISE.search(v):
        return False
    return bool(re.search(r"[a-zA-Z]", v) and re.search(r"[0-9_+/=.@$-]", v))

def scan_text(text):
    found = []
    for name, rx in VALUE_PATTERNS:
        for m in rx.finditer(text):
            v = m.group(1) if m.groups() else m.group(0)
            if name in ("priv_key",):
                found.append((name, "-----BEGIN PRIVATE KEY-----"))
            elif is_real(v):
                found.append((name, v))
    for m in KV_RE.finditer(text):
        if is_real(m.group(2)):
            found.append((f"kv:{m.group(1).lower()}", m.group(2)))
    return found

def scan_repo(repo_dir):
    path = repo_dir.name.replace("__", "/")
    hits = []
    # A) working tree
    for f in repo_dir.rglob("*"):
        if not f.is_file() or ".git" in f.parts:
            continue
        try:
            if f.stat().st_size > 2_000_000:
                continue
            data = f.read_bytes()
            if b"\x00" in data[:2048]:
                continue  # binary
            text = data.decode("utf-8", "ignore")
        except Exception:
            continue
        for kind, val in scan_text(text):
            hits.append({"repo": path, "layer": "worktree", "file": str(f.relative_to(repo_dir)),
                         "kind": kind, "value": val})
    # B) history via git grep over all revs
    try:
        revs = subprocess.run(["git", "-C", str(repo_dir), "rev-list", "--all"],
                              capture_output=True, text=True, timeout=60).stdout.split()
    except Exception:
        revs = []
    if revs:
        for marker in HISTORY_MARKERS:
            try:
                p = subprocess.run(["git", "-C", str(repo_dir), "grep", "-I", "-n", "-F", marker] + revs,
                                   capture_output=True, text=True, timeout=120)
            except Exception:
                continue
            for line in p.stdout.splitlines():
                # <rev>:<path>:<lineno>:<content>
                parts = line.split(":", 3)
                if len(parts) < 4:
                    continue
                rev, fpath, content = parts[0], parts[1], parts[3]
                for kind, val in scan_text(content):
                    hits.append({"repo": path, "layer": "history", "file": fpath,
                                 "commit": rev[:10], "kind": kind, "value": val})
    return path, hits

def main():
    t0 = time.time()
    repos = [d for d in REPOS.iterdir() if d.is_dir()]
    print(f"[+] scanning {len(repos)} repos (worktree + history)", file=sys.stderr)
    all_hits = []
    with ProcessPoolExecutor(max_workers=8) as ex:
        futs = {ex.submit(scan_repo, d): d for d in repos}
        for i, f in enumerate(as_completed(futs), 1):
            path, hits = f.result()
            all_hits.extend(hits)
            if hits:
                hist = sum(1 for h in hits if h["layer"] == "history")
                print(f"  [{i}/{len(repos)}] {path}: {len(hits)} (hist={hist})", file=sys.stderr)
    # dedup distinct values, keep provenance
    by_val = {}
    for h in all_hits:
        key = (h["kind"], h["value"])
        e = by_val.setdefault(key, {"kind": h["kind"], "value": h["value"], "seen": []})
        e["seen"].append({k: h[k] for k in ("repo", "layer", "file") if k in h} |
                         ({"commit": h["commit"]} if h.get("commit") else {}))
    distinct = list(by_val.values())
    # dedup 'seen' lists
    for e in distinct:
        uniq, seen = [], set()
        for s in e["seen"]:
            k = json.dumps(s, sort_keys=True)
            if k not in seen:
                seen.add(k); uniq.append(s)
        e["seen"] = uniq[:25]
    out = {"scan": time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime()),
           "repos": len(repos), "raw_hits": len(all_hits), "distinct_secrets": len(distinct),
           "secrets": sorted(distinct, key=lambda e: (e["kind"], e["value"]))}
    OUT_JSON.write_text(json.dumps(out, indent=1, ensure_ascii=False))
    md = [f"# Offline secrets scan (worktree + git history)",
          f"- repos: {len(repos)}, raw hits: {len(all_hits)}, distinct secrets: {len(distinct)}", ""]
    for e in out["secrets"]:
        layers = {s["layer"] for s in e["seen"]}
        repos_hit = sorted({s["repo"] for s in e["seen"]})
        md.append(f"## `{e['kind']}` = `{e['value']}`")
        md.append(f"- layers: {', '.join(sorted(layers))} | repos: {', '.join(repos_hit[:6])}")
        ex = e["seen"][0]
        md.append(f"- e.g. {ex['repo']} :: {ex['file']}" + (f" @{ex.get('commit')}" if ex.get("commit") else ""))
        md.append("")
    OUT_MD.write_text("\n".join(md))
    print(f"[+] distinct secrets: {len(distinct)} (raw {len(all_hits)}) in {time.time()-t0:.0f}s", file=sys.stderr)
    print(f"[+] → {OUT_JSON}\n[+] → {OUT_MD}", file=sys.stderr)

if __name__ == "__main__":
    main()