#!/usr/bin/env python3
"""L3 history hunt, second source (independent of trufflehog):
git log -G<regex> -p across ALL refs of every mirror. Checkpoint per repo.
Patterns: glpat-, TG bot tokens, JWT, PRIVATE KEY blocks, AKIA, sk_live/rk_live,
kv secret assignments. Diffs stored with full values (no-masking)."""
import json, re, subprocess, sys, time
from pathlib import Path

D = Path('/root/ir-assessment/redteam/gitlab_hyva_io')
CD = D / 'L3_clone'
OUT = D / 'L3_grep'; OUT.mkdir(exist_ok=True)
repos = sorted(CD.rglob('*.git'))

PATTERNS = {
 'glpat':    r'glpat-[\w-]{15,}',
 'tg_bot':   r'\d{8,10}:AA[\w-]{33}',
 'jwt':      r'eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{5,}',
 'privkey':  r'BEGIN [A-Z ]*PRIVATE KEY',
 'aws':      r'AKIA[0-9A-Z]{16}',
 'stripe':   r'[sr]k_live_[0-9a-zA-Z]{16,}',
 'secret_kv':r'(password|passwd|api_key|apikey|secret_key|access_key)\s*[:=]\s*["\'][A-Za-z0-9/+_=.!@#-]{8,}["\']',
}
# benign value denylist for secret_kv
DENY = re.compile(r'(?i)(your_|enter_|change_me|xxx|example|sample|placeholder|tobemodified|_here|<.*>|%7B|__|\.\.\.)')

def hunt(repo):
    n = repo.relative_to(CD).as_posix()
    out = OUT / (n.replace('/', '__') + '.json')
    if out.exists():
        return 'cached', 0
    hits = []
    t0 = time.time()
    for name, pat in PATTERNS.items():
        r = subprocess.run(
            ['git', '--git-dir', str(repo), 'log', '--all', '-G', pat, '-p', '--no-color',
             '-U1', '--', '*.php' '*.phtml' '*.xml' '*.yml' '*.yaml' '*.env*' '*.json'
             '*.js' '*.ts' '*.cfg' '*.ini' '*.conf' '*.txt' '*.md' '*.sh'],
            capture_output=True, text=True, timeout=900)
        if r.returncode not in (0, 1):
            continue
        rx = re.compile(pat)
        for block in r.stdout.split('commit ')[1:]:
            sha = block.split('\n', 1)[0].strip()
            for line in block.splitlines():
                if not line.startswith(('+', '-')) or line.startswith(('+++', '---')):
                    continue
                for m in rx.finditer(line):
                    val = m.group(0)
                    if name == 'secret_kv' and DENY.search(val):
                        continue
                    hits.append({'repo': n, 'pattern': name, 'sha': sha,
                                 'side': line[0], 'line': line[1:][:400]})
    out.write_text(json.dumps(hits, ensure_ascii=False))
    return len(hits), time.time() - t0

def main():
    todo = [r for r in repos if not (OUT / (r.relative_to(CD).as_posix().replace('/', '__') + '.json')).exists()]
    print(f"[*] {len(repos)} repos, {len(todo)} to hunt (resume)", file=sys.stderr)
    tot = 0; t0 = time.time()
    for i, repo in enumerate(todo, 1):
        res, dt = hunt(repo)
        if res != 'cached': tot += res
        if i % 50 == 0:
            rate = i / (time.time() - t0)
            print(f"  ... {i}/{len(todo)} hits={tot} eta={(len(todo)-i)/rate/60:.0f}min", flush=True)
    merged = []
    for f in OUT.glob('*.json'):
        merged.extend(json.load(open(f)))
    (D / 'L3_history_hits.json').write_text(json.dumps(merged, ensure_ascii=False, indent=1))
    # distinct values summary
    seen = {}
    for h in merged:
        seen.setdefault(h['line'].strip()[:200], []).append(h['repo'])
    print(f"[+] DONE hits={len(merged)} distinct_lines={len(seen)}")

if __name__ == '__main__':
    main()