#!/usr/bin/env python3
"""L3 history scan: trufflehog over 544 mirror clones. Checkpoint per repo.
Outputs: L3_th/<n>.json (all findings), merged L3_trufflehog.jsonl + verified-only file."""
import json, subprocess, sys, time
from pathlib import Path

D = Path('/root/ir-assessment/redteam/gitlab_hyva_io')
CD = D / 'L3_clone'
OUT = D / 'L3_th'; OUT.mkdir(exist_ok=True)
repos = sorted(CD.rglob('*.git'))

def scan(repo):
    n = repo.relative_to(CD).as_posix()
    out = OUT / (n.replace('/', '__') + '.json')
    if out.exists():
        return 'cached', 0
    t0 = time.time()
    r = subprocess.run(
        ['trufflehog', 'filesystem', '--json', '--concurrency=4', '--no-update', str(repo)],
        capture_output=True, text=True, timeout=900)
    lines = [ln for ln in r.stdout.splitlines() if ln.strip()]
    recs = []
    for ln in lines:
        try:
            j = json.loads(ln)
            if 'SourceMetadata' in j:  # finding, not log line
                recs.append(j)
        except json.JSONDecodeError:
            pass
    out.write_text(json.dumps(recs, ensure_ascii=False))
    return len(recs), time.time() - t0

def main():
    todo = [r for r in repos if not (OUT / (r.relative_to(CD).as_posix().replace('/', '__') + '.json')).exists()]
    print(f"[*] {len(repos)} repos, {len(todo)} to scan (resume)", file=sys.stderr)
    tot = 0; t0 = time.time()
    for i, repo in enumerate(todo, 1):
        res, dt = scan(repo)
        if res != 'cached':
            tot += res
        if i % 50 == 0:
            rate = i / (time.time() - t0)
            print(f"  ... {i}/{len(todo)} findings_so_far={tot} eta={(len(todo)-i)/rate/60:.0f}min", flush=True)
    merged, verified = [], []
    for f in OUT.glob('*.json'):
        for j in json.load(open(f)):
            merged.append(j)
            if j.get('Verified'):
                verified.append(j)
    (D / 'L3_trufflehog_all.json').write_text(json.dumps(merged, ensure_ascii=False, indent=1))
    (D / 'L3_trufflehog_verified.json').write_text(json.dumps(verified, ensure_ascii=False, indent=1))
    print(f"[+] DONE all_findings={len(merged)} verified={len(verified)}")

if __name__ == '__main__':
    main()