#!/usr/bin/env python3
"""Bulk clone all member repos of gitlab.npontutechnologies.com (engagement one-off).

History-mining layer (skill pitfall 10): blob search reads HEAD only; deleted
secrets (.env committed then removed) need full git history. Clone auth:
https://oauth2:<token>@host/<path>.git (http.extraHeader Bearer does NOT work).

Per-repo checkpoint: skip if <path>.git exists (resume-safe). Shallow is NOT
used — we need full history. Logs failures, never aborts the sweep on one repo.
Read-only against the server.
"""
import json, os, subprocess, sys, time
from pathlib import Path

ROOT = Path('/root/ir-assessment/redteam/gitlab_npontutechnologies_com')
REPOS = ROOT / 'repos'
LOG = ROOT / 'clone_run.log'
TOK = (ROOT / '.token').read_text().strip()
HOST = 'gitlab.npontutechnologies.com'

def main():
    REPOS.mkdir(exist_ok=True)
    projects = json.loads((ROOT / 'L2.json').read_text())['projects']
    log = LOG.open('a')
    log.write(f'=== clone run {time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime())} repos={len(projects)} ===\n')
    ok = skip = fail = 0
    for i, p in enumerate(projects, 1):
        path = p['path']
        dest = REPOS / (path.replace('/', '__') + '.git')
        if dest.exists():
            skip += 1
            continue
        url = f'https://oauth2:{TOK}@{HOST}/{path}.git'
        t0 = time.time()
        r = subprocess.run(['git', 'clone', '--quiet', url, str(dest)],
                           capture_output=True, text=True, timeout=600)
        dt = time.time() - t0
        if r.returncode == 0:
            sz = sum(f.stat().st_size for f in dest.rglob('*') if f.is_file()) // 1024
            ok += 1
            line = f'[{i}/{len(projects)}] OK {path} ({sz} KB, {dt:.0f}s)'
        else:
            fail += 1
            err = r.stderr.strip().replace(TOK, '<TOKEN>')[:200]
            if dest.exists():
                subprocess.run(['rm', '-rf', str(dest)])
            line = f'[{i}/{len(projects)}] FAIL {path}: {err}'
        print(line, flush=True)
        log.write(line + '\n'); log.flush()
    line = f'=== done: ok={ok} skip={skip} fail={fail} ==='
    print(line); log.write(line + '\n'); log.close()

if __name__ == '__main__':
    main()