#!/usr/bin/env python3
"""Blob-search pass 2 for gitlab.multistackexpert.com — sequential, 429-retry/backoff.
Engagement one-off. Read-only GETs only. Resumes from L2.json (dedup by project,path).
"""
import json, re, ssl, sys, time, urllib.request, urllib.parse, urllib.error
from pathlib import Path

BASE = 'https://gitlab.multistackexpert.com'
D = Path('/root/ir-assessment/redteam/gitlab_multistackexpert_com')
CTX = ssl.create_default_context(); CTX.check_hostname = False; CTX.verify_mode = ssl.CERT_NONE
UA = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) ir-assessment-l2'}
tok = (D / '.token').read_text().strip()
H = {'Authorization': f'Bearer {tok}'}

BLOB_PATTERNS = ['password','passwd','secret','api_key','apikey','token','private_key','BEGIN','AKIA','access_key','secret_key']
NOISE = re.compile(r'\.data-api|b03f5f7f11d50a3a|process\.env\.|env\(\'|tobemodified|lockfile|\.lock|vendor/', re.I)
INTERESTING = re.compile(r'glpat-|\d{8,10}:AA[\w-]{33}|eyJ[A-Za-z0-9_-]{10,}\.|BEGIN [A-Z ]*PRIVATE KEY|AKIA[0-9A-Z]{16}|(password|passwd|secret|api_key|apikey|token|access_key|secret_key)\s*[:=]\s*["\']?[^\s"\']{6,}', re.I)

def getj_retry(path, max_attempts=8):
    for att in range(max_attempts):
        r = urllib.request.Request(BASE + path, headers=dict(UA, **H))
        try:
            with urllib.request.urlopen(r, timeout=25, context=CTX) as resp:
                return json.loads(resp.read().decode('utf-8', errors='ignore')), resp.status
        except urllib.error.HTTPError as e:
            if e.code == 429:
                ra = e.headers.get('Retry-After')
                wait = int(ra) if ra and ra.isdigit() else 20 * (att + 1)
                time.sleep(wait)
                continue
            return None, e.code
        except Exception:
            return None, 0
    return None, 429

def main():
    l2 = json.loads((D / 'L2.json').read_text())
    projs = [(p['id'], p['path']) for p in l2['projects']]
    seen = set()
    for h in l2['blob_hits']:
        pid = next((p['id'] for p in l2['projects'] if p['path'] == h['proj']), None)
        if pid: seen.add((pid, h['path']))
    new_hits, still_err, total_req = [], {}, 0
    t0 = time.time()
    for pid, ppath in projs:
        perr = []
        for q in BLOB_PATTERNS:
            d, st = getj_retry(f"/api/v4/projects/{pid}/search?scope=blobs&search={urllib.parse.quote(q)}&per_page=20")
            total_req += 1
            if not isinstance(d, list):
                perr.append(f'{q}:http{st}')
                continue
            for b in d:
                key = (pid, b.get('path'))
                if key in seen: continue
                seen.add(key)
                data = b.get('data','')[:400]
                if NOISE.search(data) and not INTERESTING.search(data): continue
                new_hits.append({'proj': ppath, 'path': b.get('path'), 'pattern': q, 'snippet': data})
            time.sleep(0.7)  # gentle pacing between pattern queries
        if perr: still_err[ppath] = perr
        # checkpoint after EVERY project — a kill loses nothing
        l2['blob_hits'].extend(new_hits); new_hits.clear()
        l2['blob_hits_count'] = len(l2['blob_hits'])
        l2['blob_errors_after_retry'] = still_err
        (D / 'L2.json').write_text(json.dumps(l2, indent=1, ensure_ascii=False))
        print(f'[{time.strftime("%H:%M:%S")}] {ppath} done, total_hits={l2["blob_hits_count"]} errs={len(perr)}', flush=True)
    l2['blob_pass2_complete'] = '2026-08-10'
    (D / 'L2.json').write_text(json.dumps(l2, indent=1, ensure_ascii=False))
    print(f'DONE reqs={total_req} total_blob_hits={l2["blob_hits_count"]} err_projects={len(still_err)} elapsed={time.time()-t0:.0f}s', flush=True)

if __name__ == '__main__':
    main()