#!/usr/bin/env python3
"""gitlab.hyva.io L2 blob-search pass — per-project checkpoint, resume-safe.

User-level variant (is_admin absent on 18.11.7, 403 on /variables + /runners).
544 membership projects x 11 patterns. Read-only GETs only.
Checkpoints: L2_blobs/<project_id>.json — re-run picks up missing ids only.
"""
import json, re, ssl, sys, time, urllib.error, urllib.parse, urllib.request
from pathlib import Path

BASE = 'https://gitlab.hyva.io'
D = Path('/root/ir-assessment/redteam/gitlab_hyva_io')
CK = D / 'L2_blobs'; CK.mkdir(exist_ok=True)
TOK = (D / '.token').read_text().strip()
CTX = ssl.create_default_context(); CTX.check_hostname = False; CTX.verify_mode = ssl.CERT_NONE
H = {'Authorization': f'Bearer {TOK}', 'User-Agent': 'l2-blob/1.0'}

PATTERNS = ['password','passwd','secret','api_key','apikey','token','private_key','BEGIN','AKIA','access_key','secret_key']
NOISE = re.compile(r'\.data-api|b03f5f7f11d50a3a|process\.env\.|env\(\'|tobemodified|node_modules|vendor/|\.lock|package-lock|yarn\.lock|CHANGELOG|\.min\.js', re.I)
INTERESTING = re.compile(r'glpat-|\d{8,10}:AA[\w-]{33}|eyJ[A-Za-z0-9_-]{10,}\.|BEGIN [A-Z ]*PRIVATE KEY|AKIA[0-9A-Z]{16}|(password|passwd|secret|api_key|apikey|token|access_key|secret_key)\s*[:=]\s*["\']?[^\s"\']{6,}', re.I)


def get(path):
    req = urllib.request.Request(BASE + path, headers=H)
    for a in (1, 2, 3):
        try:
            with urllib.request.urlopen(req, timeout=25, context=CTX) as r:
                return json.loads(r.read()), 200
        except urllib.error.HTTPError as e:
            if e.code in (429, 500, 502, 503) and a < 3:
                time.sleep(5 * a); continue
            return None, e.code
        except Exception:
            if a < 3:
                time.sleep(2 * a); continue
            return None, 0
    return None, 0


def scan(p):
    ck = CK / f"{p['id']}.json"
    if ck.exists():
        return json.load(open(ck))
    hits, seen, errs = [], set(), []
    for q in PATTERNS:
        r, st = get(f"/api/v4/projects/{p['id']}/search?scope=blobs&search={urllib.parse.quote(q)}&per_page=20")
        if not isinstance(r, list):
            errs.append(f'{q}:http{st}')
            continue
        for b in r:
            k = (b.get('path'), b.get('ref'))
            if k in seen:
                continue
            seen.add(k)
            data = (b.get('data') or '')[:400]
            if NOISE.search(data) and not INTERESTING.search(data):
                continue
            hits.append({'proj': p['path'], 'file': b.get('path'), 'ref': b.get('ref'), 'q': q, 'data': data})
        time.sleep(0.4)
    rec = {'hits': hits, 'errors': errs}
    ck.write_text(json.dumps(rec, ensure_ascii=False))
    return rec


def main():
    l2 = json.load(open(D / 'L2.json'))
    projs = l2['projects']
    todo = [p for p in projs if not (CK / f"{p['id']}.json").exists()]
    print(f"[*] {len(projs)} projects, {len(todo)} to scan (resume)", file=sys.stderr)
    done = len(projs) - len(todo)
    t0 = time.time()
    for i, p in enumerate(todo, 1):
        rec = scan(p)
        done += 1
        n = len(rec['hits'])
        if n:
            print(f"  [{p['path']}] {n} hits", flush=True)
        if i % 20 == 0:
            rate = i / (time.time() - t0)
            eta = (len(todo) - i) / rate if rate else 0
            print(f"  ... {done}/{len(projs)}  eta={eta/60:.0f}min", flush=True)
    merged, all_err = [], {}
    for p in projs:
        ck = CK / f"{p['id']}.json"
        if ck.exists():
            rec = json.load(open(ck))
            merged.extend(rec['hits'])
            if rec['errors']:
                all_err[p['path']] = rec['errors']
    (D / 'L2_blobs.json').write_text(json.dumps(merged, ensure_ascii=False, indent=1))
    l2['blob_hits'] = merged
    l2['blob_hits_count'] = len(merged)
    l2['blob_errors'] = all_err
    l2['blob_pass_complete'] = '2026-08-10'
    (D / 'L2.json').write_text(json.dumps(l2, indent=1, ensure_ascii=False))
    print(f"[+] DONE {len(projs)} projects, {len(merged)} hits, {len(all_err)} projects with errors")


if __name__ == '__main__':
    main()