#!/usr/bin/env python3
"""Clone all enumerated GitLab repos (shallow depth=1). Try each token until one works.
Uses https://oauth2:TOKEN@gitlab.com/group/repo.git form.
Saves clone log to _clone_log.json.
"""
import json, os, subprocess, time
from concurrent.futures import ThreadPoolExecutor, as_completed

OUTDIR = "/root/ir-assessment/redteam/camelsoft/gitlab_repos"
TOKENS_ORDERED = [
    "glpat-sgCEpY3CBBhrAJLFAwcxmG86MQp1OjN2Mm1rCw.01.120ci2qti",      # CamelSoft
    "glpat-6UyneVwq4emxXZnpFK7-2G86MQp1Omp4cXZhCw.01.12075cpxn",      # mohamedkhabir
    "glpat-paHb-IMIGMOBkWdimvocHWM6MQpvOjEKdTozdjJtaw8.01.1709yym89", # CamelSoft (dup)
    "glpat-4QqtQ3LRuYkw8lngMcv4GmM6MQpvOjEKdTo3eTBxZA8.01.170sjfbyb", # ChouaibiMarwen
]

def clone_one(project):
    pn = project["path_with_namespace"]
    local_dir = os.path.join(OUTDIR, pn.replace("/", "__"))
    log = {"path": pn, "local_dir": local_dir, "status": "pending", "token_used": None, "error": None}

    if os.path.isdir(local_dir) and os.path.isdir(os.path.join(local_dir, ".git")):
        log["status"] = "already_cloned"
        return log

    # remove partial
    if os.path.isdir(local_dir):
        subprocess.run(["rm", "-rf", local_dir], capture_output=True)

    for tok in TOKENS_ORDERED:
        url = f"https://oauth2:{tok}@gitlab.com/{pn}.git"
        env = dict(os.environ, GIT_TERMINAL_PROMPT="0", GIT_LFS_SKIP_SMUDGE="1")
        # shallow clone, default branch
        try:
            r = subprocess.run(
                ["git", "clone", "--depth=1", "--filter=blob:none", url, local_dir],
                capture_output=True, text=True, timeout=120, env=env
            )
            if r.returncode == 0:
                log["status"] = "cloned"
                log["token_used"] = tok[:14] + "..."
                return log
            else:
                err = (r.stderr or "").strip().split("\n")
                # filter out token from error
                err_clean = [l for l in err if "oauth2:" not in l and "glpat-" not in l]
                log["error"] = "; ".join(err_clean)[:300]
                if "not found" in (r.stderr or "").lower() or "403" in (r.stderr or "") or "404" in (r.stderr or ""):
                    continue  # try next token
                # for other errors, also try next token
                continue
        except subprocess.TimeoutExpired:
            log["error"] = "timeout (120s)"
            continue
        except Exception as e:
            log["error"] = str(e)[:200]
            continue

    log["status"] = "failed_all_tokens"
    return log

def main():
    with open(os.path.join(OUTDIR, "_projects_to_clone.json")) as f:
        projects = json.load(f)
    print(f"Cloning {len(projects)} repos with up to 6 parallel workers...", flush=True)

    logs = []
    done = 0
    with ThreadPoolExecutor(max_workers=6) as ex:
        futs = {ex.submit(clone_one, p): p["path_with_namespace"] for p in projects}
        for fut in as_completed(futs):
            log = fut.result()
            logs.append(log)
            done += 1
            mark = "OK" if log["status"] in ("cloned", "already_cloned") else "FAIL"
            if done % 10 == 0 or mark == "FAIL":
                print(f"  [{done}/{len(projects)}] {mark}: {log['path']} -> {log['status']}", flush=True)

    logs.sort(key=lambda l: l["path"])
    with open(os.path.join(OUTDIR, "_clone_log.json"), "w") as f:
        json.dump(logs, f, indent=2)

    cloned = sum(1 for l in logs if l["status"] in ("cloned", "already_cloned"))
    failed = sum(1 for l in logs if l["status"] == "failed_all_tokens")
    print(f"\n=== CLONE COMPLETE ===")
    print(f"Cloned/exists: {cloned}/{len(logs)}")
    print(f"Failed: {failed}")
    if failed:
        print("Failed repos:")
        for l in logs:
            if l["status"] == "failed_all_tokens":
                print(f"  - {l['path']}: {l.get('error','')[:120]}")

if __name__ == "__main__":
    main()