#!/usr/bin/env python3 """Corporate access search in stealer logs (ULP + extracted archives). Searches for corporate/infra entry points using categorized patterns. Outputs matches grouped by category with deduplication. Usage: python3 corp_search.py # Search all sources python3 corp_search.py --category infra # Search only infra category python3 corp_search.py --pattern "mycompany.com" # Ad-hoc pattern search python3 corp_search.py --list-categories # List all categories python3 corp_search.py --tsv X.tsv --workers 8 # Parallel scan of one large TSV """ import argparse import csv import mmap import multiprocessing as mp import os import re import sys from collections import defaultdict, Counter from concurrent.futures import ProcessPoolExecutor from pathlib import Path PROJECT_DIR = Path(__file__).parent.parent ULP_DIR = PROJECT_DIR / "findings" / "telegram_logs" / "cloudxlog" BOXED_DIR = PROJECT_DIR / "findings" / "telegram_logs" / "boxed_pw" csv.field_size_limit(10 * 1024 * 1024) # ======================================================================== # PATTERN DATABASE # Organised by category. Each entry: (pattern, label, notes) # # To add new patterns: append to the relevant category list. # Patterns are case-insensitive substring matches against the URL field. # ======================================================================== PATTERNS = { # ------------------------------------------------------------------ # 1. Known corporate targets (from assessment) # ------------------------------------------------------------------ "corp_known": [ ("adar.pl", "Adar", "logistics PL"), ("hogs.live", "HOGS", "Adar SaaS"), ("piramal", "Piramal", "NBFC India"), ("drreddy", "Dr. Reddy's", "pharma"), ("drreddys", "Dr. Reddy's", "pharma"), ("jlr-ddc", "JLR DDC", "automotive"), ("jlrmotor", "JLR Motor", "automotive"), ("incontrol.landrover", "JLR InControl", "connected car"), ("incontrol.jaguar", "JLR InControl", "connected car"), ("biznetgio", "BiznetGio", "ISP ID"), ("biznet.id", "BiznetGio", "ISP ID"), ("dana.id", "Dana.id", "FinTech ID"), ("playcourt", "Playcourt (Dana.id)", "FinTech ID"), ("powergrid.in", "PowerGrid India", "gov energy"), ("cybercodetech", "CyberCodeTech", "MSSP"), ("qualtechedge", "Qualtechedge", "FinTech SaaS"), ("tomex.dk", "Tomex", "MSP DK"), ("dstelecom", "DSTelecom", "ISP EG"), ("upay.uz", "UPay", "payments UZ"), ("woconnect", "WoConnect", "IoT"), ("syngenta", "Syngenta", "agritech"), ("incode.com", "Incode KYC", "identity verification"), ("connextglobal", "Connext Global", "BPO PH"), ("youngone", "Youngone Corp", "aviation KR"), ("arirang", "Arirang Aviation", "aviation"), ("bancochile", "Banco de Chile", "banking CL"), ], # ------------------------------------------------------------------ # 2. Vendor/partner portals (cascade to their clients) # ------------------------------------------------------------------ "vendor_portals": [ ("customersso1.fortinet.com", "Fortinet Partner SSO", "firewall vendor"), ("sso.paloaltonetworks.com", "Palo Alto SSO", "security vendor"), ("paloaltonetworks.com", "Palo Alto Networks", "security vendor"), ("partners.sentinelone", "SentinelOne Partner", "EDR vendor"), ("sentinelone.com", "SentinelOne", "EDR vendor"), ("partners.f5.com", "F5 Partner", "load balancer vendor"), ("fortinet.com", "Fortinet", "firewall vendor"), ("meraki.com", "Cisco Meraki", "network vendor"), ("crowdstrike.com", "CrowdStrike", "EDR vendor"), ], # ------------------------------------------------------------------ # 3. Self-hosted infra (high value — admin panels) # ------------------------------------------------------------------ "infra_code": [ ("gitlab.", "GitLab Self-hosted", "source code"), ("gitea.", "Gitea Self-hosted", "source code"), ("gogs.", "Gogs Self-hosted", "source code"), ("bitbucket.", "Bitbucket Self-hosted", "source code"), ], "infra_cicd": [ ("jenkins", "Jenkins", "CI/CD"), ("bamboo.", "Bamboo", "CI/CD"), ("teamcity.", "TeamCity", "CI/CD"), ("drone.", "Drone CI", "CI/CD"), ("argocd", "ArgoCD", "GitOps"), ("sonarqube", "SonarQube", "code quality"), ("nexus.", "Nexus Repository", "artifact repo"), ("artifactory", "JFrog Artifactory", "artifact repo"), ("verdaccio", "Verdaccio", "npm registry"), ("gocd.", "GoCD", "CI/CD"), ], "infra_containers": [ ("portainer", "Portainer", "Docker mgmt"), (":9000", "Portainer (port)", "Docker mgmt"), (":9443", "Portainer (port)", "Docker mgmt"), ("/#/containers", "Portainer (path)", "Docker mgmt"), ("/api/endpoints", "Portainer (path)", "Docker mgmt"), ("/api/stacks", "Portainer (path)", "Docker mgmt"), ("rancher.", "Rancher", "K8s mgmt"), ("kubernetes.", "Kubernetes Dashboard", "K8s"), ("coolify", "Coolify", "PaaS"), (":8000", "Coolify (port)", "PaaS"), ("harbor.", "Harbor", "container registry"), ("nomad.", "HashiCorp Nomad", "job scheduler RCE"), ], # ------------------------------------------------------------------ # 3b. RCE-capable platforms (direct code execution) # ------------------------------------------------------------------ "infra_rce": [ ("jupyter", "Jupyter Notebook", "direct code exec"), ("code-server", "code-server (VS Code)", "browser IDE RCE"), ("vscode.", "VS Code Server", "browser IDE RCE"), ("airflow", "Apache Airflow", "DAG exec RCE"), ("n8n.", "n8n Workflow", "code node RCE"), ("node-red", "Node-RED", "function node RCE"), ("rundeck", "Rundeck", "job exec RCE"), ("awx.", "AWX (Ansible Tower)", "playbook RCE"), ("semaphore.", "Semaphore (Ansible)", "playbook RCE"), ("teleport.", "Gravitational Teleport", "SSH proxy"), ("ttyd", "ttyd", "web terminal RCE"), ("wetty", "Wetty", "web terminal RCE"), ("shellbox", "Shellbox", "web terminal RCE"), ("filebrowser", "FileBrowser", "upload+exec RCE"), ], "infra_secrets": [ ("vault.", "HashiCorp Vault", "secrets management"), ("minio.", "MinIO Console", "S3 admin"), ("consul.", "HashiCorp Consul", "service mesh"), ], # ------------------------------------------------------------------ # 3b. Password managers — self-hosted vaults = full org credential store # (T1539+T1078). Self-hosted = S-tier; SaaS = org presence marker. # ------------------------------------------------------------------ "infra_password_mgr": [ # --- Self-hosted (S-tier: full org vault, RCE-adjacent) --- ("vaultwarden", "Vaultwarden (self-hosted Bitwarden)", "org vault self-hosted"), ("vault.passwall", "Passwall (self-hosted)", "org vault self-hosted"), ("passwall.io", "Passwall (self-hosted)", "org vault self-hosted"), ("passbolt", "Passbolt (self-hosted)", "org vault self-hosted"), ("keepassweb", "KeePass Web", "org vault self-hosted"), # --- Bitwarden vault (SaaS or self-hosted) --- ("bitwarden.", "Bitwarden (vault)", "org vault"), # --- Enterprise SaaS (org presence marker, not RCE target) --- (".1password.com", "1Password Business", "SaaS org vault"), (".1password.eu", "1Password Business EU", "SaaS org vault"), ("lastpass.com", "LastPass", "SaaS org vault"), ("dashlane.com", "Dashlane", "SaaS org vault"), ("keepersecurity.com", "Keeper", "SaaS org vault"), ("onelogin.", "OneLogin", "SaaS SSO+vault"), ], "infra_server_mgmt": [ ("ipmi", "IPMI", "server BMC"), ("ilo.", "HP iLO", "server mgmt KVM"), ("idrac", "Dell iDRAC", "server mgmt KVM"), ("truenas", "TrueNAS", "NAS admin shell"), ("unraid", "Unraid", "NAS/VM admin shell"), ], "infra_proxy": [ ("traefik.", "Traefik Dashboard", "reverse proxy admin"), ("nginx-proxy-manager", "Nginx Proxy Manager", "reverse proxy"), ("nginxproxymanager", "Nginx Proxy Manager", "reverse proxy"), # FRP (Fast Reverse Proxy, fatedier/frp). Use "//frp." to match the # subdomain form (https://frp.corp.local) and avoid GSM "FRP bypass" # phone-unlock SaaS (gsmfrp.com, mifrp.in, pinfrp.com, samfrp.com ...) # which are Factory Reset Protection services, not reverse proxies. ("//frp.", "FRP (Fast Reverse Proxy)", "reverse proxy admin"), (":7500", "FRP Server Dashboard (port)", "reverse proxy admin"), (":7400", "FRP Client Admin (port)", "reverse proxy admin"), ], "infra_virtualization": [ ("vcenter", "vCenter", "VMware"), ("vsphere", "vSphere", "VMware"), ("esxi", "ESXi", "VMware hypervisor"), ("proxmox", "Proxmox", "hypervisor"), ("vmware", "VMware", "virtualization"), ], "infra_monitoring": [ ("grafana.", "Grafana", "monitoring"), ("kibana.", "Kibana", "log management"), ("zabbix.", "Zabbix", "monitoring"), ("nagios", "Nagios", "monitoring"), ("prometheus.", "Prometheus", "metrics"), ("datadog.", "Datadog", "monitoring SaaS"), ], "infra_network": [ ("fortigate", "FortiGate", "firewall"), ("openvpn", "OpenVPN", "VPN"), ("wireguard", "WireGuard", "VPN"), ("pfsense", "pfSense", "firewall"), ("mikrotik", "MikroTik", "router"), ("unifi.", "UniFi", "Ubiquiti network"), ], "infra_hosting": [ ("cpanel", "cPanel", "hosting panel"), ("whm", "WHM", "hosting panel"), ("plesk", "Plesk", "hosting panel"), ("hpanel", "hPanel", "hosting panel"), ("webmin", "Webmin", "server admin"), ("directadmin", "DirectAdmin", "hosting panel"), ], "infra_remote": [ # --- RDP gateways / web-RDP (T1021.001) — top value, rotated fast --- ("/rdweb", "RD Web Access", "RDP gateway"), ("/RDWeb", "RD Web Access", "RDP gateway"), ("//gateway.", "RD Gateway", "RDP gateway"), ("//remote.", "Remote Web Workplace", "RDP/remote"), ("//ts.", "Terminal Server", "RDP"), ("//rdp.", "RDP gateway", "RDP"), ("//rdsh.", "Remote Desktop Session Host", "RDP"), ("//broker.", "RDS Connection Broker", "RDP"), ("//rds.", "Remote Desktop Services", "RDP"), # --- Apache Guacamole (clientless web-RDP, S-tier: browser → RDP/SSH) --- ("guacamole", "Apache Guacamole", "web-RDP/SSH RCE"), # --- Web-based terminal / webtop --- ("shellinabox", "Shell In A Box", "web terminal RCE"), ("//ttyd.", "ttyd", "web terminal RCE"), ("//wetty.", "Wetty", "web terminal RCE"), ("//go2rtc", "go2rtc", "remote camera/terminal RCE"), # --- VNC variants (T1021.005, T1219) --- ("//vnc.", "VNC Web", "remote desktop"), ("noVNC", "noVNC", "web VNC RCE"), ("//novnc.", "noVNC (self-hosted)", "web VNC RCE"), ("//realvnc.", "RealVNC", "remote desktop"), ("//tightvnc.", "TightVNC", "remote desktop"), ("//tigervnc.", "TigerVNC", "remote desktop"), ("//ultravnc.", "UltraVNC", "remote desktop"), # --- Commercial remote-access SaaS (org presence, agent-based) --- ("teamviewer", "TeamViewer", "remote access"), ("anydesk", "AnyDesk", "remote access"), ("//splashtop.", "Splashtop", "remote access"), ("//goto.", "GoTo (LogMeIn)", "remote access"), ("//logmein.", "LogMeIn", "remote access"), ("//isl.", "ISL Online (ISL Light)", "remote access"), ("//bomgar.", "BeyondTrust PRA (Bomgar)", "PAM remote"), ("//beyondtrust.", "BeyondTrust PRA", "PAM remote"), # --- Sunlogin / ToDesk (China-origin, very common in stealer logs) --- ("//sunlogin.", "Sunlogin (Oray)", "remote access CN"), ("//sunlogin", "Sunlogin (Oray)", "remote access CN"), ("//todesk.", "ToDesk", "remote access CN"), ("//todesk", "ToDesk", "remote access CN"), ("//oray.", "Oray / Peanut Shell", "remote access CN"), ("//oray", "Oray / Peanut Shell", "remote access CN"), ("//peanuthull", "Oray Peanut Shell", "remote access CN"), # --- ConnectWise ScreenConnect (agent RCE) --- ("screenconnect", "ConnectWise ScreenConnect", "remote access RCE"), ("//screenconnect", "ScreenConnect (self-hosted)", "remote access RCE"), # --- DWService / Meshagent (web remote) --- ("//dwservice.", "DWService", "remote access"), # --- Chrome Remote Desktop / Google --- ("//remotedesktop.google", "Chrome Remote Desktop", "remote access"), ("//remotedesktop-pa.", "Chrome RD relay", "remote access"), # --- NoMachine / NX --- ("//nomachine.", "NoMachine", "remote desktop NX"), ("//nx.", "NoMachine NX", "remote desktop NX"), # --- Synology QuickConnect --- ("quickconnect.to", "Synology QuickConnect", "NAS remote"), # --- 3CX --- ("3cx.", "3CX", "VoIP/PBX"), ("3cx.us", "3CX Cloud", "VoIP/PBX"), ], "infra_database": [ ("phpmyadmin", "phpMyAdmin", "MySQL admin"), ("adminer", "Adminer", "DB admin"), ("pgadmin", "pgAdmin", "PostgreSQL admin"), ("mongo-express", "Mongo Express", "MongoDB admin"), ], # ------------------------------------------------------------------ # 3e. Analytics / BI (SQL or Python runner = RCE) # ------------------------------------------------------------------ "infra_analytics": [ ("//superset.", "Apache Superset", "Python RCE via SQL Lab"), ("//metabase.", "Metabase", "native SQL query"), ("//redash.", "Redash", "query runner"), ("//apache-drill.", "Apache Drill", "SQL engine"), ("//druid.", "Apache Druid", "SQL ingestion RCE"), ("//pinot.", "Apache Pinot", "realtime analytics"), ("//starrocks.", "StarRocks", "MPP DB"), ("//clickhouse.", "ClickHouse", "columnar DB admin"), ("//trino.", "Trino", "distributed SQL"), ("//presto.", "Presto", "distributed SQL"), ("//zeppelin.", "Apache Zeppelin", "notebook RCE"), ("//hue.", "Hue (Hadoop UI)", "query + file mgmt"), ("//datahub.", "DataHub", "metadata RCE"), ("//openmetadata.", "OpenMetadata", "metadata"), ], # ------------------------------------------------------------------ # 3f. ERP (admin → server action RCE) # ------------------------------------------------------------------ "infra_erp": [ ("//odoo.", "Odoo", "ERP server action RCE"), ("//openerp.", "OpenERP", "ERP server action RCE"), ("//erpnext.", "ERPNext", "ERP Python script RCE"), ("//frappe.", "Frappe Framework", "Python script RCE"), ("//dolibarr.", "Dolibarr", "ERP admin"), ("//metasfresh.", "metasfresh", "ERP admin"), ("//sap.", "SAP", "ERP RFC/BAPI RCE"), ("//sapgui.", "SAP GUI Web", "ERP"), ("//vtiger.", "vTiger CRM", "CRM admin upload"), ("//suitecrm.", "SuiteCRM", "CRM admin upload"), ("//sugarcrm.", "SugarCRM", "CRM admin"), ("//zoho.", "Zoho", "SaaS CRM/ERP"), ], # ------------------------------------------------------------------ # 3g. Hosting panels (server shell RCE) # ------------------------------------------------------------------ "infra_hosting": [ ("cpanel", "cPanel", "hosting panel"), ("whm", "WHM", "hosting panel"), ("plesk", "Plesk", "hosting panel"), ("hpanel", "hPanel", "hosting panel"), ("webmin", "Webmin", "server admin"), ("directadmin", "DirectAdmin", "hosting panel"), # CN panels (very common in stealer logs) ("//bt.", "aaPanel/BT Panel", "CN hosting panel RCE"), ("//aapanel.", "aaPanel", "hosting panel RCE"), ("//cyberpanel.", "CyberPanel", "hosting panel RCE"), ("//hestia.", "HestiaCP", "hosting panel"), ("//vesta.", "VestaCP", "hosting panel"), ("//ispconfig.", "ISPConfig", "hosting panel"), ("//froxlor.", "Froxlor", "hosting panel"), ("//cloudpanel.", "CloudPanel", "hosting panel"), ("//faspanel.", "FasPanel", "hosting panel"), ("//fastpanel.", "FastPanel", "hosting panel"), ("//ploi.", "Ploi", "hosting panel"), ("//serveravatar.", "ServerAvatar", "hosting panel"), ("//runcloud.", "RunCloud", "hosting panel"), ("//forge.", "Laravel Forge", "server mgmt"), ("//centminmod.", "CentminMod", "hosting panel"), ], # ------------------------------------------------------------------ # 3h. Security platforms (agent script exec) # ------------------------------------------------------------------ "infra_security": [ ("//wazuh.", "Wazuh", "XDR agent script exec"), ("//thehive.", "TheHive", "IR platform"), ("//cortex.", "Cortex (TheHive)", "analyzer RCE"), ("//misp.", "MISP", "threat intel"), ("//opencti.", "OpenCTI", "threat intel"), ("//securityonion.", "Security Onion", "NSM/IR"), ("//zeek.", "Zeek", "network monitor"), ("//suricata.", "Suricata", "IDS/IPS"), ("//opencanary.", "OpenCanary", "honeypot"), ("//pf.", "pfSense/Suricata", "IDS"), ("//selks.", "SELKS", "SIEM"), ("//alienvault.", "AlienVault OSSIM", "SIEM"), ("//security.", "Security Console", "SIEM/XDR"), ("//siem.", "SIEM", "log mgmt"), ("//malware.", "Malware Bazaar", "threat intel"), ], # ------------------------------------------------------------------ # 3i. Backup platforms (script exec, restore RCE) # ------------------------------------------------------------------ "infra_backup": [ ("//bacula.", "Bacula", "backup admin"), ("//bareos.", "Bareos", "backup admin"), ("//urbackup.", "UrBackup", "backup client mgmt"), ("//duplicati.", "Duplicati", "backup admin"), ("//restic.", "restic-browser", "backup restore RCE"), ("//borg.", "BorgBackup web", "backup"), ("//veeam.", "Veeam", "enterprise backup RCE"), ("//nakivo.", "NAKIVO", "enterprise backup"), ("//rsync.", "rsync web", "sync admin"), ("//bareos.", "Bareos WebUI", "backup"), ("//backuppc.", "BackupPC", "backup admin"), ("//rear.", "Relax-and-Recover", "disaster recovery"), ], # ------------------------------------------------------------------ # 3j. Workflow / automation (script node RCE) # ------------------------------------------------------------------ "infra_workflow": [ ("//camunda.", "Camunda", "BPM script RCE"), ("//temporal.", "Temporal", "workflow worker RCE"), ("//huginn.", "Huginn", "agent script RCE"), ("//flowise.", "Flowise", "LLM agent exec RCE"), ("//langflow.", "LangFlow", "LLM agent exec RCE"), ("//activepieces.", "Activepieces", "workflow RCE"), ("//n8n.", "n8n (self-hosted)", "code node RCE"), ("//trigger.", "Trigger.dev", "workflow RCE"), ("//windmill.", "Windmill", "script runner RCE"), ("//node-red.", "Node-RED (self-hosted)", "function node RCE"), ("//amplitude.", "Amplitude", "analytics"), ], # ------------------------------------------------------------------ # 3k. Self-hosted IDE / dev platforms (direct code exec) # ------------------------------------------------------------------ "infra_dev": [ ("//gitpod.", "Gitpod (self-hosted)", "cloud IDE RCE"), ("//coder.", "Coder (self-hosted)", "workspace RCE"), ("//eclipse-theia.", "Eclipse Theia", "cloud IDE RCE"), ("//jupyterhub.", "JupyterHub", "multi-user notebook RCE"), ("//theia.", "Theia IDE", "cloud IDE RCE"), ("//code-server.", "code-server (self-hosted)", "browser IDE RCE"), ("//openvscode.", "OpenVSCode Server", "browser IDE RCE"), ("//eclipse.", "Eclipse Che", "cloud IDE"), ("//devpod.", "DevPod", "dev env"), ("//jetbrains.", "JetBrains Space", "dev platform"), ], # ------------------------------------------------------------------ # 3l. Doc collab (plugin/file upload RCE) # ------------------------------------------------------------------ "infra_doc": [ ("//nextcloud.", "Nextcloud", "file upload + app RCE"), ("//owncloud.", "ownCloud", "file upload RCE"), ("//onlyoffice.", "OnlyOffice", "doc server RCE"), ("//collabora.", "Collabora Online", "doc server"), ("//seafile.", "Seafile", "file sync admin"), ("//minio.", "MinIO Console", "S3 admin RCE"), ("//filerun.", "FileRun", "file mgmt"), ("//filegator.", "FileGator", "file mgmt"), ("//pydio.", "Pydio Cells", "file mgmt"), ("//ownphotos.", "OwnPhotos", "photo mgmt"), ("//photoview.", "Photoview", "photo mgmt"), ("//linkace.", "LinkAce", "bookmark mgmt"), ], # ------------------------------------------------------------------ # 3m. Web shells (direct backdoor — highest value if found) # ------------------------------------------------------------------ "infra_webshell": [ # Web shells — match by filename in URL path, NOT substring in hostname # (e.g. "wso" alone catches ws054.corp.com web-service subdomains — 790 false positives) ("c99.php", "c99 shell", "web shell"), ("r57.php", "r57 shell", "web shell"), ("/b374k", "b374k shell", "web shell"), ("/wso.php", "WSO shell", "web shell"), ("/filesman", "FilesMan shell", "web shell"), ("/cmd.php", "cmd shell", "web shell"), ("/shell.php", "generic shell", "web shell"), ("/alfa.php", "Alfa shell", "web shell"), ("/indoxploit", "IndoXploit shell", "web shell"), ("/weevely", "Weevely shell", "web shell"), ("/byrev", "ByRev shell", "web shell"), ("/sh3ll", "sh3ll shell", "web shell"), ("/0x6868.php", "0x6868 shell", "web shell"), ("azrailshell", "Azrail shell", "web shell"), ("/munin.php", "Munin shell", "web shell"), ("ws0.php", "WSO shell", "web shell"), ], # ------------------------------------------------------------------ # 3n. Phone / VoIP / PBX (RCE via dialplan/config) # ------------------------------------------------------------------ "infra_voip": [ ("//freepbx.", "FreePBX", "PBX admin RCE"), ("//asterisk.", "Asterisk", "PBX dialplan"), ("//fusionpbx.", "FusionPBX", "PBX admin RCE"), ("//vitalpbx.", "VitalPBX", "PBX admin"), ("//pbx.", "PBX Server", "VoIP admin"), ("//sangoma.", "Sangoma PBX", "VoIP"), ("//3cx.", "3CX", "VoIP/PBX"), ("//kamailio.", "Kamailio", "SIP proxy"), ("//opensips.", "OpenSIPS", "SIP proxy"), ("//sipsak.", "SIPsa k", "SIP tool"), ("//fusion.", "Fusion PBX", "VoIP"), ("//mango.", "Mango PBX", "VoIP"), ("//sipx.", "SipX", "SIP PBX"), ("//elastix.", "Elastix", "unified comms"), ], # ------------------------------------------------------------------ # 3o. CMS admin (plugin upload = RCE) # ------------------------------------------------------------------ "infra_cms": [ ("/wp-login.php", "WordPress admin", "plugin upload RCE"), ("/wp-admin", "WordPress admin", "plugin upload RCE"), ("//wp-admin.", "WordPress admin", "plugin upload RCE"), ("//administrator.", "Joomla admin", "extension upload RCE"), ("/administrator/index.php", "Joomla admin", "extension upload RCE"), ("//drupal.", "Drupal admin", "module upload RCE"), ("/user/login", "Drupal login", "CMS login"), ("//ghost.", "Ghost admin", "CMS admin"), ("//strapi.", "Strapi admin", "CMS API admin RCE"), ("//directus.", "Directus admin", "headless CMS RCE"), ("//payload.", "Payload CMS", "headless CMS"), ("//cockpit.", "Cockpit CMS", "headless CMS"), ("//grav.", "Grav admin", "flat CMS"), ("//october.", "October CMS", "CMS admin"), ("//craft.", "Craft CMS", "CMS admin"), ("//expressionengine.", "ExpressionEngine", "CMS admin"), ("//textpattern.", "Textpattern", "CMS admin"), ("//bolt.", "Bolt CMS", "CMS admin"), ("//statamic.", "Statamic", "CMS admin"), ("//getsimple.", "GetSimple CMS", "flat CMS"), ("//concrete5.", "Concrete5", "CMS admin"), ("//modx.", "MODX", "CMS admin"), ("//contao.", "Contao", "CMS admin"), ("//neos.", "Neos CMS", "CMS admin"), ("//typo3.", "TYPO3", "CMS admin"), ], # ------------------------------------------------------------------ # 3p. Crypto exchanges & wallets (financial — high-value targets) # ------------------------------------------------------------------ "crypto_exchange": [ # Centralized exchanges (CEX) — login creds = account access + withdrawal ("binance.com", "Binance", "CEX account"), ("coinbase.com", "Coinbase", "CEX account"), ("kraken.com", "Kraken", "CEX account"), ("bybit.com", "Bybit", "CEX account"), ("okx.com", "OKX", "CEX account"), ("kucoin.com", "KuCoin", "CEX account"), ("gate.io", "Gate.io", "CEX account"), ("bitfinex.com", "Bitfinex", "CEX account"), ("bitstamp.net", "Bitstamp", "CEX account"), ("gemini.com", "Gemini", "CEX account"), ("crypto.com", "Crypto.com", "CEX account"), ("ftx.com", "FTX", "CEX (defunct)"), ("huobi.com", "Huobi", "CEX account"), ("mexc.com", "MEXC", "CEX account"), ("bitget.com", "Bitget", "CEX account"), ("poloniex.com", "Poloniex", "CEX account"), ("bittrex.com", "Bittrex", "CEX (defunct)"), ("hitbtc.com", "HitBTC", "CEX account"), ("ascendex.com", "AscendEX", "CEX account"), ("bibox.com", "Bibox", "CEX account"), ("bithumb.com", "Bithumb", "CEX account"), ("upbit.com", "Upbit", "CEX account"), ("coinone.co.kr", "Coinone", "CEX account"), ("korbit.co.kr", "Korbit", "CEX account"), ("coinspot.com.au", "CoinSpot", "CEX account"), ("independentreserve.com", "Independent Reserve", "CEX account"), ("bitso.com", "Bitso", "CEX account"), ("mercadobitcoin.net", "Mercado Bitcoin", "CEX account"), ("foxbit.com.br", "Foxbit", "CEX account"), ("bitcointrade.com.br", "BitcoinTrade", "CEX account"), ("novadax.com.br", "NovaDAX", "CEX account"), ("remitano.com", "Remitano", "P2P exchange"), ("paxful.com", "Paxful", "P2P exchange"), ("localbitcoins.com", "LocalBitcoins", "P2P (defunct)"), ("bisq.network", "Bisq", "DEX"), ("fixedfloat.com", "FixedFloat", "mixer/exchange"), ("changenow.io", "ChangeNOW", "swap"), ("simpleswap.io", "SimpleSwap", "swap"), ("exmo.com", "EXMO", "CEX account"), ("cex.io", "CEX.io", "CEX account"), ("lykke.com", "Lykke", "CEX account"), ("abbc.com", "ABBC", "CEX account"), ], "crypto_wallet": [ # Web wallets ("blockchain.com", "Blockchain.com", "web wallet"), ("myetherwallet.com", "MyEtherWallet", "ETH web wallet"), ("mycrypto.com", "MyCrypto", "ETH web wallet"), ("walletconnect.com", "WalletConnect", "wallet bridge"), ("trustwallet.com", "Trust Wallet", "mobile wallet"), ("metamask.io", "MetaMask", "browser wallet"), ("phantom.app", "Phantom", "Solana wallet"), ("solflare.com", "Solflare", "Solana wallet"), ("martianwallet.xyz", "Martian", "Aptos wallet"), ("roninwallet.com", "Ronin Wallet", "Axie wallet"), ("templewallet.com", "Temple Wallet", "Tezos wallet"), ("kukai wallet", "Kukai", "Tezos wallet"), ("nautiluswallet.com", "Nautilus", "Ergo wallet"), ("tangem.com", "Tangem", "hardware wallet"), ("trezor.io", "Trezor", "hardware wallet"), ("keepkey.com", "KeepKey", "hardware wallet"), ("onekey.so", "OneKey", "hardware wallet"), ("safepal.com", "SafePal", "hardware wallet"), ("keplr.app", "Keplr", "Cosmos wallet"), ("leapwallet.io", "Leap Wallet", "Cosmos wallet"), ("finnie.koii.io", "Finnie", "K2 wallet"), ("xdefi.io", "XDEFI Wallet", "multi-chain"), ("enjin.io", "Enjin", "NFT wallet"), ("rainbow.me", "Rainbow", "ETH mobile wallet"), ("zerion.io", "Zerion", "DeFi wallet"), ("argent.xyz", "Argent", "ETH smart wallet"), ("pillarwallet.io", "Pillar", "DeFi wallet"), ("mathwallet.org", "MathWallet", "multi-chain"), ("coinomi.com", "Coinomi", "mobile wallet"), ("exodus.com", "Exodus", "desktop wallet"), ("electrum.org", "Electrum", "BTC desktop"), ("wasabiwallet.io", "Wasabi", "BTC privacy"), ("sparrowwallet.com", "Sparrow Wallet", "BTC desktop"), ("bluewallet.io", "BlueWallet", "BTC mobile"), ("muun.com", "Muun", "BTC mobile"), ("bitcoincore.org", "Bitcoin Core", "BTC full node"), ("electrum.org", "Electrum", "BTC desktop"), # Custodial wallet services ("crypto.com", "Crypto.com", "custodial wallet"), ("okx.com", "OKX", "custodial wallet"), ("cobo.com", "Cobo", "custodial wallet"), ("bitgo.com", "BitGo", "institutional custody"), ("fireblocks.com", "Fireblocks", "institutional custody"), ("anchorage.com", "Anchorage", "institutional custody"), ("bakkt.com", "Bakkt", "institutional custody"), ("copper.co", "Copper", "institutional custody"), ], "crypto_defi": [ # DeFi protocols — creds = API keys to managed positions ("compound.finance", "Compound", "DeFi lending"), ("aave.com", "Aave", "DeFi lending"), ("uniswap.org", "Uniswap", "DEX"), ("sushi.com", "SushiSwap", "DEX"), ("curve.fi", "Curve", "DEX stable"), ("balancer.fi", "Balancer", "DEX"), ("pancakeswap.finance", "PancakeSwap", "BSC DEX"), ("1inch.io", "1inch", "DEX aggregator"), ("yearn.finance", "Yearn", "yield aggregator"), ("harvest.finance", "Harvest", "yield aggregator"), ("convexfinance.com", "Convex", "yield"), ("stake.ly", "Stake.ly", "staking"), ("lido.fi", "Lido", "liquid staking"), ("rocketpool.net", "Rocket Pool", "ETH staking"), ("stakewise.io", "StakeWise", "staking"), ("bloxstaking.com", "Blox", "staking"), ("allnodes.com", "Allnodes", "staking infra"), ("staked.us", "Staked", "staking"), ("figment.io", "Figment", "staking"), ("chainnodes.org", "Chainnodes", "staking infra"), ("ankr.com", "Ankr", "staking infra"), ("immunefi.com", "Immunefi", "bug bounty"), ("tornado.cash", "Tornado Cash", "mixer"), ("app.tornado", "Tornado Cash", "mixer"), ("chainalysis.com", "Chainalysis", "blockchain analysis"), ("etherscan.io", "Etherscan", "block explorer"), ("bscscan.com", "BscScan", "block explorer"), ("polygonscan.com", "PolygonScan", "block explorer"), ("snowtrace.io", "SnowTrace", "block explorer"), ("solscan.io", "Solscan", "block explorer"), ("mempool.space", "Mempool", "BTC explorer"), ("blockcypher.com", "BlockCypher", "blockchain API"), ("alchemy.com", "Alchemy", "web3 infra"), ("infura.io", "Infura", "web3 infra"), ("quicknode.com", "QuickNode", "web3 infra"), ("ankr.com", "Ankr", "web3 infra"), ("moralis.io", "Moralis", "web3 API"), ("thegraph.com", "The Graph", "indexing"), ("dune.com", "Dune Analytics", "analytics"), ("nansen.ai", "Nansen", "on-chain analytics"), ("debank.com", "DeBank", "portfolio tracker"), ("zapper.fi", "Zapper", "portfolio tracker"), ("zerion.io", "Zerion", "portfolio tracker"), ("rotki.com", "Rotki", "portfolio tracker"), ("cointracker.io", "CoinTracker", "tax tracker"), ("koinly.io", "Koinly", "tax tracker"), ("cryptocompare.com", "CryptoCompare", "analytics"), ("coingecko.com", "CoinGecko", "price tracker"), ("coinmarketcap.com", "CoinMarketCap", "price tracker"), ("messari.io", "Messari", "research"), ], # ------------------------------------------------------------------ # 3q. Self-hosted crypto payment / gateway / node infrastructure # (provider-side — NOT consumer SaaS logins) # ------------------------------------------------------------------ "crypto_selfhosted": [ # BTCPay Server — self-hosted Bitcoin payment processor (admin = invoice + payout control) ("//btcpay.", "BTCPay Server", "payment processor admin"), ("//btcpayserver.", "BTCPay Server", "payment processor admin"), ("//btcpay.", "BTCPay Server", "payment admin"), # Lightning Network nodes (LND, c-lightning, Eclair) ("//lnd.", "LND (Lightning)", "Lightning node admin"), ("//lightning.", "Lightning Node", "Lightning admin"), ("//c-lightning.", "c-lightning", "Lightning node"), ("//eclair.", "Eclair Lightning", "Lightning node"), ("//rtl.", "RTL (Ride the Lightning)", "LND web UI"), ("//lndhub.", "LNDHub", "Lightning accounts"), ("//zeus.", "Zeus (Lightning)", "mobile LND"), ("//bos.", "Balance of Satoshis", "LND tool"), ("//thunderhub.", "ThunderHub", "LND web UI"), ("//lndg.", "LNDg", "LND autopia"), ("//lndmon.", "LND Monitor", "Lightning monitoring"), # Full nodes — RPC interfaces (credentials = wallet control) ("//bitcoind.", "Bitcoin Core RPC", "BTC full node RPC"), ("//bitcoin-rpc.", "Bitcoin RPC", "BTC node RPC"), ("//geth.", "Geth (Ethereum)", "ETH full node RPC"), ("//parity.", "Parity/OpenEthereum", "ETH node RPC"), ("//nethermind.", "Nethermind", "ETH node RPC"), ("//besu.", "Hyperledger Besu", "ETH node RPC"), ("//erigon.", "Erigon", "ETH node RPC"), ("//reth.", "Reth", "ETH node RPC"), ("//lighthouse.", "Lighthouse", "ETH beacon node"), ("//teku.", "Teku", "ETH beacon node"), ("//prysm.", "Prysm", "ETH beacon node"), ("//nimbus.", "Nimbus", "ETH beacon node"), ("//lodestar.", "Lodestar", "ETH beacon node"), # RPC proxy / API gateway for blockchain nodes ("//rpc.", "RPC endpoint", "blockchain RPC"), ("//node.", "Node RPC", "blockchain RPC"), ("//validator.", "Validator", "staking validator"), # Crypto payment gateways / processors (self-hosted) ("//btcpay.", "BTCPay", "payment gateway"), ("//coingate.", "CoinGate", "payment gateway"), ("//nowpayments.", "NOWPayments", "payment gateway"), ("//coinpayments.", "CoinPayments", "payment gateway"), ("//gourl.", "GoURL", "payment gateway"), ("//cryptodonate.", "Crypto Donate", "donation widget"), ("//btcpayserver", "BTCPay", "payment"), # Monero nodes ("//monero.", "Monero Node", "XMR RPC"), ("//xmrd.", "Monero Daemon", "XMR node"), ("//monero-wallet.", "Monero Wallet RPC", "XMR wallet RPC"), ("//monero-wallet-rpc", "Monero Wallet RPC", "XMR wallet"), # Dogecoin / Litecoin nodes ("//doged.", "Dogecoin Node", "DOGE RPC"), ("//litecoind.", "Litecoin Node", "LTC RPC"), ("//dashd.", "Dash Node", "DASH RPC"), ("//zcashd.", "Zcash Node", "ZEC RPC"), # Block explorer self-hosted (BTC RPC Explorer, etc) ("//bitcoin-explorer.", "BTC Explorer", "block explorer"), ("//block-explorer.", "Block Explorer", "self-hosted explorer"), ("//explorer.", "Block Explorer", "self-hosted explorer"), ("//blockstream.", "Blockstream Explorer", "esplora"), ("//esplora.", "Esplora", "block explorer"), ("//mempool.", "Mempool Space", "self-hosted explorer"), # Wallet infrastructure (self-hosted) ("//specter.", "Specter Desktop", "BTC wallet"), ("//sparrow.", "Sparrow Server", "BTC wallet"), ("//electrum-server.", "Electrum Server", "BTC Electrum"), ("//electrs.", "Electrs", "BTC Electrum server"), ("//fulcrum.", "Fulcrum", "BTC Electrum server"), # Staking validator web UIs (self-hosted) ("//staking.", "Staking UI", "staking node admin"), ("//validator.", "Validator UI", "staking"), ("//picasso.", "Prysm validator", "ETH validator UI"), ("//vouch.", "Vouch", "validator"), # Web3 node providers (self-hosted RPC) ("//quorum.", "Quorum", "enterprise ETH"), ("//fantom.", "Fantom Node", "FTM RPC"), ("//avax.", "Avalanche Node", "AVAX RPC"), ("//polygon.", "Polygon Node", "MATIC RPC"), ("//bsc.", "BSC Node", "BSC RPC"), ("//arbitrum.", "Arbitrum Node", "ARB RPC"), ("//optimism.", "Optimism Node", "OP RPC"), # Exchange self-hosted (matching engine / hot wallet) ("//matching.", "Matching Engine", "exchange engine"), ("//hotwallet.", "Hot Wallet", "exchange hot wallet"), ("//coldwallet.", "Cold Wallet", "exchange cold storage"), ("//treasury.", "Treasury", "exchange treasury"), ], "infra_mail": [ ("roundcube", "Roundcube", "webmail"), ("zimbra", "Zimbra", "mail server"), ("horde", "Horde", "webmail"), ("/owa/", "OWA", "Exchange"), ("mail.", "Mail Server", "corporate mail"), ], "infra_comms": [ ("mattermost", "Mattermost", "team chat"), ("rocketchat", "Rocket.Chat", "team chat"), ("slack.com", "Slack", "team chat"), ], "infra_wiki": [ ("confluence.", "Confluence", "wiki"), ("jira.", "Jira", "project mgmt"), ("redmine", "Redmine", "project mgmt"), ], # ------------------------------------------------------------------ # 3c. ITSM / helpdesk — tickets store creds, scripts/automation = RCE # ------------------------------------------------------------------ "infra_itsm": [ # Self-hosted (S-tier: full ticket DB with creds + custom scripts) # Use "//." subdomain prefix to avoid false positives # (e.g. "rt." alone catches cart., court., report., start. — 366K noise on JUNE-17). ("//osticket.", "osTicket", "self-hosted helpdesk"), ("//zammad.", "Zammad", "self-hosted helpdesk"), ("//rt.", "Request Tracker (RT)", "self-hosted helpdesk"), ("//glpi.", "GLPI", "self-hosted ITAM/helpdesk"), ("//uvdesk.", "UVDesk", "self-hosted helpdesk"), ("//helpspot.", "HelpSpot", "self-hosted helpdesk"), ("//trellix.", "Trellix (ex-McAfee ePO)", "endpoint mgmt"), ("//servicenow.", "ServiceNow (self-hosted)", "enterprise ITSM"), ("//remedy.", "BMC Remedy", "enterprise ITSM"), ("//cherwell.", "Cherwell Service Mgmt", "enterprise ITSM"), # Enterprise SaaS (org presence, secrets in tickets) — keep bare domain = SaaS tenant (".servicenowservices.com", "ServiceNow (SaaS)", "enterprise ITSM"), (".service-now.com", "ServiceNow (SaaS)", "enterprise ITSM"), (".zendesk.com", "Zendesk (SaaS)", "SaaS helpdesk"), ("//freshdesk.", "Freshdesk", "SaaS helpdesk"), ("//freshservice.", "Freshservice", "SaaS ITSM"), ("//jira-servicedesk", "Jira Service Mgmt", "SaaS ITSM"), ("//bamboohr.", "BambooHR", "SaaS HR (creds in onboarding)"), ], # ------------------------------------------------------------------ # 3d. RMM (Remote Monitoring & Mgmt) — agent RCE + cred vault # T1562 + T1219: disable EDR, remote code exec via agent # ------------------------------------------------------------------ "infra_rmm": [ # Self-hosted / on-prem RMM (S-tier: push scripts to all agents) # Use "//." subdomain prefix to match self-hosted instances only. # Bare vendor domains (atera.com, mesh.life, pulseway.com) are marketing pages — skip. ("//connectwise", "ConnectWise Automate", "RMM agent RCE"), ("//labtech.", "ConnectWise LabTech", "RMM agent RCE"), ("//n-sight", "N-Sight RMM (N-able)", "RMM agent RCE"), ("//n-able", "N-Able N-Central", "RMM agent RCE"), ("//atera.", "Atera (self-hosted)", "RMM agent RCE"), ("//datto.", "Datto RMM", "RMM agent RCE"), ("//action1.", "Action1 (self-hosted)", "RMM agent RCE"), ("//syncromsp", "Syncro RMM", "RMM agent RCE"), ("//pulseway.", "Pulseway (self-hosted)", "RMM agent RCE"), ("//lansweeper.", "Lansweeper", "ITAM + remote exec"), ("//pdq.", "PDQ Deploy", "remote sw deploy RCE"), ("//meshcentral", "MeshCentral", "remote web-terminal RCE"), ("//rustdesk", "RustDesk (self-hosted)", "remote desktop RCE"), ("//go2rtc", "go2rtc", "remote camera/terminal RCE"), ("//mesh.", "Mesh agent (self-hosted)", "remote terminal RCE"), # Addigy / MDM variants (self-hosted subdomain only) ("//addigy.", "Addigy MDM", "macOS fleet RCE"), ("//mosyle.", "Mosyle MDM", "macOS fleet RCE"), ], # ------------------------------------------------------------------ # 4. Cloud consoles # ------------------------------------------------------------------ "cloud": [ ("console.aws.amazon.com", "AWS Console", "cloud"), ("signin.aws.amazon.com", "AWS SignIn", "cloud"), ("portal.azure.com", "Azure Portal", "cloud"), ("console.cloud.google.com", "GCP Console", "cloud"), ("cloud.digitalocean.com", "DigitalOcean", "cloud"), ("my.vultr.com", "Vultr", "cloud"), ("cloud.linode.com", "Linode/Akamai", "cloud"), ("app.hetzner.cloud", "Hetzner Cloud", "cloud"), ("dash.cloudflare.com", "Cloudflare Dashboard", "CDN/DNS"), ], # ------------------------------------------------------------------ # 5. Private network (LAN IPs with admin ports) # ------------------------------------------------------------------ "private_network": [ ("192.168.", "Private LAN (192.168)", "internal"), ("10.0.", "Private LAN (10.0)", "internal"), ("10.1.", "Private LAN (10.1)", "internal"), ("10.10.", "Private LAN (10.10)", "internal"), ("172.16.", "Private LAN (172.16)", "internal"), ], # ------------------------------------------------------------------ # 6. Cameras / IoT / SCADA # ------------------------------------------------------------------ "iot": [ ("cam", "IP Camera", "surveillance"), ("hikvision", "Hikvision", "surveillance"), ("dahua", "Dahua", "surveillance"), ("withsecure", "WithSecure", "security"), ("mega.nz", "MEGA", "cloud storage"), ], } # Categories to skip by default (too noisy) NOISY_CATEGORIES = {"private_network", "iot"} def load_ulp_files(): files = [] for f in sorted(ULP_DIR.glob("*_t7*.txt")): files.append(("ulp", f)) return files def load_tsv_files(source_filter=None): files = [] if source_filter: # Explicit single-source mode: load only the matching creds.tsv symlink. candidate = BOXED_DIR / f"{source_filter}_creds.tsv" if candidate.exists() or candidate.is_symlink(): files.append(("tsv", candidate)) else: print(f" [warn] --source {source_filter}: {candidate} not found, scanning all", file=sys.stderr) source_filter = None if not source_filter: for f in sorted(BOXED_DIR.glob("*_creds.tsv")): files.append(("tsv", f)) return files def search_file_ulp(filepath, patterns_flat): hits = defaultdict(list) try: with open(filepath, encoding="utf-8", errors="ignore") as f: for line in f: line = line.strip() if not line or line.startswith("#") or line.startswith("="): continue lower = line.lower() for pat, label, cat in patterns_flat: if pat in lower: parts = line.rsplit(":", 2) if len(parts) == 3 and "://" in parts[0]: url, user, pw = parts[0], parts[1], parts[2] else: url, user, pw = line, "", "" hits[cat].append({ "label": label, "url": url[:100], "user": user[:50], "pw_len": len(pw), "source": filepath.name[:20], }) except Exception as e: print(f" Error reading {filepath}: {e}", file=sys.stderr) return hits def search_file_tsv(filepath, patterns_flat): hits = defaultdict(list) # Эти TSV не имеют header-строки: колонки зафиксированы как url\tuser\tpassword. # csv.DictReader здесь неприменим (он съедает первую строку как заголовок). try: with open(filepath, newline="", encoding="utf-8", errors="ignore") as f: for line in f: parts = line.rstrip("\n").split("\t") if len(parts) < 3: continue url_full = parts[0] user = parts[1] pw = parts[2] url = url_full.lower() for pat, label, cat in patterns_flat: if pat in url: hits[cat].append({ "label": label, "url": url_full[:100], "user": user[:50], "pw_len": len(pw), "source": filepath.name[:20], }) except Exception as e: print(f" Error reading {filepath}: {e}", file=sys.stderr) return hits # ====================================================================== # Parallel TSV scan (multiprocessing over byte chunks + combined-regex # prefilter). Single TSV is split on line boundaries; every worker runs a # cheap single combined regex first and only does per-pattern attribution # on matching lines. Falls back to sequential when workers<=1. # ====================================================================== # Worker-global state, initialised once per process via _worker_init. _W = {} def _line_bounds(filepath: Path, n_chunks: int): """Return [(start, end)] byte offsets aligned to '\n' boundaries.""" size = filepath.stat().st_size if n_chunks <= 1 or size == 0: return [(0, size)] bounds = [] step = max(1, size // n_chunks) with open(filepath, "rb") as f: mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) try: start = 0 for i in range(1, n_chunks): approx = i * step if approx >= size: break nl = mm.find(b"\n", approx) end = size if nl == -1 else nl + 1 bounds.append((start, end)) start = end bounds.append((start, size)) finally: mm.close() return [b for b in bounds if b[1] > b[0]] def _worker_init(patterns_flat, combined_src, fname, ftype): _W["patterns"] = patterns_flat _W["combined"] = re.compile(combined_src) _W["fname"] = fname _W["ftype"] = ftype def _scan_chunk(args): filepath, start, end = args patterns = _W["patterns"] combined = _W["combined"] fname = _W["fname"] ftype = _W["ftype"] hits = defaultdict(list) with open(filepath, "rb") as f: f.seek(start) data = f.read(end - start) for raw in data.split(b"\n"): line = raw.decode("utf-8", errors="ignore").strip() if not line or line.startswith("#") or line.startswith("="): continue if ftype == "ulp": lower = line.lower() if not combined.search(lower): continue for pat, label, cat in patterns: if pat in lower: parts = line.rsplit(":", 2) if len(parts) == 3 and "://" in parts[0]: url, user, pw = parts[0], parts[1], parts[2] else: url, user, pw = line, "", "" hits[cat].append({ "label": label, "url": url[:100], "user": user[:50], "pw_len": len(pw), "source": fname, }) else: # tsv parts = line.rstrip("\n").split("\t") if len(parts) < 3: continue url_full, user, pw = parts[0], parts[1], parts[2] url = url_full.lower() if not combined.search(url): continue for pat, label, cat in patterns: if pat in url: hits[cat].append({ "label": label, "url": url_full[:100], "user": user[:50], "pw_len": len(pw), "source": fname, }) return dict(hits) def parallel_search_file(ftype: str, filepath: Path, patterns_flat, workers: int): """Scan one file across `workers` processes. Returns defaultdict(list).""" combined_src = "|".join(re.escape(p[0]) for p in patterns_flat) bounds = _line_bounds(filepath, workers * 4) # oversubscribe for load balance if len(bounds) <= 1 or workers <= 1: if ftype == "ulp": return search_file_ulp(filepath, patterns_flat) return search_file_tsv(filepath, patterns_flat) merged = defaultdict(list) fname = filepath.name[:20] ctx = mp.get_context("fork") with ProcessPoolExecutor( max_workers=workers, mp_context=ctx, initializer=_worker_init, initargs=(patterns_flat, combined_src, fname, ftype)) as ex: for part in ex.map(_scan_chunk, [(filepath, s, e) for s, e in bounds], chunksize=1): for cat, entries in part.items(): merged[cat].extend(entries) return merged def main(): parser = argparse.ArgumentParser(description="Corporate access search in stealer logs") parser.add_argument("--category", "-c", help="Search only this category") parser.add_argument("--pattern", "-p", help="Ad-hoc pattern (substring match)") parser.add_argument("--list-categories", action="store_true", help="List all categories") parser.add_argument("--all", action="store_true", help="Include noisy categories (private_network, iot)") parser.add_argument("--source", "-s", help="Restrict scan to a single boxed_pw/_creds.tsv (e.g. AYANKOUJI-528). Avoids the rename-hack.") parser.add_argument("--tsv", help="Scan an arbitrary TSV file (urluserpass) directly") parser.add_argument("--workers", "-w", type=int, default=1, help="Parallel worker processes per file (default 1 = sequential). " "Large single-TSV scans benefit from --workers $(nproc).") args = parser.parse_args() if args.list_categories: for cat, patterns in PATTERNS.items(): noisy = " (noisy, use --all)" if cat in NOISY_CATEGORIES else "" print(f" {cat}: {len(patterns)} patterns{noisy}") return # Build flat pattern list patterns_flat = [] if args.pattern: patterns_flat.append((args.pattern.lower(), args.pattern, "adhoc")) else: for cat, pats in PATTERNS.items(): if args.category and cat != args.category: continue if cat in NOISY_CATEGORIES and not args.all: continue for pat, label, notes in pats: patterns_flat.append((pat.lower(), label, cat)) if not patterns_flat: print("No patterns to search. Use --list-categories to see options.") return print(f"Searching {len(patterns_flat)} patterns...\n", file=sys.stderr) # Collect files if args.tsv: tsv_path = Path(args.tsv) if not tsv_path.is_file(): print(f"[error] {tsv_path} not found", file=sys.stderr) return files = [("tsv", tsv_path)] else: files = load_ulp_files() + load_tsv_files(source_filter=args.source) print(f"Files: {len(files)} ({'tsv:' + args.tsv if args.tsv else 'ULP + TSV' + (' [' + args.source + ']' if args.source else '')})\n", file=sys.stderr) # Search all_hits = defaultdict(list) for ftype, fpath in files: par = args.workers > 1 print(f" Scanning {fpath.name}{f' [{args.workers} workers]' if par else ''}...", file=sys.stderr) if par: hits = parallel_search_file(ftype, fpath, patterns_flat, args.workers) elif ftype == "ulp": hits = search_file_ulp(fpath, patterns_flat) else: hits = search_file_tsv(fpath, patterns_flat) for cat, entries in hits.items(): all_hits[cat].extend(entries) # Deduplicate by (url, user) for cat in all_hits: seen = set() deduped = [] for h in all_hits[cat]: key = (h["url"].lower(), h["user"].lower()) if key not in seen: seen.add(key) deduped.append(h) all_hits[cat] = deduped # Output total = sum(len(v) for v in all_hits.values()) print(f"\n# Corporate Access Search Results\n") print(f"**Patterns:** {len(patterns_flat)}, **Hits:** {total}\n") for cat in sorted(all_hits.keys()): entries = all_hits[cat] if not entries: continue print(f"## {cat} ({len(entries)} hits)\n") print(f"| # | Label | URL | User | PW Len | Source |") print(f"|---|-------|-----|------|:------:|--------|") for i, e in enumerate(sorted(entries, key=lambda x: x["label"]), 1): print(f"| {i} | {e['label']} | {e['url'][:70]} | {e['user'][:35]} | {e['pw_len']} | {e['source']} |") print() if __name__ == "__main__": main()