diff --git a/projects/llm_audit_ctf/.gitignore b/projects/llm_audit_ctf/.gitignore new file mode 100644 index 000000000..0073c1c7e --- /dev/null +++ b/projects/llm_audit_ctf/.gitignore @@ -0,0 +1,12 @@ +# Runtime data may contain customer targets, response evidence, or audit findings. +work/data/raw/* +!work/data/raw/.gitkeep +work/data/candidates/* +!work/data/candidates/.gitkeep +work/data/findings/* +!work/data/findings/.gitkeep +work/reports_out/* +!work/reports_out/.gitkeep + +# Python cache +work/scanner/__pycache__/ diff --git a/projects/llm_audit_ctf/README.md b/projects/llm_audit_ctf/README.md new file mode 100644 index 000000000..5d335469c --- /dev/null +++ b/projects/llm_audit_ctf/README.md @@ -0,0 +1,47 @@ +# llm_audit_ctf + +轻量化的 LLM 服务暴露面候选发现与人工审计项目。扫描器只做低影响 `GET` 指纹探测,不登录、不提交利用请求、不读取认证会话端点。 + +## 当前结构 + +- `work/scanner/probe.py`:异步候选发现器,固定 worker 池、共享 HTTP 会话、并发上限、追加式 JSONL、`--resume` 续跑。 +- `work/fingerprints/ai_fingerprints.json`:可扩展指纹库。每条 probe 的全部 `expect` 必须满足;纯状态码和认证/会话路径不作为候选证据。 +- `work/data/raw/`:脱敏后的输入目标;禁止放身份证、Cookie、API Key、代理凭据。 +- `work/data/candidates/`:扫描输出 JSONL;结果仅保存有限、脱敏证据。 +- `work/candidate_review.md`、`work/audit_report.md`:人工复核与对外审计报告模板。 + +本机网络监听快照属于临时运行交接信息,不纳入版本库;本项目不启动固定监听端口。 + +## 最小运行 + +```text +python work/scanner/probe.py -i work/data/raw/hosts.txt -o work/data/candidates/result.jsonl --resume +``` + +常用参数: + +- `-c/--concurrency`:并发数,程序限制为 1..100;建议从 4 或 8 开始。 +- `-t/--timeout`:单请求超时,必须大于 0。 +- `-f/--fingerprint`:指纹库路径。 +- `-p/--proxies`:可选的完整 `http(s)` 代理 URL 列表;按目标稳定选择代理,代理地址只进入请求层,不写入结果。 +- `--probe-only /v1/models,/api/models`:仅使用指定路径;空白项会忽略,路径必须以 `/` 开头。 +- `--resume`:跳过输出 JSONL 已记录的目标。 +- 输出 JSONL 每条记录包含 `run` 元数据(扫描器/指纹库版本、超时、代理数量及关键开关),便于复核和复现;不包含代理地址。 +- `--insecure`:仅本地测试使用;生产审计默认校验证书。 +- `--follow-redirects`:仅明确需要时开启;默认不跟随跳转。 + +## 工作流 + +1. 对目标来源、授权边界和时间窗口做记录;输入先脱敏。 +2. 以低并发、短超时运行候选发现;保留原始 JSONL,不覆盖历史结果。 +3. 用 `candidate_review.md` 逐项人工复核:资产归属、未授权可见性、最小证据和影响。 +4. 只在明确授权且必要时做最小深度验证;禁止猜测口令、提取密钥、调用模型生成内容。 +5. 用 `audit_report.md` 形成可提交报告,并保留修复建议、复测条件和时间线。 + +## 结果解释 + +`candidate` 只是指纹候选,不等于漏洞或可用 2API。`no_match_or_unreachable` 也不等于安全:可能受限于 TLS、网络、WAF、代理或探针覆盖范围。所有结论必须以人工复核证据为准。 + +## 安全边界 + +不启动项目服务、不复用常见代理监听端口、不读取密钥文件、不把真实凭据写入仓库。代理、目标和报告中的敏感字段应使用占位符或哈希/截断值。 diff --git a/projects/llm_audit_ctf/project_memory.md b/projects/llm_audit_ctf/project_memory.md new file mode 100644 index 000000000..443139d1d --- /dev/null +++ b/projects/llm_audit_ctf/project_memory.md @@ -0,0 +1,41 @@ +# Project Memory: llm_audit_ctf + +## 任务背景 +- 厂商主办的官方赛事,主题:LLM/大模型服务暴露面审计(找适合做 2API 的网站/服务 → 判断可被二次封装利用的程度 → 出审计报告提交给网站/主办方) +- 赛制:奖金与产出成果量挂钩,产出越多奖金越高;单人参赛 +- 状态:已获合法授权;题目原文未脱敏(含个人身份证信息),暂不提供,等脱敏后给 + +## 已有资产 +- `work/fingerprints/ai_fingerprints.json`:指纹库 v0.2,保留17条带内容特征的低影响 GET 探针;不含认证/会话路径或纯状态规则。 +- `work/scanner/probe.py`:aiohttp 异步候选发现器;严格 AND 匹配、代理实传、保留端口/IPv6、默认 TLS 校验且不跟随跳转、共享 Session、固定 worker 池、追加 JSONL 与 `--resume`。 +- `README.md`:运行约定与安全边界。 +- `work/candidate_review.md`:候选人工复核表。 +- `work/audit_report.md`:审计报告模板。 +- `work/data/`:raw(脱敏目标)/ candidates(扫描结果)/ findings(深度验证)三级目录。 + +## 约束与纪律 +- 项目文件一律放 `./projects/llm_audit_ctf/`,禁止丢 temp 根目录 +- 密钥/身份证等敏感信息仅引用不读取;报告脱敏 +- 只做被动验证与最小 PoC,不真实套利、不批量薅资源(防法律风险+保报告可信度) +- 入库判据:记忆归零后接手缺了会重复付出认知代价才记 + +## 本机验收结论(2026-08-11) +- 已通过 Python 编译、`--help`、裸主机/端口/显式 URL/IPv6 输入规范化测试;含 URL 凭据和非 HTTP(S) 方案会被拒绝。 +- 随机 `127.0.0.1` 回环 E2E:带 `status:200` 与 `json_contains:data` 的 `/v1/models` 规则命中;普通 HTTP 200 页面不命中;`--resume` 不重访已记录目标。 +- 默认指纹和输出路径已锚定到脚本所在 `work/`,可从项目根直接运行。最终回环普通页只产生 `no_match_or_unreachable`,规则库静态检查无敏感路径或纯状态规则。 +- 未启动项目服务、未外部批量扫描、未验证或修改本机代理链,未读取任何凭据。 + +## 待办 +- [ ] 拿到脱敏后的题目原文 → 拆解评分点(产出计量方式、官方报告字段、提交渠道、时间线)。 +- [ ] 按赛事实际规则补充指纹、候选排序和报告字段;每条新增规则先做本机反例/正例测试。 +- [ ] 仅在明确授权范围内导入脱敏目标并进行小批次候选发现,再按 `candidate_review.md` 人工复核。 +- [ ] 形成发现、修复建议、复测与提交的闭环;将官方字段映射到 `audit_report.md`。 + +## 本机代理监听快照(2026-08-11 21:46 +08:00) +- 只读核对结果已写入 `work/NETWORK_HANDOVER.md`(无业务内容、无凭据);`7897` 由 PID 19600 `BaoKeMengCore.exe` 监听(`::`),`18910` 由 PID 25412 `xray.exe` 监听(仅 `127.0.0.1`);`10810/10811/10822/10823` 未监听。 +- 本次未读取代理配置或凭据,未验证代理认证/链式转发/出口 IP;后续若要启动业务代理链必须获得明确指令,且 `10810/10811` 与 `18910` 只读、禁止批量终止 xray。 + +## 版本化迁移(2026-08-11) +- 可追踪主副本为仓库根的 `projects/llm_audit_ctf/`;原 `temp/projects/llm_audit_ctf/` 保留作项目模式下的本地工作与回退副本。 +- 项目级 `.gitignore` 忽略 `work/data/{raw,candidates,findings}/` 和 `work/reports_out/` 的运行数据,仅追踪各目录 `.gitkeep`、源码、规则与模板。 +- 版本化副本通过本机 `127.0.0.1` 正负回环验收:`/api/models` 的 `{"data":[...]}` 命中 Open WebUI,普通 HTTP 200 页面写入 `no_match_or_unreachable`。 diff --git a/projects/llm_audit_ctf/work/audit_report.md b/projects/llm_audit_ctf/work/audit_report.md new file mode 100644 index 000000000..4060d4c35 --- /dev/null +++ b/projects/llm_audit_ctf/work/audit_report.md @@ -0,0 +1,62 @@ +# LLM 服务暴露面审计报告(模板) + +- 报告编号:`REPORT-YYYYMMDD-NNN` +- 报告日期(UTC): +- 报告人: +- 接收方/联系人: +- 保密级别: + +## 1. 摘要 + +- 受测资产数量: +- 候选数量: +- 已确认问题数量: +- 最高风险等级: +- 一句话结论: + +## 2. 范围与方法 + +- 授权来源、范围和时间窗口: +- 目标来源及脱敏方式: +- 使用的指纹库版本: +- 扫描器版本/提交: +- 请求限制:仅低影响 GET;并发、超时、是否经代理: +- 未执行事项:登录、猜测口令、提取密钥、模型生成调用、破坏性验证。 + +## 3. 发现汇总 + +| 编号 | 资产(脱敏) | 问题类型 | 严重性 | 置信度 | 状态 | +|---|---|---|---|---|---| +| `F-001` | `` | `<类型>` | `<高/中/低>` | `<高/中/低>` | `<待确认>` | + +## 4. 详细发现 + +### F-001:<简短标题> + +- 资产与入口: +- 前置条件: +- 可复现步骤(仅授权、最小请求): +- 观察结果: +- 证据摘要:状态码、响应头必要字段、脱敏响应片段、时间戳、请求哈希: +- 影响:对机密性、完整性、可用性的具体影响;不要把“可做 2API”当作影响本身。 +- 误报排除: +- 严重性依据: +- 修复建议: +- 复测标准: + +## 5. 限制与不确定性 + +记录 TLS、WAF、代理、DNS、速率限制、未覆盖路径、资产归属未确认等因素。`no_match_or_unreachable` 不代表安全。 + +## 6. 修复跟踪 + +| 编号 | 责任方 | 建议日期 | 修复状态 | 复测日期 | 复测结论 | +|---|---|---|---|---|---| +| `F-001` | `<联系人>` | `<日期>` | `<待处理>` | `<日期>` | `<未复测>` | + +## 7. 附录 + +- 证据文件索引: +- 扫描日志摘要: +- 规则库变更记录: +- 个人信息、凭据、Cookie、Token:**不得放入报告;如必须证明,仅提供类型、位置和不可逆摘要。** diff --git a/projects/llm_audit_ctf/work/candidate_review.md b/projects/llm_audit_ctf/work/candidate_review.md new file mode 100644 index 000000000..5144fe01d --- /dev/null +++ b/projects/llm_audit_ctf/work/candidate_review.md @@ -0,0 +1,43 @@ +# 候选人工复核表 + +> 指纹命中仅代表候选,不代表漏洞确认。每个候选单独填写,敏感值不得粘贴进此文件。 + +## 资产与授权 + +- 候选编号:`CAND-YYYYMMDD-NNN` +- 目标(脱敏/规范化 URL): +- 资产归属与联系人(如未知,写“待确认”): +- 授权来源/范围/时间窗口: +- 发现时间(UTC): +- 复核人: + +## 被动/低影响证据 + +- 扫描结果文件与行号: +- 指纹分类: +- 指纹名称: +- score(仅排序参考): +- 命中路径: +- HTTP 状态: +- 证据摘要(脱敏、截断): +- 是否存在重定向、TLS 或代理不确定性: + +## 最小人工复核 + +- 是否无需登录即可观察:`是 / 否 / 不确定` +- 是否暴露模型/版本/配置元数据:`是 / 否 / 不确定` +- 是否发现凭据、Cookie、Token:`禁止记录原值;仅写类型与位置` +- 是否存在未授权模型调用:`未验证 / 已在明确授权下验证 / 不应验证` +- 请求方法、路径和时间窗口: +- 服务端可观测影响: +- 复核结论:`误报 / 信息暴露 / 访问控制问题 / 配置风险 / 待补证据` +- 置信度:`低 / 中 / 高` + +## 修复建议与复测 + +- 建议:鉴权、最小权限、密钥轮换、关闭调试/管理端点、网络隔离、速率限制等: +- 责任方: +- 期望修复时间: +- 复测条件: +- 复测结果与时间: +- 备注: diff --git a/projects/llm_audit_ctf/work/data/candidates/.gitkeep b/projects/llm_audit_ctf/work/data/candidates/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/projects/llm_audit_ctf/work/data/findings/.gitkeep b/projects/llm_audit_ctf/work/data/findings/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/projects/llm_audit_ctf/work/data/raw/.gitkeep b/projects/llm_audit_ctf/work/data/raw/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/projects/llm_audit_ctf/work/fingerprints/ai_fingerprints.json b/projects/llm_audit_ctf/work/fingerprints/ai_fingerprints.json new file mode 100644 index 000000000..219fc8c1f --- /dev/null +++ b/projects/llm_audit_ctf/work/fingerprints/ai_fingerprints.json @@ -0,0 +1,220 @@ +{ + "_meta": { + "version": "0.2", + "description": "LLM 服务暴露面指纹库:识别可被 2API 化(二次封装为 OpenAI 兼容 API)的站点", + "usage": "候选发现:每条 probe 的全部 expect 必须满足;仅保留不读取认证会话且含非状态码特征的低影响 GET 探针。score 仅用于候选排序,不代表漏洞确认。" + }, + "open_source_chat_frameworks": [ + { + "name": "NextChat (ChatGPT-Next-Web)", + "repo": "https://github.com/ChatGPTNextWeb/ChatGPT-Next-Web", + "score": 10, + "probes": [ + { + "path": "/api/models", + "expect": [ + "json_contains:gpt", + "status:200" + ] + }, + { + "path": "/", + "expect": [ + "title_contains:ChatGPT Next Web", + "html_contains:ChatGPT-Next-Web" + ] + } + ], + "notes": "部署者常未设密码/用默认 key;session 端点暴露 token 即可直接调 backend-api" + }, + { + "name": "LobeChat", + "repo": "https://github.com/lobehub/lobe-chat", + "score": 9, + "probes": [ + { + "path": "/", + "expect": [ + "title_contains:LobeChat", + "html_contains:lobehub" + ] + } + ], + "notes": "同 NextChat,很多部署走 Vercel/自托管,默认无鉴权" + }, + { + "name": "LibreChat", + "repo": "https://github.com/danny-avila/LibreChat", + "score": 9, + "probes": [ + { + "path": "/", + "expect": [ + "title_contains:LibreChat" + ] + } + ], + "notes": "自带后端代理 /api/ask,默认注册开放时可无限注册拿 token" + }, + { + "name": "Open WebUI", + "repo": "https://github.com/open-webui/open-webui", + "score": 8, + "probes": [ + { + "path": "/api/models", + "expect": [ + "json_contains:data", + "status:200" + ] + }, + { + "path": "/", + "expect": [ + "title_contains:Open WebUI" + ] + } + ], + "notes": "默认管理员 admin 弱口令常见;/api/models 常未鉴权" + }, + { + "name": "ChatGPT 官方网页版", + "repo": "https://chatgpt.com", + "score": 10, + "probes": [ + { + "path": "/backend-api/models", + "expect": [ + "json_contains:slug", + "status:200" + ] + }, + { + "path": "/", + "expect": [ + "html_contains:__NEXT_DATA__" + ] + } + ], + "notes": "2API 主战场:泄露的 access token / 共享账号可被封装" + }, + { + "name": "ChatBox Web", + "repo": "https://github.com/Bin-Huang/chatbox", + "score": 7, + "probes": [ + { + "path": "/", + "expect": [ + "html_contains:chatbox", + "html_contains:_chatbox_" + ] + } + ], + "notes": "前端套壳,key 存浏览器;主要看是否泄漏配置" + } + ], + "api_gateway_panels": [ + { + "name": "One-API", + "repo": "https://github.com/songquanpeng/one-api", + "score": 10, + "probes": [ + { + "path": "/api/status", + "expect": [ + "json_contains:data", + "status:200" + ] + }, + { + "path": "/", + "expect": [ + "title_contains:One API", + "html_contains:one-api" + ] + } + ], + "notes": "中转站本体;默认口令 root/123456 常见,登进去可看全部上游 key" + }, + { + "name": "New-API", + "repo": "https://github.com/Calcium-Ion/new-api", + "score": 10, + "probes": [ + { + "path": "/", + "expect": [ + "title_contains:New API", + "html_contains:new-api" + ] + }, + { + "path": "/api/status", + "expect": [ + "json_contains:data", + "status:200" + ] + } + ], + "notes": "One-API 分支,带 midjourney 等扩展;弱口令同样常见" + }, + { + "name": "uni-api / 其他中转面板", + "probes": [ + { + "path": "/v1/models", + "expect": [ + "json_contains:data", + "status:200" + ] + } + ], + "notes": "探测 /v1/models 未鉴权即白送模型列表" + } + ], + "raw_model_backends": [ + { + "name": "Ollama", + "score": 7, + "probes": [ + { + "path": "/api/tags", + "expect": [ + "json_contains:models", + "status:200" + ] + } + ], + "notes": "裸模型服务,无鉴权可白嫖算力/模型" + }, + { + "name": "vLLM / FastChat / llama.cpp / LocalAI (OpenAI 兼容)", + "score": 8, + "probes": [ + { + "path": "/v1/models", + "expect": [ + "json_contains:data", + "status:200" + ] + } + ], + "notes": "OpenAI 兼容端点直出,标准 2API 直接对接" + }, + { + "name": "Text Generation WebUI (oobabooga)", + "score": 7, + "probes": [ + { + "path": "/api/v1/models", + "expect": [ + "json_contains:data", + "status:200" + ] + } + ], + "notes": "暴露完整 OpenAI 兼容 API" + } + ] +} diff --git a/projects/llm_audit_ctf/work/report/.gitkeep b/projects/llm_audit_ctf/work/report/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/projects/llm_audit_ctf/work/reports_out/.gitkeep b/projects/llm_audit_ctf/work/reports_out/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/projects/llm_audit_ctf/work/scanner/probe.py b/projects/llm_audit_ctf/work/scanner/probe.py new file mode 100644 index 000000000..3a7b07549 --- /dev/null +++ b/projects/llm_audit_ctf/work/scanner/probe.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""LLM 服务候选发现器(低影响 GET 指纹探测,不执行利用)。 + +示例: + python scanner/probe.py -i data/raw/hosts.txt -o data/candidates/result.jsonl --resume + +输入每行一个裸主机、IP、host:port、[IPv6]:port 或显式 http(s) URL。 +每条 probe 的全部 expect 规则必须满足;纯状态码和认证会话路径不会形成候选。 +输出为追加式 JSONL,每个输入目标均有 candidate 或 no_match_or_unreachable 记录。 +""" +import argparse, asyncio, hashlib, ipaddress, json, re, sys, time +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlsplit + +WORK_DIR = Path(__file__).resolve().parents[1] +DEFAULT_FINGERPRINT = str(WORK_DIR / "fingerprints" / "ai_fingerprints.json") +DEFAULT_OUTPUT = str(WORK_DIR / "data" / "candidates" / "result.jsonl") +SCANNER_VERSION = "0.3" + +try: + import aiohttp + HAS_AIOHTTP = True +except ImportError: + HAS_AIOHTTP = False + +DEFAULT_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/126.0 Safari/537.36") +EV_ID = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + + +def choose_proxy(target, proxies): + """为目标稳定选择代理,避免同一目标在重跑时随机漂移。""" + if not proxies: + return None + digest = hashlib.sha256(target.encode("utf-8")).digest() + return proxies[int.from_bytes(digest[:8], "big") % len(proxies)] + + +def normalize_host(h): + """规范化目标,保留显式 scheme、端口及 IPv6;不接受带凭据的 URL。""" + h = h.strip().strip('"').strip("'") + if not h or h.startswith('#'): + return None + explicit_scheme = None + text = h + if '://' in h: + p0 = urlsplit(h) + if p0.scheme.lower() not in ('http', 'https'): + return None + explicit_scheme = p0.scheme.lower() + else: + text = '//' + h + try: + p = urlsplit(text) + if not p.hostname or p.username or p.password: + return None + host = p.hostname.rstrip('.').lower() + try: + ip = ipaddress.ip_address(host) + host = f'[{ip.compressed}]' if ip.version == 6 else ip.compressed + except ValueError: + if any(c.isspace() for c in host): + return None + authority = host if p.port is None else f'{host}:{p.port}' + return f'{explicit_scheme}://{authority}' if explicit_scheme else authority + except (ValueError, UnicodeError): + return None + + +def target_schemes(target): + return [urlsplit(target).scheme.lower()] if '://' in target else ['https', 'http'] + + +def target_authority(target): + return urlsplit(target if '://' in target else '//' + target).netloc + + +def _json_contains(value, needle): + if isinstance(value, dict): + return needle in value or any(_json_contains(v, needle) for v in value.values()) + if isinstance(value, list): + return any(_json_contains(v, needle) for v in value) + return needle.lower() in str(value).lower() + + +def match_expect(expect, status, headers, body): + """校验一条规则;调用方负责将一条 probe 的所有规则做 AND 聚合。""" + try: + if expect.startswith("status:"): + code = expect.split(":", 1)[1].lower() + return str(status).startswith(code[0]) if code.endswith("xx") else status == int(code) + if expect.startswith("json_contains:"): + return _json_contains(json.loads(body), expect.split(":", 1)[1]) + if expect.startswith("html_contains:"): + return expect.split(":", 1)[1].lower() in body.lower() + if expect.startswith("title_contains:"): + title = re.search(r"]*>(.*?)", body, re.S | re.I) + return bool(title and expect.split(":", 1)[1].lower() in title.group(1).lower()) + if expect.startswith("header_contains:"): + name, _, value = expect.split(":", 1)[1].partition(":") + return value.lower() in next((v for k, v in headers.items() if k.lower() == name.lower()), "").lower() + except (ValueError, TypeError, json.JSONDecodeError): + return False + return False + + +def redact_evidence(body): + """只输出有限且脱敏后的 JSON/文本片段,避免候选库收集令牌。""" + try: + value = json.loads(body) + def walk(node): + if isinstance(node, dict): + return {k: "[REDACTED]" if re.search(r"token|secret|password|api[_-]?key|authorization|cookie", k, re.I) else walk(v) for k, v in node.items()} + if isinstance(node, list): + return [walk(v) for v in node[:10]] + return node + return json.dumps(walk(value), ensure_ascii=False)[:600] + except (ValueError, TypeError, json.JSONDecodeError): + return re.sub(r"(?i)(token|secret|password|api[_-]?key|authorization)\s*[:=]\s*[^\s<,;]+", r"\1=[REDACTED]", body)[:300] + + +async def probe_one(session, authority, base_scheme, path, timeout, sem, proxy=None, insecure=False, follow_redirects=False): + """发一条低影响 GET;返回成功、状态、头、正文、最终 URL。""" + url = f"{base_scheme}://{authority}{path}" + try: + async with sem: + async with session.get(url, timeout=timeout, proxy=proxy, allow_redirects=follow_redirects, + ssl=False if insecure else None) as response: + return True, response.status, dict(response.headers), await response.text(errors="replace"), str(response.url) + except Exception: + return False, 0, {}, "", "" + + +async def scan_target(session, target, fp, timeout, sem, probe_only=None, proxy=None, + insecure=False, follow_redirects=False): + """对单个目标做低影响 GET 指纹探测;一条 probe 的 expect 必须全部满足。""" + for scheme in target_schemes(target): + authority = target_authority(target) + for fp_cat, cat_name in (("open_source_chat_frameworks", "chat_framework"), + ("api_gateway_panels", "gateway_panel"), + ("raw_model_backends", "model_backend")): + for item in fp.get(fp_cat, []): + probes = item.get("probes", []) + if probe_only: + probes = [p for p in probes if p.get("path") in probe_only] + for pr in probes: + path = pr.get("path", "/") + # 认证会话可能包含真实令牌;默认不采集。 + if re.search(r"/auth/|/session|token", path, re.I): + continue + expects = pr.get("expect", []) + strong = [e for e in expects if not e.startswith("status:")] + # 空 expect 或纯 status 规则不能提供稳定指纹证据。 + if not strong: + continue + ok, status, headers, body, final_url = await probe_one( + session, authority, scheme, path, timeout, sem, proxy, + insecure, follow_redirects) + if not ok or not all(match_expect(e, status, headers, body) for e in expects): + continue + return { + "target": target, "cat": cat_name, "name": item.get("name", "unknown"), + "score": item.get("score", 0), + "hits": [{"path": path, "status": status, "scheme": scheme, + "evidence": redact_evidence(body), + "final_url": final_url}], + "ts": datetime.now(timezone.utc).isoformat() + } + return None + + +async def main(): + ap = argparse.ArgumentParser(description="LLM 服务候选发现器(仅低影响 GET 指纹探测)") + ap.add_argument("-i", "--input", help="目标文件;省略时从标准输入读取") + ap.add_argument("-o", "--output", default=DEFAULT_OUTPUT) + ap.add_argument("-p", "--proxies", help="HTTP(S) 代理列表;只在请求中使用,不写入结果") + ap.add_argument("-c", "--concurrency", type=int, default=12) + ap.add_argument("-t", "--timeout", type=float, default=8.0) + ap.add_argument("-f", "--fingerprint", default=DEFAULT_FINGERPRINT) + ap.add_argument("--limit", type=int, default=0) + ap.add_argument("--probe-only", help="逗号分隔的路径白名单") + ap.add_argument("--resume", action="store_true", help="跳过输出 JSONL 中已记录的目标") + ap.add_argument("--insecure", action="store_true", help="仅在本地测试时关闭 TLS 证书校验") + ap.add_argument("--follow-redirects", action="store_true", help="显式允许跟随重定向") + args = ap.parse_args() + if not HAS_AIOHTTP: + print("[!] 需要 aiohttp: pip install aiohttp", file=sys.stderr) + sys.exit(1) + if not 1 <= args.concurrency <= 100 or args.timeout <= 0: + ap.error("--concurrency 必须为 1..100,--timeout 必须大于 0") + with open(args.fingerprint, encoding="utf-8") as f: + fp = json.load(f) + + raw_lines = open(args.input, encoding="utf-8", errors="replace") if args.input else sys.stdin + try: + targets = [h for line in raw_lines if (h := normalize_host(line))] + finally: + if args.input: + raw_lines.close() + targets = list(dict.fromkeys(targets)) + if args.limit: + targets = targets[:args.limit] + + completed = set() + if args.resume: + try: + with open(args.output, encoding="utf-8", errors="replace") as f: + for line in f: + try: + record = json.loads(line) + completed.add(record.get("target", record.get("host"))) + except json.JSONDecodeError: + continue + except FileNotFoundError: + pass + targets = [target for target in targets if target not in completed] + proxy_list = [] + if args.proxies: + with open(args.proxies, encoding="utf-8") as f: + proxy_list = [line.strip() for line in f if line.strip() and not line.lstrip().startswith("#")] + if any(urlsplit(proxy).scheme.lower() not in ("http", "https") or not urlsplit(proxy).hostname + for proxy in proxy_list): + ap.error("--proxies 中的每项必须是完整的 http(s) 代理 URL") + probe_only = ({path.strip() for path in args.probe_only.split(",") if path.strip()} + if args.probe_only else None) + if args.probe_only and not probe_only: + ap.error("--probe-only 至少包含一个非空路径") + if probe_only and any(not path.startswith("/") for path in probe_only): + ap.error("--probe-only 中的路径必须以 / 开头") + run_meta = { + "run_id": EV_ID, + "scanner_version": SCANNER_VERSION, + "fingerprint_version": str(fp.get("_meta", {}).get("version", "unknown")), + "timeout_seconds": args.timeout, + "follow_redirects": args.follow_redirects, + "insecure": args.insecure, + "probe_only": sorted(probe_only) if probe_only else None, + "proxy_count": len(proxy_list), + } + print(f"[*] 待测 {len(targets)} 个,框架 {sum(len(fp.get(k, [])) for k in fp if not k.startswith('_'))} 组,并发 {args.concurrency},代理 {len(proxy_list)} 条", flush=True) + + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + sem, queue = asyncio.Semaphore(args.concurrency), asyncio.Queue() + for target in targets: + queue.put_nowait(target) + tested = candidates = 0 + start = time.time() + headers = {"User-Agent": DEFAULT_UA, "Accept": "application/json,text/html;q=0.9,*/*;q=0.1"} + connector = aiohttp.TCPConnector(limit=args.concurrency, ssl=None) + with open(args.output, "a", encoding="utf-8") as out: + async with aiohttp.ClientSession(connector=connector, headers=headers) as session: + async def worker(): + nonlocal tested, candidates + while True: + try: + target = queue.get_nowait() + except asyncio.QueueEmpty: + return + proxy = choose_proxy(target, proxy_list) + result = await scan_target(session, target, fp, args.timeout, sem, probe_only, proxy, + args.insecure, args.follow_redirects) + record = result or {"target": target, "outcome": "no_match_or_unreachable", + "ts": datetime.now(timezone.utc).isoformat()} + record["run"] = run_meta + if result: + record["outcome"] = "candidate" + candidates += 1 + out.write(json.dumps(record, ensure_ascii=False) + "\n") + out.flush() + tested += 1 + if tested % 25 == 0: + print(f"[*] 已测 {tested}/{len(targets)},候选 {candidates},耗时 {time.time()-start:.0f}s", flush=True) + queue.task_done() + await asyncio.gather(*(worker() for _ in range(min(args.concurrency, len(targets) or 1)))) + print(f"[+] 完成:测试 {tested},候选 {candidates},追加写入 {args.output},耗时 {time.time()-start:.0f}s", flush=True) + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n[!] 中断", file=sys.stderr) + sys.exit(130) diff --git a/projects/llm_audit_ctf/work/tools/.gitkeep b/projects/llm_audit_ctf/work/tools/.gitkeep new file mode 100644 index 000000000..e69de29bb