diff --git a/capabilities/android-apk-research/capability.yaml b/capabilities/android-apk-research/capability.yaml new file mode 100644 index 0000000..abf49ed --- /dev/null +++ b/capabilities/android-apk-research/capability.yaml @@ -0,0 +1,54 @@ +schema: 1 +name: android-apk-research +version: "0.2.0" +description: > + Static semantic-bug research on Android APKs — deep-link routers, intent + redirection, WebView trust boundaries, auth/session/client-state bypass, + Dirty Stream share targets, and APK-derived backend API chains. Ships a + 10-tool orchestration MCP (parallel Androguard/APKiD inventory, + component ranking, runtime classification, DexProtector detection + + static unpack, API map extraction, finding-schema normalization) and + skills teaching the JADX / ripgrep / Semgrep / Joern / CodeQL pipeline + against MASVS/MASTG, CWE, and MASWE. + +mcp: + servers: + android-research: + command: "uv" + args: + - "run" + - "${CAPABILITY_ROOT}/mcp/android_research.py" + env: + ANDROID_RESEARCH_MAX_OUTPUT_CHARS: "${ANDROID_RESEARCH_MAX_OUTPUT_CHARS:-20000}" + ANDROID_RESEARCH_TIMEOUT: "${ANDROID_RESEARCH_TIMEOUT:-300}" + init_timeout: 120 + +checks: + - name: uv + command: 'command -v uv >/dev/null 2>&1' + - name: jadx + command: 'command -v jadx >/dev/null 2>&1' + - name: apktool + command: 'command -v apktool >/dev/null 2>&1' + - name: aapt-or-aapt2 + command: 'command -v aapt >/dev/null 2>&1 || command -v aapt2 >/dev/null 2>&1' + - name: semgrep + command: 'command -v semgrep >/dev/null 2>&1' + - name: apkid + command: 'command -v apkid >/dev/null 2>&1' + +author: + name: Dreadnode + url: https://dreadnode.io +license: MIT +repository: https://github.com/dreadnode/capabilities +keywords: + - android + - apk + - mobile-security + - vulnerability-research + - logic-bugs + - masvs + - mastg + - maswe + - cwe diff --git a/capabilities/android-apk-research/mcp/android_research.py b/capabilities/android-apk-research/mcp/android_research.py new file mode 100644 index 0000000..39b2fe6 --- /dev/null +++ b/capabilities/android-apk-research/mcp/android_research.py @@ -0,0 +1,670 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "fastmcp>=2.0", +# ] +# /// +"""Orchestration MCP for Android APK semantic-bug research. + +Wraps the per-script scripts in `../scripts/` behind a typed tool surface so +agents get tools/list discovery and uniform return shapes, while the heavyweight +methodology (jadx heap tiers, semgrep rule-pack ensembles, joern recipes, codeql +query packs, rg pattern selection) stays in the skills as bash. See +`skills/android-semantic-vuln-hunting/references/workflow.md` for the +"Why bash, not MCP" rationale on JADX / Semgrep / Joern. + +Tools registered: + + * inventory_status — probe which underlying tools are wired up + * run_corpus_inventory — parallel Androguard+APKiD inventory of an APK set + * extract_components — one row per (apk, component) for ranking + * rank_components — apply risk priors; emit components_ranked.md + * detect_runtime_kind — classify APK runtime (native / RN / Flutter / ...) + * detect_protector — detect DexProtector / Promon Shield signals + * dexprotector_unpack — static libdp.so recovery (arm64-v8a only) + * extract_api_map — regex-based APK→backend API/DTO/auth map + * rank_backend_richness — sort backend_richness.json summaries + * normalize_semantic_findings — render finding JSONL into Markdown/CSV/JSONL + +Per-APK artifacts (inventory, findings, hypotheses, reports) are operator-owned +and live under the path the caller supplies (typically `findings//`). The +MCP itself does not own any cache. If the capability later grows a derived- +artifact cache, follow the sibling convention +`${ANDROID_RESEARCH_CACHE_ROOT:-~/.dreadnode/cache/android-apk-research/}`. + +Script invocation style: scripts with third-party deps (`protector_detect.py`, +`dexprotector_unpack.py`) carry a `uv run --script` shebang + PEP 723 block and +are invoked directly; stdlib-only scripts are invoked via `python3 path/to.py`. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import shutil +import sys +from pathlib import Path +from typing import Annotated, Any, Literal + +from fastmcp import FastMCP + +mcp = FastMCP("android-research") + +CAPABILITY_ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = CAPABILITY_ROOT / "scripts" + +MAX_OUTPUT_CHARS = int(os.environ.get("ANDROID_RESEARCH_MAX_OUTPUT_CHARS", "20000")) +DEFAULT_TIMEOUT = int(os.environ.get("ANDROID_RESEARCH_TIMEOUT", "300")) + +RUNTIME_KINDS = { + "native", + "react_native_js", + "react_native_hermes", + "flutter_aot", + "capacitor", + "cordova", + "unity", + "xamarin", + "maui", + "unknown", +} + + +def _truncate(text: str) -> str: + if len(text) <= MAX_OUTPUT_CHARS: + return text + return text[:MAX_OUTPUT_CHARS] + "\n...[truncated]..." + + +def _resolve_existing(path: str) -> Path: + p = Path(path).expanduser().resolve() + if not p.exists(): + raise FileNotFoundError(f"path does not exist: {p}") + return p + + +def _resolve_out(path: str) -> Path: + p = Path(path).expanduser().resolve() + p.parent.mkdir(parents=True, exist_ok=True) + return p + + +def _which(name: str) -> str | None: + return shutil.which(name) + + +async def _run( + argv: list[str], + *, + timeout: int, +) -> tuple[int, str]: + """Run a subprocess, return (returncode, merged_stdout_stderr_text). + + Output is truncated to MAX_OUTPUT_CHARS at the tail. Raises TimeoutError + on timeout so the MCP surfaces it to the agent as a tool-call error. + """ + proc = await asyncio.create_subprocess_exec( + *argv, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + try: + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except asyncio.TimeoutError: + proc.kill() + await proc.communicate() + raise TimeoutError(f"command timed out after {timeout}s: {argv[0]}") from None + rc = proc.returncode if proc.returncode is not None else 1 + return rc, _truncate(stdout.decode("utf-8", errors="ignore")) + + +def _strip_none(d: dict[str, Any]) -> dict[str, Any]: + """Drop keys whose values are None/empty so the agent doesn't pay tokens + for null-coerced fields. Keeps 0 and False on purpose.""" + return { + k: v for k, v in d.items() if v is not None and v != "" and v != [] and v != {} + } + + +# ── inventory_status ───────────────────────────────────────────────── + + +@mcp.tool +async def inventory_status() -> dict[str, Any]: + """Probe whether the underlying CLIs and scripts this MCP relies on are + reachable on this host. + + Each value is either "ok" (present) or a one-line reason it isn't. The + methodology tools (jadx, semgrep, joern, codeql) live in the skill prose + but are checked here so the agent can decide which skill steps will work + end-to-end. Read this once at session start, not before every tool call. + """ + status: dict[str, Any] = {"capability_root": str(CAPABILITY_ROOT)} + + # Hard prerequisites for this MCP's tools. + for name, hint in [ + ("python3", "needed for the orchestrator scripts"), + ( + "uv", + "PEP 723 scripts (androguard_inventory, protector_detect, dexprotector_unpack)", + ), + ("apkid", "packer/protector signal during run_corpus_inventory"), + ("aapt2", "manifest fallback when androguard errors on multi-dex APKs"), + ("aapt", "manifest fallback alternative"), + ]: + path = _which(name) + status[name] = path if path else f"missing — {hint}" + + # Skill-step tools — not required to boot the MCP, but the agent should + # know whether step 3-9 of android-semantic-vuln-hunting will work. + for name in ("jadx", "semgrep", "joern", "codeql", "adb"): + path = _which(name) + status[name] = path if path else "missing" + + # Hybrid-runtime follow-ups (Step 7.5 / 7.6). + for name in ("hbctool", "blutter", "prettier", "npx"): + path = _which(name) + status[name] = path if path else "missing" + + # Per-script presence is covered by mcp/test_server.py::TestScriptWiring. + + return _strip_none(status) + + +# ── run_corpus_inventory ───────────────────────────────────────────── + + +@mcp.tool +async def run_corpus_inventory( + paths: Annotated[ + list[str], "APK files or directories containing APKs to inventory in parallel" + ], + out_dir: Annotated[ + str, "Output run directory for per-APK artifacts and aggregate JSONL" + ], + jobs: Annotated[int, "Parallel worker count"] = 4, + resume: Annotated[bool, "Skip APKs with existing ok status"] = True, + timeout: Annotated[int, "Per-APK inventory timeout in seconds"] = 180, + limit: Annotated[int | None, "Optional APK limit for smoke tests"] = None, + include_apkid: Annotated[ + bool, "Run APKiD when installed for packer/protector signal" + ] = True, + preview_limit: Annotated[ + int, "Number of compact preview records to return inline" + ] = 20, +) -> dict[str, Any]: + """Run a parallel, resumable first-pass inventory over an APK corpus. + + Each APK becomes a SHA256-keyed artifact directory with `inventory.json`, + `androguard.json` (decoded manifest, components, schemes, hosts, browsable + components), optional `apkid.json`, and `status.json`. Aggregate + `attack_surface.jsonl` and `status.jsonl` files are written for ranking. + """ + if not paths: + raise ValueError("paths must contain at least one APK file or directory") + out_path = _resolve_out(out_dir) + cmd = [ + "python3", + str(SCRIPTS / "run_corpus_inventory.py"), + *[str(Path(p).expanduser().resolve()) for p in paths], + "--out-dir", + str(out_path), + "--jobs", + str(jobs), + "--timeout", + str(timeout), + "--preview", + str(preview_limit), + ] + if resume: + cmd.append("--resume") + if limit is not None: + cmd.extend(["--limit", str(limit)]) + if include_apkid: + cmd.append("--include-apkid") + if limit is not None: + outer_timeout = max(timeout * limit // max(1, jobs) + 600, 900) + else: + outer_timeout = 8 * 3600 + rc, output = await _run(cmd, timeout=outer_timeout) + + summary_path = out_path / "run_status.json" + summary: dict[str, Any] = {} + if summary_path.exists(): + try: + summary = json.loads(summary_path.read_text()) + except Exception: + summary = {} + + response: dict[str, Any] = { + "returncode": rc, + "out_dir": str(out_path), + "attack_surface_jsonl": str(out_path / "attack_surface.jsonl"), + "status_jsonl": str(out_path / "status.jsonl"), + "summary_path": str(summary_path), + } + if summary: + response["summary"] = summary + else: + response["output"] = output + return _strip_none(response) + + +# ── extract_components ─────────────────────────────────────────────── + + +@mcp.tool +async def extract_components( + inventory_dir: Annotated[ + str, + "Directory containing per-APK SHA256 subdirs (the apks/ root from run_corpus_inventory)", + ], + out: Annotated[str, "Output components.jsonl path (one row per component)"], + triage_manifest: Annotated[ + str | None, + "Optional triage manifest JSONL keyed by sha256 for apk_path / impact_class joins", + ] = None, + runtime_kind: Annotated[ + str | None, + "Optional runtime_kind JSONL from detect_runtime_kind sweep", + ] = None, + timeout: Annotated[int, "Timeout in seconds"] = 600, +) -> dict[str, Any]: + """Emit one JSONL row per (apk, component) by streaming every + `androguard.json` under the inventory directory, falling back to + `aapt2 dump xmltree` for APKs where Androguard errored. + + Output rows carry exported/permission/scheme/host/path/action facts plus + APK-level joins (apkid_tier, impact_class, runtime_kind). Feeds + `rank_components`. See `scripts/extract_corpus_components.py` for the + full schema. + """ + inv = _resolve_existing(inventory_dir) + out_path = _resolve_out(out) + cmd = [ + "python3", + str(SCRIPTS / "extract_corpus_components.py"), + "--inventory-dir", + str(inv), + "--out", + str(out_path), + ] + if triage_manifest: + cmd.extend(["--triage-manifest", str(_resolve_existing(triage_manifest))]) + if runtime_kind: + cmd.extend(["--runtime-kind", str(_resolve_existing(runtime_kind))]) + rc, output = await _run(cmd, timeout=timeout) + return _strip_none( + { + "returncode": rc, + "components_jsonl": str(out_path), + "row_count": _count_lines(out_path) if out_path.exists() else 0, + "output": output, + } + ) + + +# ── rank_components ────────────────────────────────────────────────── + + +@mcp.tool +async def rank_components( + components: Annotated[str, "Input components.jsonl from extract_components"], + out_jsonl: Annotated[str, "Output ranked components JSONL path"], + out_md: Annotated[str | None, "Optional Markdown operator-inbox path"] = None, + top_md: Annotated[int, "Max rows in the Markdown inbox (default 150)"] = 150, + min_score: Annotated[ + int, "Drop components scoring below this in the Markdown inbox" + ] = 4, + timeout: Annotated[int, "Timeout in seconds"] = 300, +) -> dict[str, Any]: + """Apply risk priors to each component row and emit a ranked inbox. + + The full prior table lives in `scripts/rank_components.py`. Short version: + exported BROWSABLE without permission = +5; host wildcard or high-risk + path/scheme = +3; heavy packer = -12. Each row gets a `read_budget` tag + (`5m` if score>=7, `1m` if 3-6, `skip` otherwise) so Tier C can budget. + """ + src = _resolve_existing(components) + out_path = _resolve_out(out_jsonl) + cmd = [ + "python3", + str(SCRIPTS / "rank_components.py"), + "--components", + str(src), + "--out-jsonl", + str(out_path), + "--top-md", + str(top_md), + "--min-score", + str(min_score), + ] + md_out: Path | None = None + if out_md: + md_out = _resolve_out(out_md) + cmd.extend(["--out-md", str(md_out)]) + rc, output = await _run(cmd, timeout=timeout) + return _strip_none( + { + "returncode": rc, + "ranked_jsonl": str(out_path), + "ranked_md": str(md_out) if md_out else None, + "row_count": _count_lines(out_path) if out_path.exists() else 0, + "output": output, + } + ) + + +# ── detect_runtime_kind ────────────────────────────────────────────── + + +@mcp.tool +async def detect_runtime_kind( + apk: Annotated[str, "Path to a single APK to classify"], + timeout: Annotated[int, "Timeout in seconds"] = 30, +) -> dict[str, Any]: + """Classify an APK's runtime in one second using `unzip -l` only. + + Returns one of: `native`, `react_native_js`, `react_native_hermes`, + `flutter_aot`, `capacitor`, `cordova`, `unity`, `xamarin`, `maui`, + `unknown`. Drives JADX heap sizing and routes to Step 7.5 (JS bundle + trace) or 7.6 (Dart AOT trace) when non-native. + """ + apk_path = _resolve_existing(apk) + cmd = [ + "bash", + str(SCRIPTS / "detect_runtime_kind.sh"), + "--jsonl", + str(apk_path), + ] + rc, output = await _run(cmd, timeout=timeout) + if rc != 0: + raise RuntimeError(f"detect_runtime_kind.sh failed (rc={rc}): {output}") + # Script prints JSONL — one line per APK. + line = output.strip().splitlines()[-1] if output.strip() else "" + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"detect_runtime_kind.sh produced non-JSON: {line!r}" + ) from exc + kind = record.get("runtime_kind", "unknown") + if kind not in RUNTIME_KINDS: + # Defensive: surface the value but tag it. + record["runtime_kind_warning"] = f"unrecognized runtime_kind: {kind!r}" + return record + + +# ── detect_protector ───────────────────────────────────────────────── + + +@mcp.tool +async def detect_protector( + target: Annotated[str, "APK path or inventory dir with per-SHA artifacts"], + out: Annotated[ + str | None, + "Optional output protector.json path; default is alongside the target", + ] = None, + timeout: Annotated[int, "Timeout in seconds"] = 120, +) -> dict[str, Any]: + """Detect commercial Android protectors (DexProtector, Promon Shield) and + recommend a triage strategy. + + Key fields in the returned record: `protector` (`dexprotector` / + `promon_shield` / `unknown`), `confidence` (high/medium/low), + `triage_strategy` (`protector_aware` vs `default`), and + `artifacts.dexprotector_unpack_supported` (true when arm64-v8a libdexprotector + is present and `dexprotector_unpack` will succeed). + + If `dexprotector_unpack_supported` is true, the next step is calling + `dexprotector_unpack` on the same APK. Otherwise, the recommended path is + documented in `skills/android-protector-triage/SKILL.md` §3 (adjacency + analysis only). + """ + target_path = _resolve_existing(target) + cmd = [ + str(SCRIPTS / "protector_detect.py"), + str(target_path), + ] + if out: + out_path = _resolve_out(out) + cmd.extend(["-o", str(out_path)]) + rc, output = await _run(cmd, timeout=timeout) + if rc != 0: + raise RuntimeError(f"protector_detect.py failed (rc={rc}): {output}") + # The script writes the JSON to stdout or to -o; parse stdout when no -o. + if out: + try: + return _strip_none(json.loads(Path(out).expanduser().resolve().read_text())) + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"could not read protector_detect output at {out}: {exc}" + ) from exc + # Stdout path — last JSON object in the output. + try: + return _strip_none(json.loads(output)) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"protector_detect.py produced non-JSON stdout: {output[:500]!r}" + ) from exc + + +# ── dexprotector_unpack ────────────────────────────────────────────── + + +@mcp.tool +async def dexprotector_unpack( + apk: Annotated[ + str, + "Path to a DexProtector-protected APK (arm64-v8a libdexprotector required)", + ], + out: Annotated[str, "Output path for the recovered libdp.so"], + timeout: Annotated[int, "Timeout in seconds (Unicorn emulation can be slow)"] = 600, +) -> dict[str, Any]: + """Static-unpack DexProtector's libdp.so without an Android device. + + Emulates the libdexprotector.so bootstrap chain via Unicorn to recover the + plain libdp.so (which is the entry point for subsequent classes.dex.dat + and `assets/se.dat` recovery). Does NOT execute libdp.so; everything is + static. Does NOT trigger the master-key corruption described in Romain + Thomas's writeup (libdp.so is never hooked). + + Run `detect_protector` first and only call this if + `artifacts.dexprotector_unpack_supported` is true. ARM64-v8a only today. + """ + apk_path = _resolve_existing(apk) + out_path = _resolve_out(out) + cmd = [ + str(SCRIPTS / "dexprotector_unpack.py"), + str(apk_path), + "-o", + str(out_path), + ] + rc, output = await _run(cmd, timeout=timeout) + if rc != 0: + raise RuntimeError( + f"dexprotector_unpack.py failed (rc={rc}). " + f"Check that the APK ships arm64-v8a libdexprotector.so and that " + f"detect_protector reported dexprotector_unpack_supported=true. " + f"Output: {output}" + ) + if not out_path.exists(): + raise RuntimeError( + f"dexprotector_unpack.py returned 0 but wrote no output at {out_path}" + ) + return { + "returncode": rc, + "libdp_so": str(out_path), + "size": out_path.stat().st_size, + "output": output, + } + + +# ── extract_api_map ────────────────────────────────────────────────── + + +@mcp.tool +async def extract_api_map( + src: Annotated[ + str, + "Decompiled source tree, JS bundle dir, or Dart blutter output dir to scan", + ], + out: Annotated[str, "Output api_map.jsonl path (one row per finding)"], + summary: Annotated[ + str | None, + "Optional output backend_richness.json path with aggregate scores", + ] = None, + dedupe: Annotated[bool, "Deduplicate rows by (kind, value, file)"] = True, + timeout: Annotated[int, "Timeout in seconds"] = 600, +) -> dict[str, Any]: + """Regex-extract API endpoints, generated clients, request-signing hints, + feature flags, object IDs, and workflow verbs from decompiled APK sources. + + Output is a target map for APK→backend hypotheses, NOT proof of + vulnerability. Backend findings default to `needs_backend_validation` per + `references/backend-rich-apk-workflows.md` until tested against authorized + accounts/QA. Also works on `findings//js-analysis` (Step 7.5 output) + and `findings//dart-analysis` (Step 7.6 / blutter output). + """ + src_path = _resolve_existing(src) + out_path = _resolve_out(out) + cmd = [ + "python3", + str(SCRIPTS / "extract_api_map.py"), + "--src", + str(src_path), + "--out", + str(out_path), + ] + summary_path: Path | None = None + if summary: + summary_path = _resolve_out(summary) + cmd.extend(["--summary", str(summary_path)]) + if dedupe: + cmd.append("--dedupe") + rc, output = await _run(cmd, timeout=timeout) + summary_data: dict[str, Any] = {} + if summary_path and summary_path.exists(): + try: + summary_data = json.loads(summary_path.read_text()) + except Exception: + pass + return _strip_none( + { + "returncode": rc, + "api_map_jsonl": str(out_path), + "summary_path": str(summary_path) if summary_path else None, + "row_count": _count_lines(out_path) if out_path.exists() else 0, + "summary": summary_data, + "output": output if not summary_data else None, + } + ) + + +# ── rank_backend_richness ──────────────────────────────────────────── + + +@mcp.tool +async def rank_backend_richness( + summaries: Annotated[ + list[str], + "List of backend_richness.json files produced by extract_api_map", + ], + out_jsonl: Annotated[str, "Output sorted JSONL path"], + out_md: Annotated[str | None, "Optional Markdown inbox path"] = None, + timeout: Annotated[int, "Timeout in seconds"] = 120, +) -> dict[str, Any]: + """Sort backend_richness summaries by score and emit an operator inbox. + + Each row carries score / richness band / unique-value counts / synergy + flags (signed_requests + tenant_ids + workflow_verbs, etc.). Read the + Markdown top-down as the next-targets-to-probe queue. + """ + if not summaries: + raise ValueError("summaries must contain at least one backend_richness.json") + out_path = _resolve_out(out_jsonl) + cmd = [ + "python3", + str(SCRIPTS / "rank_backend_richness.py"), + *[str(_resolve_existing(s)) for s in summaries], + "--out-jsonl", + str(out_path), + ] + md_out: Path | None = None + if out_md: + md_out = _resolve_out(out_md) + cmd.extend(["--out-md", str(md_out)]) + rc, output = await _run(cmd, timeout=timeout) + return _strip_none( + { + "returncode": rc, + "ranked_jsonl": str(out_path), + "ranked_md": str(md_out) if md_out else None, + "row_count": _count_lines(out_path) if out_path.exists() else 0, + "output": output, + } + ) + + +# ── normalize_semantic_findings ────────────────────────────────────── + + +@mcp.tool +async def normalize_semantic_findings( + inputs: Annotated[ + list[str], "JSON or JSONL files containing semantic finding hypotheses" + ], + output_format: Annotated[ + Literal["markdown", "jsonl", "csv"], "Render format" + ] = "markdown", + out: Annotated[str | None, "Optional output file path"] = None, + timeout: Annotated[int, "Timeout in seconds"] = 120, +) -> dict[str, Any]: + """Normalize, deduplicate, and render Android semantic finding hypotheses. + + Enforces a deterministic schema (entrypoint, source, trust boundary, sink, + impact, evidence, validation plan, MASVS / CWE / MASWE tags, scanner gap, + confidence and validation tiers, missing evidence) so reports stay + comparable across runs. See + `skills/android-semantic-vuln-hunting/references/output-schema.md` for + the per-field reference. + """ + if not inputs: + raise ValueError("inputs must contain at least one findings file") + cmd = [ + "python3", + str(SCRIPTS / "normalize_findings.py"), + *[str(_resolve_existing(p)) for p in inputs], + "--format", + output_format, + ] + out_path: Path | None = None + if out: + out_path = _resolve_out(out) + cmd.extend(["--out", str(out_path)]) + rc, output = await _run(cmd, timeout=timeout) + response: dict[str, Any] = { + "returncode": rc, + "output": output, + } + if out_path: + response["output_path"] = str(out_path) + response["bytes"] = out_path.stat().st_size if out_path.exists() else 0 + return _strip_none(response) + + +# ── internal helpers ───────────────────────────────────────────────── + + +def _count_lines(path: Path) -> int: + try: + with path.open("rb") as f: + return sum(1 for _ in f) + except OSError: + return 0 + + +if __name__ == "__main__": + mcp.run() diff --git a/capabilities/android-apk-research/mcp/test_server.py b/capabilities/android-apk-research/mcp/test_server.py new file mode 100644 index 0000000..15ccb72 --- /dev/null +++ b/capabilities/android-apk-research/mcp/test_server.py @@ -0,0 +1,184 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "pytest>=8.0", +# "pytest-asyncio>=0.23", +# "fastmcp>=2.0", +# ] +# /// +"""Tests for the android-apk-research MCP server — no APK required. + +Covers pure helpers and verifies the tool surface the skills and agent +reference. The heavyweight downstream tools (jadx, semgrep, joern, codeql) +have their own test suites and are out of scope here. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +HERE = Path(__file__).parent + + +def _load(filename: str, alias: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(alias, HERE / f"{filename}.py") + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + sys.modules[alias] = mod + spec.loader.exec_module(mod) + return mod + + +@pytest.fixture(scope="module") +def server() -> ModuleType: + return _load("android_research", "android_research_mcp") + + +class TestHelpers: + def test_truncate_passthrough_short(self, server): + assert server._truncate("hi") == "hi" + + def test_truncate_chops_long(self, server, monkeypatch): + monkeypatch.setattr(server, "MAX_OUTPUT_CHARS", 8) + out = server._truncate("0123456789") + assert out.startswith("01234567") + assert "truncated" in out + + def test_strip_none_drops_empties_keeps_falsy(self, server): + d = server._strip_none( + { + "a": 0, + "b": False, + "c": None, + "d": "", + "e": [], + "f": {}, + "g": "keep", + "h": [1, 2], + } + ) + assert d == {"a": 0, "b": False, "g": "keep", "h": [1, 2]} + + def test_resolve_existing_raises_on_missing(self, server, tmp_path): + with pytest.raises(FileNotFoundError): + server._resolve_existing(str(tmp_path / "nope")) + + def test_resolve_existing_returns_resolved(self, server, tmp_path): + f = tmp_path / "f.txt" + f.write_text("hi") + assert server._resolve_existing(str(f)) == f.resolve() + + def test_resolve_out_creates_parent(self, server, tmp_path): + out = tmp_path / "a" / "b" / "c.json" + resolved = server._resolve_out(str(out)) + assert resolved == out.resolve() + assert out.parent.exists() + + def test_count_lines(self, server, tmp_path): + f = tmp_path / "lines.jsonl" + f.write_text("a\nb\nc\n") + assert server._count_lines(f) == 3 + assert server._count_lines(tmp_path / "missing") == 0 + + def test_runtime_kinds_complete(self, server): + # Must include every value detect_runtime_kind.sh emits. + # Source of truth: scripts/detect_runtime_kind.sh header. + assert { + "native", + "react_native_js", + "react_native_hermes", + "flutter_aot", + "capacitor", + "cordova", + "unity", + "xamarin", + "maui", + }.issubset(server.RUNTIME_KINDS) + + +class TestRegistration: + @pytest.mark.asyncio + async def test_lists_documented_tools(self, server): + tools = await server.mcp.list_tools() + names = {t.name for t in tools} + # Lock the surface. If a tool is added or removed, the skill prose + # and agent-utility-index must be updated in the same change. + assert names == { + "inventory_status", + "run_corpus_inventory", + "extract_components", + "rank_components", + "detect_runtime_kind", + "detect_protector", + "dexprotector_unpack", + "extract_api_map", + "rank_backend_richness", + "normalize_semantic_findings", + } + + @pytest.mark.asyncio + async def test_inventory_status_returns_dict(self, server): + result = await server.mcp.call_tool("inventory_status", {}) + body = result.structured_content + # Always reports the capability root — that's the one field we can + # rely on regardless of which CLIs the host has. + assert "capability_root" in body + + @pytest.mark.asyncio + async def test_run_corpus_inventory_requires_paths(self, server): + with pytest.raises(Exception) as ei: + await server.mcp.call_tool( + "run_corpus_inventory", {"paths": [], "out_dir": "/tmp/x"} + ) + assert "paths" in str(ei.value).lower() + + @pytest.mark.asyncio + async def test_normalize_findings_requires_inputs(self, server): + with pytest.raises(Exception) as ei: + await server.mcp.call_tool("normalize_semantic_findings", {"inputs": []}) + assert "inputs" in str(ei.value).lower() + + @pytest.mark.asyncio + async def test_rank_backend_richness_requires_summaries(self, server): + with pytest.raises(Exception) as ei: + await server.mcp.call_tool( + "rank_backend_richness", {"summaries": [], "out_jsonl": "/tmp/x"} + ) + assert "summaries" in str(ei.value).lower() + + +class TestScriptWiring: + """Verify every script the MCP shells out to actually exists. + + Catches the rename-drift bug class flagged in the audit: tool says + `python3 scripts/foo.py` but `foo.py` was deleted. We don't run the + scripts here — just verify the path the MCP would invoke is present. + """ + + @pytest.mark.parametrize( + "script", + [ + "run_corpus_inventory.py", + "extract_corpus_components.py", + "rank_components.py", + "detect_runtime_kind.sh", + "protector_detect.py", + "dexprotector_unpack.py", + "extract_api_map.py", + "rank_backend_richness.py", + "normalize_findings.py", + ], + ) + def test_script_exists(self, server, script): + path = server.SCRIPTS / script + assert path.exists(), f"MCP-referenced script missing: {path}" + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/capabilities/android-apk-research/references/sources.md b/capabilities/android-apk-research/references/sources.md new file mode 100644 index 0000000..a91ce3a --- /dev/null +++ b/capabilities/android-apk-research/references/sources.md @@ -0,0 +1,107 @@ +# Sources + +Citation registry for the `android-apk-research` capability. Every external claim in the skills and per-skill references should be traceable from here. Inline links in the references point to canonical sources; this file is the consolidated index a future agent can read to ground every methodology decision. + +When extending the references with a new pattern, citation, or empirical observation, add the source here. + +--- + +## Industry standards & frameworks + +The capability grounds its methodology in OWASP's mobile security work and Android's first-party security guidance. + +- **OWASP MASVS** — Mobile Application Security Verification Standard, current revision v2.1.0 (2024-01). +- **OWASP MASTG** — Mobile Application Security Testing Guide, current revision v1.7.0 (2024-10). + - MASTG-TECH-0025 — Static analysis caveats (scanners are noisy, require review). + - MASTG-TECH-0019 — Dynamic analysis pre-flight. + - MASTG-TEST-0028 — Deep link testing. + - MASTG-TEST-0089 — Testing Resiliency Against Reverse Engineering. +- **OWASP MASWE** — Mobile Application Security Weakness Enumeration (beta). The capability uses MASWE-0058 (Insecure Deep Links), MASWE-0064 (Insecure Content Providers), MASWE-0066 (Insecure Intents), MASWE-0068 (JavaScript Bridges in WebViews) as the four PLATFORM anchors. Where no current MASWE cleanly maps a bug class, the normalizer omits the field rather than asserting an unrelated ID. +- **CWE** — MITRE Common Weakness Enumeration. +- **Android developer — risks/guidance** (Google): + - Deep link risks: + - Intent redirection risk: + - Unsafe URI loading into WebView: + - WebView native bridges: + - Untrustworthy ContentProvider-provided filename: +- **CodeQL Android query help** — Java/Kotlin Android queries: + - Intent redirection: + - Unsafe WebView fetch: + +--- + +## Public security research drawn on by this capability + +Named CVEs, advisories, and writeups whose shape the skills reproduce as detection patterns. When a pattern in `references/bug-classes.md`, `references/leaked-host-triage.md`, or `references/historical-patterns-2023-2026.md` cites one of these, the URL here is the canonical source. + +### Deep links / intent redirection / WebView + +- **Home Assistant Android arbitrary WebView URL** — GHSL-2023-142 / CVE-2023-41898. +- **Rakuten Ichiba custom URL scheme** — JVN56648919 / CVE-2024-41918. +- **Element Android intent redirection** — Shielder, *Never Take Intents from Strangers*; CVE-2024-26131, CVE-2024-26132. +- **Oversecured — Android deep-link account takeover patterns.** + +### Content providers / file/share targets + +- **Microsoft Dirty Stream** — *Dirty Stream Attack: discovering and mitigating a common vulnerability pattern in Android apps*, 2024-05-01. +- **ownCloud Android — provider SQLi / path validation** — GHSL-2022-059, GHSL-2022-060; CVE-2023-24804, CVE-2023-23948. + +### Commercial protector reverse engineering + +- **Romain Thomas — *A Glimpse Into DexProtector***, 2026-01-04. — the source for the in-tree DexProtector detector and `scripts/dexprotector_unpack.py` AArch64 layout. Companion artifact pairs: +- **APKiD ELF rules for Promon Shield** — +- **APKiD issue #72** — Promon use in German banking apps, refers to 34C3 talk. +- **34C3 — *Die fabelhafte Welt des Mobilebankings***, Vincent Haupert. +- **DIMVA 2018 — *Honey, I Shrunk Your App Security: The State of Android App Hardening***. · +- **KiFilterFiberContext — promon-reversal** (community RE notes). +- **RevealedSoulEven — promon-string-deobfuscator.** + +--- + +## Tool documentation + +Capabilities the skills assume the operator can run. Install/usage is each tool's own responsibility; the skills wrap orchestration. + +- **JADX** — DEX-to-Java decompiler. +- **Androguard** — APK parsing / manifest decoding. +- **APKiD** — packer / protector / obfuscator detection. +- **Semgrep** rule packs used as scanner baseline: `p/security-audit`, `p/mobsfscan`, `r/java`. +- **MobSF** / `mobsfscan` — Mobile Security Framework, scanner alternative. +- **APKHunt** — OWASP-aligned static scanner. +- **Joern** — Code Property Graph for call/data context. Docs: · +- **CodeQL** — path-precise queries. Android query packs cited above. +- **FlowDroid** — lifecycle-aware taint. · +- **hbctool** — Hermes bytecode disassembler for React Native bundles. +- **blutter** — Flutter / Dart AOT reverse engineering. +- **gplaydl** — anonymous Google Play APK downloader. +- **Aurora Store** — public token dispenser used by `gplaydl` for anonymous Play auth. +- **AndroZoo** — academic APK corpus. + +--- + +## Internal research provenance + +Some empirical observations in the references derive from **internal corpus research conducted on publicly available APK samples** downloaded from AndroZoo / Google Play and analyzed statically with the same scripts and tools this capability ships. The patterns documented from that work — Dagger compile-time environment pinning, bootstrap-initializer environment override, R8 + Kotlin Metadata dead-string retention, server-flippable feature-flag CDN swap, deep-link to account-state-change without WebView, MethodChannel routing in Flutter, Hermes bytecode bundle adoption in wallet RN apps — are presented as **pattern shapes** an agent can detect and reproduce, not as advisories about specific applications. + +Where references name a specific public app as an illustrative example of a pattern (Trust Wallet at 27k classes for JADX heap sizing, MetaMask using Hermes, the Hermes-adoption set across wallets, the leaked-host gate sub-patterns observed in finance/retail/media apps), the empirical claim is **static-analysis observation from corpus research**, not vendor-confirmed exploitability: + +- Static-analysis findings have not been validated against live backends or production accounts. +- Build-pinned and feature-flag-gated chains are reported as latent shapes, not as exploitable issues in the released artifact. +- Where private corpus identifiers (`corpus-N`, pass numbers, P-tier counts) appeared in earlier iterations of these references, they have been generalized — the empirics survive; the internal bookkeeping does not. + +A future agent picking up this capability with no prior context should: + +1. Use the public sources above to ground the methodology in canonical research. +2. Treat illustrative app names as pattern anchors for recognition, not as a vulnerability roster. +3. Reproduce the empirical observations by running this capability's scripts (`extract_corpus_components.py`, `rank_components.py`, the per-class `run_class_rg.sh` profiles, `extract_api_map.py`) against any AndroZoo or Play corpus of comparable size — the patterns are robust across re-runs because they're shape-based, not name-based. +4. For responsible-disclosure handling of any finding produced against a specific shipped app, follow the relevant vendor's security policy or coordinated-disclosure process before publishing. + +--- + +## Adding a new source + +When extending the references with a new claim: + +1. If the claim is grounded in a public CVE, paper, blog, or vendor doc, add the URL under the matching section above. +2. If the claim is an internal empirical observation, attribute as "observed in internal corpus research" inline, and (optionally) add a one-line note here describing the pattern shape and the corpus size. +3. Use the existing reference files for inline cross-links; this file is the canonical index, not a replacement for prose. diff --git a/capabilities/android-apk-research/scripts/androguard_inventory.py b/capabilities/android-apk-research/scripts/androguard_inventory.py new file mode 100755 index 0000000..0b54923 --- /dev/null +++ b/capabilities/android-apk-research/scripts/androguard_inventory.py @@ -0,0 +1,172 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.10" +# dependencies = ["androguard==4.1.3"] +# /// +"""Extract authoritative APK metadata via Androguard. + +Single-APK extractor invoked by the corpus runner. Uses Androguard's public APK +API instead of reimplementing manifest parsing. PEP 723 inline metadata lets +`uv run` resolve dependencies on the fly without polluting the active venv. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + + +def _androguard_version() -> str | None: + try: + import androguard # type: ignore[import-not-found] + + return getattr(androguard, "__version__", None) + except Exception: + return None + + +def _silence_androguard_logging() -> None: + try: + from loguru import logger as _logger # type: ignore[import-not-found] + + _logger.remove() + except Exception: + pass + import logging + + for noisy in ( + "androguard", + "androguard.core", + "androguard.core.axml", + "androguard.core.apk", + ): + logging.getLogger(noisy).setLevel(logging.ERROR) + + +def extract(apk_path: Path) -> dict[str, Any]: + _silence_androguard_logging() + try: + from androguard.core.apk import APK # type: ignore[import-not-found] + except ImportError: + from androguard.core.bytecodes.apk import APK # type: ignore[import-not-found] + + apk = APK(str(apk_path)) + package = apk.get_package() or None + permissions = sorted(apk.get_permissions() or []) + + component_kinds = [ + ("activity", apk.get_activities), + ("service", apk.get_services), + ("receiver", apk.get_receivers), + ("provider", apk.get_providers), + ] + + components: list[dict[str, Any]] = [] + schemes: set[str] = set() + hosts: set[str] = set() + browsable_components: list[str] = [] + + for kind, getter in component_kinds: + try: + names = getter() or [] + except Exception: + names = [] + for name in names: + full_name = ( + name + if "." in name + else f"{package}.{name.lstrip('.')}" + if package + else name + ) + entry: dict[str, Any] = { + "type": kind, + "name": full_name, + "raw_name": name, + "exported": None, + "permission": None, + "intent_filters": [], + "browsable": False, + "view": False, + } + try: + exported_attr = apk.get_element(kind, "exported", name) + if exported_attr is not None: + entry["exported"] = str(exported_attr).lower() == "true" + except Exception: + pass + try: + entry["permission"] = apk.get_element(kind, "permission", name) or None + except Exception: + pass + try: + raw_filters = apk.get_intent_filters(kind, name) or {} + except Exception: + raw_filters = {} + if isinstance(raw_filters, dict) and raw_filters: + filt_record: dict[str, Any] = { + "actions": sorted(raw_filters.get("action", []) or []), + "categories": sorted(raw_filters.get("category", []) or []), + "data": list(raw_filters.get("data", []) or []), + } + filt_record["browsable"] = ( + "android.intent.category.BROWSABLE" in filt_record["categories"] + ) + filt_record["view"] = ( + "android.intent.action.VIEW" in filt_record["actions"] + ) + if filt_record["browsable"]: + entry["browsable"] = True + if filt_record["view"]: + entry["view"] = True + for data in filt_record["data"]: + if not isinstance(data, dict): + continue + if data.get("scheme"): + schemes.add(data["scheme"]) + if data.get("host"): + hosts.add(data["host"]) + entry["intent_filters"] = [filt_record] + if entry["browsable"] and entry["name"]: + browsable_components.append(entry["name"]) + components.append(entry) + + return { + "tool": "androguard", + "tool_version": _androguard_version(), + "package": package, + "version_name": apk.get_androidversion_name(), + "version_code": apk.get_androidversion_code(), + "min_sdk": apk.get_min_sdk_version(), + "target_sdk": apk.get_target_sdk_version(), + "app_label": apk.get_app_name() or None, + "permissions": permissions, + "components": components, + "browsable_components": sorted(set(browsable_components)), + "schemes": sorted(schemes), + "hosts": sorted(hosts), + "is_valid_apk": bool(apk.is_valid_APK()), + } + + +def main() -> int: + ap = argparse.ArgumentParser( + description="Extract APK manifest facts via Androguard" + ) + ap.add_argument("apk", type=Path) + ap.add_argument("--out", type=Path, required=True) + args = ap.parse_args() + try: + record = extract(args.apk.expanduser().resolve()) + except Exception as exc: + record = {"tool": "androguard", "status": "error", "error": str(exc)} + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/capabilities/android-apk-research/scripts/androzoo_download.py b/capabilities/android-apk-research/scripts/androzoo_download.py new file mode 100755 index 0000000..fff7e8d --- /dev/null +++ b/capabilities/android-apk-research/scripts/androzoo_download.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Download APKs from AndroZoo for a JSONL selection manifest. + +The API key is read from --api-key, ANDROZOO_API_KEY, or a file specified by +--api-key-file. The key is never written to the output manifest. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures as cf +import hashlib +import json +import os +import sys +import threading +import time +import shutil +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Iterator + +API_URL = "https://androzoo.uni.lu/api/download" + + +def iter_jsonl(path: Path) -> Iterator[dict[str, Any]]: + """Stream the selection JSONL line by line. + + Previous implementation loaded the whole file via read_text().splitlines(); + fine for small selections but the same anti-pattern we fixed elsewhere. + """ + with path.open("r", errors="ignore") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, dict) and obj.get("sha256"): + yield obj + + +def api_key(args: argparse.Namespace) -> str: + if args.api_key: + return args.api_key.strip() + if args.api_key_file: + return Path(args.api_key_file).expanduser().read_text().strip() + key = os.environ.get("ANDROZOO_API_KEY", "").strip() + if key: + return key + raise SystemExit( + "AndroZoo API key required via --api-key, --api-key-file, or ANDROZOO_API_KEY" + ) + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def safe_name(row: dict[str, Any]) -> str: + pkg = str(row.get("package") or "unknown").replace("/", "_") + vc = str(row.get("version_code") or "novc").replace("/", "_") + sha = str(row["sha256"]) + return f"{pkg}_{vc}_{sha[:12]}.apk" + + +def download_one( + row: dict[str, Any], key: str, out_dir: Path, timeout: int, force: bool +) -> dict[str, Any]: + out_path = out_dir / safe_name(row) + expected = str(row["sha256"]).lower() + result = dict(row) + result["path"] = str(out_path) + if out_path.exists() and not force: + actual = sha256_file(out_path) + result["download_status"] = "exists" + result["downloaded_sha256"] = actual + result["sha256_ok"] = actual.lower() == expected + return result + params = urllib.parse.urlencode({"apikey": key, "sha256": row["sha256"]}) + url = f"{API_URL}?{params}" + try: + req = urllib.request.Request( + url, headers={"User-Agent": "dreadnode-android-apk-research/0.1"} + ) + tmp_path = out_path.with_suffix(out_path.suffix + ".part") + # Stream the response to disk so we never hold a 200 MB APK in RAM. + # The previous resp.read() landed the whole body in memory before write. + with ( + urllib.request.urlopen(req, timeout=timeout) as resp, + tmp_path.open("wb") as out_fh, + ): + shutil.copyfileobj(resp, out_fh, length=1024 * 1024) + tmp_path.replace(out_path) + actual = sha256_file(out_path) + result["download_status"] = "downloaded" + result["downloaded_sha256"] = actual + result["sha256_ok"] = actual.lower() == expected + if not result["sha256_ok"]: + result["error"] = "sha256 mismatch" + except Exception as exc: # noqa: BLE001 - written to manifest for retry decisions. + result["download_status"] = "error" + result["error"] = str(exc) + return result + + +def main() -> int: + ap = argparse.ArgumentParser( + description="Download APKs from AndroZoo using a JSONL selection manifest" + ) + ap.add_argument( + "selection_jsonl", + type=Path, + help="APK selection JSONL (one row per APK with at least 'sha256'; see android-corpus-prep skill)", + ) + ap.add_argument( + "--out-dir", type=Path, required=True, help="Directory to write APKs" + ) + ap.add_argument( + "--manifest-out", + type=Path, + help="Download manifest JSONL; default /download_manifest.jsonl", + ) + ap.add_argument( + "--api-key", + help="AndroZoo API key. Prefer ANDROZOO_API_KEY or --api-key-file to avoid shell history exposure", + ) + ap.add_argument("--api-key-file", help="File containing AndroZoo API key") + ap.add_argument( + "--limit", type=int, help="Download at most this many selected rows" + ) + ap.add_argument( + "--jobs", + type=int, + default=12, + help="Parallel downloads. AndroZoo throttles each connection to ~440 KB/s but allows " + "up to ~20 concurrent downloads per their API docs. 12 is a safe default. Pass 1 for serial.", + ) + ap.add_argument( + "--sleep", + type=float, + default=0.0, + help="Per-job dispatch delay in seconds. Use 0 for max throughput; raise if AndroZoo " + "is being polite to your IP and 429s start showing up in the manifest.", + ) + ap.add_argument( + "--timeout", + type=int, + default=600, + help="Per-download timeout seconds. Large APKs at ~440 KB/s can take 10+ minutes.", + ) + ap.add_argument( + "--force", action="store_true", help="Re-download even if file exists" + ) + args = ap.parse_args() + + key = api_key(args) + args.out_dir.mkdir(parents=True, exist_ok=True) + manifest_path = args.manifest_out or (args.out_dir / "download_manifest.jsonl") + manifest_path.parent.mkdir(parents=True, exist_ok=True) + tmp_manifest = manifest_path.with_suffix( + manifest_path.suffix + f".tmp.{os.getpid()}" + ) + + rows = list(iter_jsonl(args.selection_jsonl)) + if args.limit is not None: + rows = rows[: args.limit] + total = len(rows) + + counters = {"downloaded_or_exists": 0, "sha256_ok": 0, "errored": 0} + started = time.time() + manifest_lock = threading.Lock() + + def _work(row: dict[str, Any]) -> dict[str, Any]: + return download_one(row, key, args.out_dir, args.timeout, args.force) + + with ( + tmp_manifest.open("w") as manifest_fh, + cf.ThreadPoolExecutor(max_workers=max(1, args.jobs)) as pool, + ): + futures = {} + for row in rows: + futures[pool.submit(_work, row)] = row + if args.sleep: + time.sleep(args.sleep) + completed = 0 + for fut in cf.as_completed(futures): + row = futures[fut] + try: + res = fut.result() + except Exception as exc: # noqa: BLE001 + res = dict(row) + res["download_status"] = "error" + res["error"] = str(exc) + status = res.get("download_status") + if status in {"downloaded", "exists"}: + counters["downloaded_or_exists"] += 1 + else: + counters["errored"] += 1 + if res.get("sha256_ok"): + counters["sha256_ok"] += 1 + with manifest_lock: + manifest_fh.write(json.dumps(res, sort_keys=True)) + manifest_fh.write("\n") + manifest_fh.flush() + completed += 1 + elapsed = time.time() - started + rate = completed / elapsed if elapsed > 0 else 0 + eta = (total - completed) / rate if rate > 0 else float("inf") + print( + json.dumps( + { + "completed": completed, + "total": total, + "rate_per_sec": round(rate, 3), + "eta_sec": round(eta, 1) if eta != float("inf") else None, + "sha256": row["sha256"], + "status": status, + "ok": res.get("sha256_ok"), + "path": res.get("path"), + "error": res.get("error"), + }, + sort_keys=True, + ), + flush=True, + ) + tmp_manifest.replace(manifest_path) + summary = { + "requested": total, + **counters, + "duration_sec": round(time.time() - started, 1), + "manifest": str(manifest_path), + "jobs": args.jobs, + } + print(json.dumps(summary, sort_keys=True)) + return 0 if counters["sha256_ok"] == total else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/capabilities/android-apk-research/scripts/androzoo_gp_metadata.py b/capabilities/android-apk-research/scripts/androzoo_gp_metadata.py new file mode 100755 index 0000000..1c382a0 --- /dev/null +++ b/capabilities/android-apk-research/scripts/androzoo_gp_metadata.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Download and query AndroZoo Google Play metadata aggregate files.""" + +from __future__ import annotations + +import argparse +import gzip +import json +import os +import random +import re +import sys +import time +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Iterable + +AGG_URL = "https://androzoo.uni.lu/api/get_gp_metadata_file/aggregate" +FULL_URL = "https://androzoo.uni.lu/api/get_gp_metadata_file/full" + + +def api_key(args: argparse.Namespace) -> str: + if getattr(args, "api_key", None): + return args.api_key.strip() + if getattr(args, "api_key_file", None): + return Path(args.api_key_file).expanduser().read_text().strip() + key = os.environ.get("ANDROZOO_API_KEY", "").strip() + if key: + return key + raise SystemExit( + "AndroZoo API key required via --api-key, --api-key-file, or ANDROZOO_API_KEY" + ) + + +def download(args: argparse.Namespace) -> int: + key = api_key(args) + endpoint = AGG_URL if args.kind == "aggregate" else FULL_URL + url = endpoint + "?" + urllib.parse.urlencode({"apikey": key}) + out = args.out or Path(f"gp-metadata-{args.kind}.jsonl.gz") + out.parent.mkdir(parents=True, exist_ok=True) + req = urllib.request.Request( + url, headers={"User-Agent": "dreadnode-android-apk-research/0.1"} + ) + with ( + urllib.request.urlopen(req, timeout=args.timeout) as resp, + out.open("wb") as fh, + ): + total = int(resp.headers.get("content-length") or 0) + done = 0 + last = time.time() + while True: + chunk = resp.read(1024 * 1024) + if not chunk: + break + fh.write(chunk) + done += len(chunk) + if args.progress and time.time() - last > 5: + pct = f"{done / total * 100:.1f}%" if total else "?%" + print( + json.dumps( + { + "downloaded": done, + "total": total, + "pct": pct, + "out": str(out), + } + ), + file=sys.stderr, + ) + last = time.time() + print(json.dumps({"out": str(out), "bytes": out.stat().st_size}, sort_keys=True)) + return 0 + + +def open_jsonl(path: Path): + if path.suffix == ".gz": + return gzip.open(path, "rt", errors="replace") + return path.open("r", errors="replace") + + +def get_path(obj: Any, path: str) -> Any: + cur = obj + for part in path.split("."): + if isinstance(cur, dict): + cur = cur.get(part) + else: + return None + return cur + + +def walk_values( + obj: Any, key_regex: re.Pattern[str], values: list[Any], limit: int = 20 +) -> None: + if len(values) >= limit: + return + if isinstance(obj, dict): + for k, v in obj.items(): + if key_regex.search(k): + values.append(v) + if len(values) >= limit: + return + walk_values(v, key_regex, values, limit) + elif isinstance(obj, list): + for v in obj: + walk_values(v, key_regex, values, limit) + if len(values) >= limit: + return + + +def first_value_by_key(obj: Any, pattern: str) -> Any: + vals: list[Any] = [] + walk_values(obj, re.compile(pattern, re.I), vals, limit=1) + return vals[0] if vals else None + + +def as_float(v: Any, default: float = -1.0) -> float: + if isinstance(v, (int, float)): + return float(v) + if isinstance(v, str): + try: + return float(v.replace(",", "")) + except ValueError: + return default + return default + + +def as_int(v: Any, default: int = -1) -> int: + if isinstance(v, int): + return v + if isinstance(v, float): + return int(v) + if isinstance(v, str): + digits = re.sub(r"[^0-9]", "", v) + if digits: + try: + return int(digits) + except ValueError: + return default + return default + + +def extract_record(obj: dict[str, Any]) -> dict[str, Any]: + pkg = ( + obj.get("pkg_name") + or obj.get("package") + or obj.get("packageName") + or first_value_by_key(obj, r"^(docid|packageName|package)$") + ) + title = ( + obj.get("title") + or obj.get("name") + or first_value_by_key(obj, r"title|appTitle|name") + ) + max_downloads = ( + obj.get("max_numDownloads") + or obj.get("max_downloads") + or obj.get("details.appDetails.numDownloads") + or first_value_by_key(obj, r"numDownloads|downloads") + ) + max_rating = ( + obj.get("max_star_rating") + or obj.get("max_starRating") + or obj.get("max_rating") + or first_value_by_key(obj, r"star.?rating|rating") + ) + ratings_count = ( + obj.get("max_ratingsCount") + or obj.get("max_rating_count") + or first_value_by_key(obj, r"ratingsCount|ratingCount") + ) + comment_count = obj.get("max_commentCount") or first_value_by_key( + obj, r"commentCount|reviewCount" + ) + category = ( + obj.get("category") + or obj.get("appCategory") + or first_value_by_key(obj, r"category|genre") + ) + version_code = ( + obj.get("max_versionCode") + or obj.get("versionCode") + or first_value_by_key(obj, r"versionCode") + ) + return { + "package": pkg, + "title": title, + "category": category, + "max_downloads": as_int(max_downloads), + "max_star_rating": as_float(max_rating), + "max_ratings_count": as_int(ratings_count), + "max_comment_count": as_int(comment_count), + "version_code_hint": str(version_code) if version_code is not None else None, + "az_metadata_date": obj.get("az_metadata_date") + or first_value_by_key(obj, r"az_metadata_date"), + "raw_keys": sorted(obj.keys())[:80], + } + + +def matches(rec: dict[str, Any], args: argparse.Namespace) -> bool: + if not rec.get("package"): + return False + if ( + args.min_downloads is not None + and rec.get("max_downloads", -1) < args.min_downloads + ): + return False + if ( + args.min_rating is not None + and rec.get("max_star_rating", -1.0) < args.min_rating + ): + return False + if ( + args.min_ratings_count is not None + and rec.get("max_ratings_count", -1) < args.min_ratings_count + ): + return False + hay = " ".join( + str(rec.get(k) or "") for k in ["package", "title", "category"] + ).lower() + if args.keyword and not any(k.lower() in hay for k in args.keyword): + return False + if args.package_contains and not any( + k.lower() in str(rec.get("package") or "").lower() + for k in args.package_contains + ): + return False + return True + + +def select(args: argparse.Namespace) -> int: + rng = random.Random(args.seed) + selected: list[dict[str, Any]] = [] + seen = matched = 0 + with open_jsonl(args.metadata_path) as fh: + for line in fh: + if not line.strip(): + continue + seen += 1 + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + rec = extract_record(obj) + if not matches(rec, args): + continue + matched += 1 + if args.sample: + if len(selected) < args.limit: + selected.append(rec) + else: + j = rng.randrange(matched) + if j < args.limit: + selected[j] = rec + else: + selected.append(rec) + if len(selected) >= args.limit: + break + selected.sort( + key=lambda r: (r.get("max_downloads", -1), r.get("max_ratings_count", -1)), + reverse=True, + ) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in selected)) + print( + json.dumps( + { + "seen": seen, + "matched": matched, + "written": len(selected), + "out": str(args.out), + }, + sort_keys=True, + ) + ) + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser( + description="Download/query AndroZoo Google Play metadata" + ) + sub = ap.add_subparsers(dest="cmd", required=True) + + dl = sub.add_parser( + "download", help="Download gp-metadata aggregate/full JSONL gzip" + ) + dl.add_argument("--kind", choices=["aggregate", "full"], default="aggregate") + dl.add_argument("--out", type=Path) + dl.add_argument("--api-key") + dl.add_argument("--api-key-file") + dl.add_argument("--timeout", type=int, default=120) + dl.add_argument("--progress", action="store_true") + dl.set_defaults(func=download) + + sel = sub.add_parser( + "select", help="Select packages from gp-metadata aggregate JSONL(.gz)" + ) + sel.add_argument("metadata_path", type=Path) + sel.add_argument("--out", type=Path, required=True) + sel.add_argument("--limit", type=int, default=100) + sel.add_argument("--sample", action="store_true") + sel.add_argument("--seed", type=int, default=1337) + sel.add_argument("--min-downloads", type=int, default=1_000_000) + sel.add_argument("--min-rating", type=float, default=3.5) + sel.add_argument("--min-ratings-count", type=int, default=1_000) + sel.add_argument( + "--keyword", + action="append", + help="Keyword in package/title/category; repeatable", + ) + sel.add_argument( + "--package-contains", + action="append", + help="Substring in package name; repeatable", + ) + sel.set_defaults(func=select) + + args = ap.parse_args() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/capabilities/android-apk-research/scripts/androzoo_to_parquet.py b/capabilities/android-apk-research/scripts/androzoo_to_parquet.py new file mode 100755 index 0000000..dcc8e36 --- /dev/null +++ b/capabilities/android-apk-research/scripts/androzoo_to_parquet.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +"""Convert AndroZoo metadata sources to ZSTD Parquet for cheap repeated joins. + +Two inputs: + +- ``latest_with-added-date.csv.gz`` — multi-GB row-per-APK CSV. Converted via the + DuckDB CLI piping a gunzip stream through a named pipe so we never land the + decompressed 7 GB on disk. The whole CSV becomes ~3 GB ZSTD Parquet. +- ``gp-metadata-aggregate.jsonl.gz`` — multi-GB row-per-package JSONL with one + nested ``related_apks_in_AZ_info`` dict and a few list-typed fields. We + stream-decompress with Python, normalize the lists into ``"; "`` joined strings, + serialize the nested dict to a JSON text column, and write fixed-schema Parquet + with pyarrow. Memory stays at one batch (default 25k rows). + +Why not let DuckDB sniff the JSONL schema? On a 1.3 GB file it OOM'd at the +schema-inference stage at 6 GB. Explicit schema + Python streaming keeps the +working set bounded and the columns typed. + +Usage: + + androzoo_to_parquet.py csv corpus/androzoo/meta/latest_with-added-date.csv.gz \\ + corpus/androzoo/meta/parquet/androzoo_latest.parquet + androzoo_to_parquet.py json corpus/androzoo/meta/gp-metadata-aggregate.jsonl.gz \\ + corpus/androzoo/meta/parquet/androzoo_gp_metadata.parquet + +The output files are the canonical input to DuckDB / Polars / pandas joins. Both +keep the original semantics (no rows dropped, no columns renamed) so they are a +drop-in replacement for the gz sources. +""" + +from __future__ import annotations + +import argparse +import gzip +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + + +def _which_duckdb() -> str: + path = shutil.which("duckdb") + if not path: + raise SystemExit( + "duckdb CLI not on PATH; brew install duckdb or install the platform binary" + ) + return path + + +def convert_csv( + src: Path, dst: Path, memory_limit: str = "6GB", threads: int = 4 +) -> dict[str, Any]: + """Convert the AndroZoo CSV(.gz) to Parquet via DuckDB through a named pipe.""" + duckdb = _which_duckdb() + dst.parent.mkdir(parents=True, exist_ok=True) + tmp_dir = Path(tempfile.mkdtemp(prefix="azcsv_")) + pipe = tmp_dir / "csv.fifo" + os.mkfifo(pipe) + started = time.time() + if src.suffix == ".gz": + decomp = subprocess.Popen(["gzip", "-dc", str(src)], stdout=open(pipe, "wb")) + else: + decomp = subprocess.Popen(["cat", str(src)], stdout=open(pipe, "wb")) + try: + sql = ( + f"SET memory_limit='{memory_limit}';\n" + f"SET threads={threads};\n" + "COPY (" + f" SELECT * FROM read_csv('{pipe}', header=true, all_varchar=true)" + ") TO '" + + str(dst) + + "' (FORMAT PARQUET, COMPRESSION ZSTD, ROW_GROUP_SIZE 100000);" + ) + proc = subprocess.run( + [duckdb, "-c", sql], capture_output=True, text=True, check=False + ) + if proc.returncode != 0: + return { + "status": "error", + "stderr": proc.stderr[-4000:], + "stdout": proc.stdout[-4000:], + } + finally: + decomp.wait() + try: + pipe.unlink() + except FileNotFoundError: + pass + tmp_dir.rmdir() + return { + "status": "ok", + "duration_sec": round(time.time() - started, 2), + "src": str(src), + "dst": str(dst), + "dst_size_bytes": dst.stat().st_size, + } + + +# --- JSONL → Parquet ----------------------------------------------------------------- + +import pyarrow as pa +import pyarrow.parquet as pq + +GP_SCHEMA = pa.schema( + [ + ("pkg_name", pa.string()), + ("nb_meta", pa.int64()), + ("nb_versionCode", pa.int64()), + ("min_versionCode", pa.int64()), + ("max_versionCode", pa.int64()), + ("first_seen", pa.string()), + ("last_seen", pa.string()), + ("min_star_rating", pa.float64()), + ("max_star_rating", pa.float64()), + ("min_ratingsCount", pa.int64()), + ("max_ratingsCount", pa.int64()), + ("min_commentCount", pa.int64()), + ("max_commentCount", pa.int64()), + ("min_upload_date", pa.string()), + ("max_upload_date", pa.string()), + ("min_nb_downloads", pa.int64()), + ("max_nb_downloads", pa.int64()), + ("min_installationSize", pa.int64()), + ("max_installationSize", pa.int64()), + ("developerName", pa.string()), + ("developerEmail", pa.string()), + ("developerWebsite", pa.string()), + ("developerAddress", pa.string()), + ("related_apks_in_AZ_info_json", pa.string()), + ] +) + + +def _coerce_float(v: Any) -> float | None: + if v is None: + return None + try: + return float(v) + except (TypeError, ValueError): + return None + + +def _join_list(v: Any) -> str | None: + if v is None: + return None + if isinstance(v, list): + return "; ".join(str(x) for x in v if x is not None) or None + return str(v) + + +def _coerce_gp_row(o: dict[str, Any]) -> dict[str, Any]: + nested = o.get("related_apks_in_AZ_info") + return { + "pkg_name": o.get("pkg_name"), + "nb_meta": o.get("nb_meta"), + "nb_versionCode": o.get("nb_versionCode"), + "min_versionCode": o.get("min_versionCode"), + "max_versionCode": o.get("max_versionCode"), + "first_seen": o.get("first_seen"), + "last_seen": o.get("last_seen"), + "min_star_rating": _coerce_float(o.get("min_star_rating")), + "max_star_rating": _coerce_float(o.get("max_star_rating")), + "min_ratingsCount": o.get("min_ratingsCount"), + "max_ratingsCount": o.get("max_ratingsCount"), + "min_commentCount": o.get("min_commentCount"), + "max_commentCount": o.get("max_commentCount"), + "min_upload_date": o.get("min_upload_date"), + "max_upload_date": o.get("max_upload_date"), + "min_nb_downloads": o.get("min_nb_downloads"), + "max_nb_downloads": o.get("max_nb_downloads"), + "min_installationSize": o.get("min_installationSize"), + "max_installationSize": o.get("max_installationSize"), + "developerName": _join_list(o.get("developerName")), + "developerEmail": _join_list(o.get("developerEmail")), + "developerWebsite": _join_list(o.get("developerWebsite")), + "developerAddress": _join_list(o.get("developerAddress")), + "related_apks_in_AZ_info_json": json.dumps(nested, separators=(",", ":")) + if nested + else None, + } + + +def convert_gp_jsonl( + src: Path, dst: Path, batch_size: int = 25_000, progress: bool = True +) -> dict[str, Any]: + dst.parent.mkdir(parents=True, exist_ok=True) + started = time.time() + total = 0 + batch: list[dict[str, Any]] = [] + open_fn = gzip.open if src.suffix == ".gz" else open + writer = pq.ParquetWriter(str(dst), GP_SCHEMA, compression="zstd") + try: + with open_fn(src, "rt", encoding="utf-8", errors="replace") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + batch.append(_coerce_gp_row(obj)) + if len(batch) >= batch_size: + writer.write_table(pa.Table.from_pylist(batch, schema=GP_SCHEMA)) + total += len(batch) + batch.clear() + if progress and total % (batch_size * 10) == 0: + elapsed = time.time() - started + print( + f" {total:>12,} rows {elapsed:6.1f}s ({total/elapsed:,.0f} rps)", + file=sys.stderr, + ) + if batch: + writer.write_table(pa.Table.from_pylist(batch, schema=GP_SCHEMA)) + total += len(batch) + finally: + writer.close() + return { + "status": "ok", + "duration_sec": round(time.time() - started, 2), + "rows": total, + "src": str(src), + "dst": str(dst), + "dst_size_bytes": dst.stat().st_size, + } + + +# --- CLI ----------------------------------------------------------------------------- + + +def main() -> int: + ap = argparse.ArgumentParser( + description="Convert AndroZoo metadata to ZSTD Parquet" + ) + sub = ap.add_subparsers(dest="kind", required=True) + + p_csv = sub.add_parser("csv", help="Convert latest.csv(.gz) -> Parquet via DuckDB") + p_csv.add_argument("src", type=Path) + p_csv.add_argument("dst", type=Path) + p_csv.add_argument("--memory-limit", default="6GB") + p_csv.add_argument("--threads", type=int, default=4) + + p_json = sub.add_parser( + "json", help="Convert gp-metadata-aggregate.jsonl(.gz) -> Parquet (streamed)" + ) + p_json.add_argument("src", type=Path) + p_json.add_argument("dst", type=Path) + p_json.add_argument("--batch-size", type=int, default=25_000) + p_json.add_argument("--no-progress", action="store_true") + + args = ap.parse_args() + if args.kind == "csv": + result = convert_csv( + args.src.expanduser().resolve(), + args.dst.expanduser().resolve(), + memory_limit=args.memory_limit, + threads=args.threads, + ) + else: + result = convert_gp_jsonl( + args.src.expanduser().resolve(), + args.dst.expanduser().resolve(), + batch_size=args.batch_size, + progress=not args.no_progress, + ) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 if result.get("status") == "ok" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/capabilities/android-apk-research/scripts/detect_runtime_kind.sh b/capabilities/android-apk-research/scripts/detect_runtime_kind.sh new file mode 100755 index 0000000..a1ef7e2 --- /dev/null +++ b/capabilities/android-apk-research/scripts/detect_runtime_kind.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# detect_runtime_kind.sh — classify an APK's app runtime in one second. +# +# Usage: +# detect_runtime_kind.sh path/to.apk +# detect_runtime_kind.sh --jsonl path/to.apk # emit one JSONL record +# +# Output one of: +# native (Java/Kotlin only) +# react_native_js (RN with plain JS bundle) +# react_native_hermes (RN with Hermes bytecode bundle) +# flutter_aot (Flutter, libapp.so present) +# capacitor (Capacitor: assets/public/ webview content) +# cordova (Cordova: assets/www/ + cordova.js) +# xamarin (libmonodroid.so / Mono assemblies) +# maui (.NET MAUI: libmono-component-* + Microsoft.Maui dlls) +# unity (libunity.so present) +# +# Drives: +# - JADX heap setting (RN/Flutter still need native pass; xamarin/unity skip JADX) +# - Step 7.5 (RN JS-bundle trace) and Step 7.6 (Flutter Dart-AOT trace) routing +# - Pre-bundle hypothesis grading (force needs_route_map_validation for RN/Flutter) +# +# Bug class signal: presence of a non-native runtime predicts that Java-side +# routing is a thin relay and the real trust decision is in the bundle. +# +# Detection corner cases (corpus-3, 2026-05-19): +# - Microsoft Teams ships per-feature Hermes bundles under +# `assets/app_packages//hermes.android.bundle` rather than the +# canonical `assets/index.android.bundle`. The old probe missed these and +# classified Teams as native; the corrected matcher widens the bundle glob +# to any `*.bundle` / `*.jsbundle` under `assets/**`. +# - Mint and similar RN apps ship a file-based unbundle: each module is its +# own JS file under `assets/js-modules/*.js` plus an `UNBUNDLE` header +# file. There is no single bundle file to probe magic on, so we infer +# `react_native_js` from the directory + `libhermes-executor*.so` +# presence (Hermes-engine link without a precompiled bundle means the +# engine evals the JS modules at runtime). When `libhermes-executor*.so` +# is present but no precompiled `.hbc`/`.bundle` is, output is +# `react_native_js` (engine runs raw JS), not `react_native_hermes`. + +set -u +# pipefail intentionally OFF: `grep -q` and `head -c` close pipes early, producing +# SIGPIPE (rc=141) in upstream processes. We rely on the last-stage rc only. + +JSONL=0 +if [ "${1-}" = "--jsonl" ]; then JSONL=1; shift; fi +APK=${1?usage: $0 [--jsonl] path/to.apk} +[ -r "$APK" ] || { echo "cannot read $APK" >&2; exit 1; } + +# unzip -l is enough; we never need to extract. +LIST=$(unzip -l "$APK" 2>/dev/null) || { echo "not a zip: $APK" >&2; exit 1; } + +KIND=native +HERMES=0 + +# React Native bundle detection — three shapes: +# 1) canonical: assets/index.android.bundle | assets/*.jsbundle +# 2) per-feature: assets//.bundle | hermes.android.bundle +# 3) file-based unbundle: assets/js-modules/*.js + assets/UNBUNDLE (Mint pattern) +BUNDLE_ENTRY=$(printf '%s\n' "$LIST" \ + | awk '{print $NF}' \ + | grep -E '^assets/(.*/)*(index[^/]*\.bundle|[^/]+\.jsbundle|hermes\.android\.bundle)$' \ + | head -1 || true) + +# File-based unbundle (no single bundle entry but split JS modules present). +UNBUNDLE_DIR=$(printf '%s\n' "$LIST" \ + | awk '{print $NF}' \ + | grep -E '^assets/(js-modules|js_modules)/[^/]+\.js$' \ + | head -1 || true) + +if [ -n "$BUNDLE_ENTRY" ]; then + # Probe magic to distinguish Hermes from plain JS without extracting fully. + # Hermes header magic is 0xc61fbc03 (facebook/hermes BCVersion.h). + MAGIC=$( { unzip -p "$APK" "$BUNDLE_ENTRY" 2>/dev/null || true; } | head -c 4 | xxd -p) + if [ "$MAGIC" = "c61fbc03" ]; then + KIND=react_native_hermes; HERMES=1 + else + KIND=react_native_js + fi +elif [ -n "$UNBUNDLE_DIR" ]; then + # File-based unbundle: raw JS modules. The presence of a Hermes engine .so + # without a precompiled bundle still means JS is evaluated at runtime — keep + # as react_native_js (not _hermes) so the operator knows there is no .hbc + # to disasm; they need the raw JS path (Step 7.5b, not 7.5c). + KIND=react_native_js +elif printf '%s\n' "$LIST" | awk '{print $NF}' | grep -qE '^assets/flutter_assets/.'; then + KIND=flutter_aot +elif printf '%s\n' "$LIST" | awk '{print $NF}' | grep -qE '^lib/[^/]+/libapp\.so$'; then + KIND=flutter_aot +elif printf '%s\n' "$LIST" | grep -qE 'lib/[^/]+/libunity\.so'; then + KIND=unity +elif printf '%s\n' "$LIST" | grep -qE 'lib/[^/]+/libmonodroid\.so'; then + # Distinguish Xamarin classic vs .NET MAUI + if printf '%s\n' "$LIST" | grep -qiE 'assemblies/Microsoft\.Maui'; then + KIND=maui + else + KIND=xamarin + fi +elif printf '%s\n' "$LIST" | grep -q 'assets/public/index.html'; then + KIND=capacitor +elif printf '%s\n' "$LIST" | grep -q 'assets/www/cordova.js'; then + KIND=cordova +fi + +if [ "$JSONL" = 1 ]; then + printf '{"apk":"%s","runtime_kind":"%s","hermes":%s}\n' \ + "$APK" "$KIND" "$( [ $HERMES = 1 ] && echo true || echo false )" +else + echo "$KIND" +fi diff --git a/capabilities/android-apk-research/scripts/dexprotector_unpack.py b/capabilities/android-apk-research/scripts/dexprotector_unpack.py new file mode 100644 index 0000000..5a6c4cd --- /dev/null +++ b/capabilities/android-apk-research/scripts/dexprotector_unpack.py @@ -0,0 +1,324 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.10" +# dependencies = ["unicorn>=2.1.0", "capstone>=5.0"] +# /// +"""DexProtector libdexprotector.so -> libdp.so static unpacker. + +Reproduces the public research in https://www.romainthomas.fr/post/26-01-dexprotector/ +without requiring a running Android device. + +Tier-2 capability: given a DexProtector-protected APK (or a bare +`libdexprotector.so`), recover the plain libdp.so. Subsequent steps +(master-key derivation, asset decryption, classes.dex.dat unpack) need +further RE on the recovered libdp.so and live in companion scripts. + +Approach (validated on com.playnet.androidtv.ads 5.0.1 / arm64-v8a): + +1. Map libdexprotector.so into Unicorn. Stub the linker (FUN_001021a0 + returns a fake r_debug whose r_brk points at a synthesized + rtld_db_dlactivity = single `ret`). Stub Linux syscalls so mmap + returns our preallocated image buffer and mprotect/memcpy/munmap + succeed cleanly. +2. Call FUN_0010114c(payload+4, payload_size-4, &segtable). It + internally runs: + - the obfuscated 16 KB-bytecode VM at FUN_00100790 -> 32-byte key + (key bytes 0/4/8/12 are XORed with the first 4 bytes of + rtld_db_dlactivity, which is the frida-server-persistence check + described in the post) + - a counter-mode-style Feistel cipher (FUN_00100c2c) to decrypt + the 0x24-byte super-header and each segment's dynamic-tag table + - LZ4 block decompression (FUN_00101b58) of each segment +3. Read the populated segment table to learn {vaddr, size, flags} per + PT_LOAD, slice the image buffer, and emit a valid AArch64 ET_DYN ELF. + +ARM64 is the only target wired today; armeabi-v7a / x86 / x86_64 +variants of libdexprotector.so use the same layout but different +function addresses. Adding those is a small port (re-derive +FUN_001021a0 / FUN_00100790 / FUN_0010114c offsets via the DPLF +watermark and INIT_0 disassembly). +""" + +from __future__ import annotations + +import argparse +import io +import struct +import sys +import zipfile +from pathlib import Path + +from unicorn import ( + UC_ARCH_ARM64, + UC_HOOK_CODE, + UC_HOOK_INTR, + UC_MODE_ARM, + UC_PROT_ALL, + UC_PROT_READ, + UC_PROT_WRITE, + Uc, +) +from unicorn.arm64_const import ( + UC_ARM64_REG_LR, + UC_ARM64_REG_PC, + UC_ARM64_REG_SP, + UC_ARM64_REG_W8, + UC_ARM64_REG_X0, + UC_ARM64_REG_X1, + UC_ARM64_REG_X2, +) + +# arm64 build offsets — verified against the LiveNet (com.playnet.androidtv.ads +# 5.0.1) and confirmed identical in Revolut 10.109 by DPLF watermark layout. +ARM64_OFFSETS = { + "INIT_0": 0x29D4, + "JNI_OnLoad": 0x2AB4, + "FUN_resolve_dlactivity": 0x21A0, # returns DAT_0010b698 (r_debug*) + "FUN_KDF": 0x00790, # writes 32-byte key into x0 + "FUN_cipher": 0x00C2C, # Feistel keystream cipher + "FUN_payload_header_decrypt": 0x114C, # parses & expands the payload + "FUN_lz4_decompress": 0x01B58, + "PT_LOAD_payload_vaddr": 0xFAC0, # where the encrypted blob lives + "DPLF_magic_offset_in_payload": 0x0, +} + +PAGE = 0x1000 + + +def align_up(x: int, a: int = PAGE) -> int: + return (x + a - 1) & ~(a - 1) + + +def open_libdexprotector(path: Path) -> bytes: + """Accept either an APK or a bare libdexprotector.so.""" + data = path.read_bytes() + if data[:4] == b"\x7fELF": + return data + if data[:2] in (b"PK",): + with zipfile.ZipFile(io.BytesIO(data)) as z: + for member in ( + "lib/arm64-v8a/libdexprotector.so", + "lib/arm64-v8a/libdexprotector_h.so", + ): + if member in z.namelist(): + return z.read(member) + raise SystemExit( + "no arm64-v8a libdexprotector.so found inside APK; this unpacker " + "currently only handles the arm64 variant" + ) + raise SystemExit("input is neither ELF nor APK") + + +def find_dplf_payload(lib: bytes) -> tuple[int, int]: + """Return (file_offset, length) of the encrypted DPLF payload. + + The payload either starts with DPLF magic somewhere in the file, or it + lives in the last PT_LOAD whose contents happen to start with DPLF. + """ + idx = lib.find(b"DPLF") + if idx < 0: + raise SystemExit("no DPLF magic found in libdexprotector.so") + + e_phoff = struct.unpack_from(" list[tuple[int, int]]: + e_phoff = struct.unpack_from("= b) for a, b in mapped): + uc.mem_map(va, end - va, UC_PROT_ALL) + mapped.append((va, end)) + uc.mem_write(load_base + p_vaddr, lib[p_offset : p_offset + p_filesz]) + return mapped + + +def unpack_libdp(lib: bytes, *, verbose: bool = False) -> bytes: + if lib[:4] != b"\x7fELF": + raise SystemExit("not an ELF") + + payload_off, _payload_len = find_dplf_payload(lib) + if verbose: + print(f"[+] DPLF payload at file offset {payload_off:#x}") + + LOAD_BASE = 0x40000000 + uc = Uc(UC_ARCH_ARM64, UC_MODE_ARM) + map_lib(uc, lib, LOAD_BASE) + + # Fake r_debug. FUN_resolve_dlactivity returns a pointer P such that + # *(uintptr_t*)(P + 0x10) is the address of rtld_db_dlactivity. The first + # 4 bytes there go straight into the round-key. We synthesize the + # *unmodified* prologue (single `ret`) — frida-server's persistence + # trampoline would corrupt these bytes, so emitting a clean `ret` gives us + # the same key the real loader sees on a clean device. + FAKE_RDEBUG = 0x30000000 + uc.mem_map(FAKE_RDEBUG, 0x1000, UC_PROT_READ | UC_PROT_WRITE) + RTLD = FAKE_RDEBUG + 0x100 + uc.mem_write(RTLD, struct.pack("= len(fields): + break + addr, size, flags = fields[base_i : base_i + 3] + if addr == 0 or size == 0: + break + segs.append((addr - IMG_BASE, size, flags)) + if not segs: + raise SystemExit("no segments parsed; segment table may be invalid") + + return build_elf(img, segs) + + +def build_elf(img: bytes, segs: list[tuple[int, int, int]]) -> bytes: + """Wrap the unpacked segments in a synthetic AArch64 ET_DYN ELF.""" + ehdr_size = 0x40 + phdr_size = 0x38 + + elf = bytearray() + elf += b"\x7fELF" + b"\x02\x01\x01\x00" + b"\x00" * 8 + elf += struct.pack(" None: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument( + "input", + type=Path, + help="APK, XAPK, or bare libdexprotector.so (arm64-v8a only today)", + ) + ap.add_argument( + "-o", + "--output", + type=Path, + default=Path("libdp.so"), + help="output path (default: libdp.so)", + ) + ap.add_argument("-v", "--verbose", action="store_true") + args = ap.parse_args() + + lib = open_libdexprotector(args.input) + libdp = unpack_libdp(lib, verbose=args.verbose) + args.output.write_bytes(libdp) + print(f"wrote {args.output} ({len(libdp)} bytes)") + + +if __name__ == "__main__": + main() diff --git a/capabilities/android-apk-research/scripts/extract_api_map.py b/capabilities/android-apk-research/scripts/extract_api_map.py new file mode 100644 index 0000000..356cad5 --- /dev/null +++ b/capabilities/android-apk-research/scripts/extract_api_map.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +"""Extract a lightweight API/backend map from decompiled APK sources or bundle text. + +This is intentionally regex-based and dependency-free. It does not prove +vulnerabilities; it produces a target map for APK -> backend/API hypotheses. + +Outputs: + - JSONL rows with category/kind/value/file/line/context + - Optional summary JSON with backend-richness scores and top terms + +Usage: + python3 scripts/extract_api_map.py \ + --src findings/decompiled/com.example/sources \ + --out findings/com.example/api_map.jsonl \ + --summary findings/com.example/backend_richness.json +""" + +from __future__ import annotations + +import argparse +import json +import re +from collections import Counter, defaultdict +from pathlib import Path +from typing import Iterable, Iterator, Any + +TEXT_EXTS = { + ".java", + ".kt", + ".kts", + ".xml", + ".json", + ".properties", + ".gradle", + ".js", + ".jsx", + ".ts", + ".tsx", + ".graphql", + ".gql", + ".proto", + ".txt", + ".smali", + ".dart", +} +SKIP_DIRS = { + ".git", + "build", + "dist", + "node_modules", + "__pycache__", + ".gradle", + "androidx", + "kotlin", + "kotlinx", +} +MAX_FILE_BYTES = 2_000_000 + +PATTERNS: list[tuple[str, str, re.Pattern[str]]] = [ + ("endpoint", "url", re.compile(r"https?://[A-Za-z0-9._~:/?#\[\]@!$&'()*+,;=%-]+")), + ( + "endpoint", + "api_path", + re.compile( + r"(? Iterator[Path]: + if root.is_file(): + yield root + return + for p in root.rglob("*"): + if not p.is_file(): + continue + if any(part in SKIP_DIRS for part in p.parts): + continue + if p.suffix and p.suffix not in TEXT_EXTS: + continue + try: + if p.stat().st_size > MAX_FILE_BYTES: + continue + except OSError: + continue + yield p + + +def read_text(path: Path) -> str | None: + try: + return path.read_text(errors="ignore") + except (OSError, UnicodeDecodeError): + return None + + +def clean_value(value: str) -> str: + return value.strip().strip("\"'`,);")[:500] + + +def extract(root: Path) -> Iterator[dict[str, Any]]: + for path in iter_files(root): + text = read_text(path) + if text is None: + continue + rel = str(path.relative_to(root)) if root.is_dir() else str(path) + for lineno, line in enumerate(text.splitlines(), 1): + if not line.strip(): + continue + for category, kind, pattern in PATTERNS: + for match in pattern.finditer(line): + value = clean_value(match.group(0)) + if not value or value in {"url", "URL"}: + continue + yield { + "category": category, + "kind": kind, + "value": value, + "file": rel, + "line": lineno, + "context": line.strip()[:1000], + } + + +def summarize(rows: list[dict[str, Any]]) -> dict[str, Any]: + by_category = Counter(r["category"] for r in rows) + unique_values_by_category: dict[str, set[str]] = defaultdict(set) + top_terms_by_category: dict[str, Counter[str]] = defaultdict(Counter) + files_by_category: dict[str, set[str]] = defaultdict(set) + for r in rows: + cat = r["category"] + val = r["value"] + unique_values_by_category[cat].add(val) + top_terms_by_category[cat][val] += 1 + files_by_category[cat].add(r["file"]) + + scores = { + cat: len(vals) * RISK_WEIGHTS.get(cat, 1) + for cat, vals in unique_values_by_category.items() + } + total = sum(scores.values()) + if {"object_id", "workflow"}.issubset(unique_values_by_category): + total += 20 + if {"bridge", "auth"}.issubset(unique_values_by_category): + total += 15 + if {"signing", "endpoint"}.issubset(unique_values_by_category): + total += 15 + if {"feature_flag", "workflow"}.issubset(unique_values_by_category): + total += 10 + if {"url_fetch", "endpoint"}.issubset(unique_values_by_category): + total += 10 + + richness = "low" + if total >= 180: + richness = "very_high" + elif total >= 100: + richness = "high" + elif total >= 45: + richness = "medium" + + return { + "backend_richness": richness, + "total_score": total, + "row_count": len(rows), + "category_counts": dict(sorted(by_category.items())), + "unique_value_counts": { + cat: len(vals) for cat, vals in sorted(unique_values_by_category.items()) + }, + "category_file_counts": { + cat: len(vals) for cat, vals in sorted(files_by_category.items()) + }, + "scores": dict(sorted(scores.items())), + "top_terms": { + cat: terms.most_common(25) + for cat, terms in sorted(top_terms_by_category.items()) + }, + "synergy_flags": { + "object_workflow_pair": {"object_id", "workflow"}.issubset( + unique_values_by_category + ), + "bridge_auth_pair": {"bridge", "auth"}.issubset(unique_values_by_category), + "signing_endpoint_pair": {"signing", "endpoint"}.issubset( + unique_values_by_category + ), + "feature_workflow_pair": {"feature_flag", "workflow"}.issubset( + unique_values_by_category + ), + "url_fetch_endpoint_pair": {"url_fetch", "endpoint"}.issubset( + unique_values_by_category + ), + }, + } + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument( + "--src", + required=True, + type=Path, + help="source tree, JS analysis dir, Dart analysis dir, or text file", + ) + ap.add_argument("--out", required=True, type=Path, help="output JSONL path") + ap.add_argument( + "--summary", type=Path, default=None, help="optional summary JSON path" + ) + ap.add_argument( + "--dedupe", + action="store_true", + help="dedupe exact category/kind/value/file/line rows", + ) + args = ap.parse_args() + + rows = list(extract(args.src)) + if args.dedupe: + seen: set[tuple[str, str, str, str, int]] = set() + deduped = [] + for r in rows: + key = (r["category"], r["kind"], r["value"], r["file"], r["line"]) + if key in seen: + continue + seen.add(key) + deduped.append(r) + rows = deduped + + args.out.parent.mkdir(parents=True, exist_ok=True) + with args.out.open("w") as f: + for r in rows: + f.write(json.dumps(r, sort_keys=True) + "\n") + + summary = summarize(rows) + if args.summary: + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") + + print(f"wrote {len(rows)} rows to {args.out}") + print( + f"backend_richness={summary['backend_richness']} total_score={summary['total_score']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/capabilities/android-apk-research/scripts/extract_attack_surface.py b/capabilities/android-apk-research/scripts/extract_attack_surface.py new file mode 100755 index 0000000..a7c55ad --- /dev/null +++ b/capabilities/android-apk-research/scripts/extract_attack_surface.py @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 +"""Extract a lightweight attack-surface inventory from Android APKs. + +The script is intentionally dependency-free. It uses Python zip/string parsing plus +optional local Android tooling (`aapt`, `aapt2`, `apkanalyzer`) when available. It +produces JSONL records suitable for ranking APKs before deeper semantic slicing. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path +from typing import Any, Iterable + +URL_RE = re.compile(rb"https?://[A-Za-z0-9._~:/?#\[\]@!$&'()*+,;=%-]{4,}") +DOMAIN_RE = re.compile( + rb"\b(?:[A-Za-z0-9-]{1,63}\.)+(?:com|net|org|io|co|app|dev|me|xyz|cloud|firebaseio|googleapis|amazonaws|azurewebsites)\b" +) +CUSTOM_SCHEME_RE = re.compile( + r"scheme\(0x[0-9a-f]+\)=\"([^\"]+)\"|android:scheme\(.*?\)=\"([^\"]+)\"" +) +HOST_RE = re.compile( + r"host\(0x[0-9a-f]+\)=\"([^\"]+)\"|android:host\(.*?\)=\"([^\"]+)\"" +) +PACKAGE_RE = re.compile(r"package: name='([^']+)'.*?versionName='([^']*)'", re.S) +SDK_RE = re.compile(r"sdkVersion:'([^']+)'|targetSdkVersion:'([^']+)'") +PERM_RE = re.compile(r"uses-permission(?:-sdk-\d+)?: name='([^']+)'") + +HIGH_VALUE_LIB_HINTS = { + "oauth": ["oauth", "openid", "appauth"], + "firebase": ["firebase", "google-services", "firebaseremoteconfig"], + "payment": [ + "stripe", + "braintree", + "adyen", + "paypal", + "checkout", + "paytm", + "razorpay", + ], + "webview_bridge": ["javascriptinterface", "addjavascriptinterface"], + "react_native": ["reactnative", "com.facebook.react"], + "flutter": ["flutter", "io.flutter"], +} + +# Binary AndroidManifest.xml is hard to decode correctly without Android build +# tools, but package names and component/link strings often survive in the string +# pool. These regexes provide a dependency-free fallback so target ranking is not +# blind when aapt/aapt2 is unavailable. +PACKAGE_BYTES_RE = re.compile( + rb"\b[A-Za-z][A-Za-z0-9_]*(?:\.[A-Za-z][A-Za-z0-9_]*){2,}\b" +) +DEEP_LINK_HOST_HINT_RE = re.compile( + rb"\b(?:[A-Za-z0-9-]+\.)+(?:com|net|org|io|co|app|dev|me|xyz|cloud|br|vn|th|id|in)\b" +) +SCHEME_HINT_RE = re.compile(rb"\b[a-z][a-z0-9+.-]{2,24}://") +ANDROID_COMPONENT_HINTS = [b"Activity", b"Service", b"Receiver", b"Provider"] + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def run_cmd(cmd: list[str], timeout: int = 20) -> str | None: + if not shutil.which(cmd[0]): + return None + try: + proc = subprocess.run( + cmd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + except Exception: + return None + return proc.stdout if proc.stdout else None + + +def iter_apks(paths: Iterable[Path]) -> list[Path]: + out: list[Path] = [] + for p in paths: + if p.is_dir(): + out.extend(sorted(x for x in p.rglob("*.apk") if x.is_file())) + elif p.is_file() and p.suffix.lower() == ".apk": + out.append(p) + return sorted(set(out)) + + +def printable_ascii_runs(data: bytes, min_len: int = 4) -> list[bytes]: + runs = re.findall(rb"[ -~]{%d,}" % min_len, data) + # Binary XML UTF-16LE string pools often appear as ASCII bytes separated by NULs. + nul_stripped = data.replace(b"\x00", b"") + if nul_stripped != data: + runs.extend(re.findall(rb"[ -~]{%d,}" % min_len, nul_stripped)) + return runs + + +def manifest_string_hints(data: bytes) -> dict[str, Any]: + runs = printable_ascii_runs(data) + joined = b"\n".join(runs) + packages = sorted( + {x.decode("utf-8", "ignore") for x in PACKAGE_BYTES_RE.findall(joined)} + ) + component_like = [ + p for p in packages if any(h.decode() in p for h in ANDROID_COMPONENT_HINTS) + ] + schemes = sorted( + { + m.group(0)[:-3].decode("utf-8", "ignore") + for m in SCHEME_HINT_RE.finditer(joined) + } + ) + hosts = sorted( + {x.decode("utf-8", "ignore") for x in DEEP_LINK_HOST_HINT_RE.findall(joined)} + ) + lower = joined.lower() + return { + "package_like_strings": packages[:200], + "component_like_strings": component_like[:200], + "scheme_hints": schemes[:100], + "host_hints": hosts[:200], + "mentions_browsable": b"android.intent.category.browsable" in lower + or b"browsable" in lower, + "mentions_view_action": b"android.intent.action.view" in lower, + "mentions_exported": b"exported" in lower, + } + + +def zip_inventory(apk: Path, max_bytes_per_file: int = 2_000_000) -> dict[str, Any]: + """Stream per-file content through regex/substring matchers. + + The previous implementation buffered the lowercased concatenation of every + interesting file (up to 80 × 2 MB = 160 MB per APK) into a bytearray, then + ran four passes over the full blob. That made resident memory scale with + APK content size and multiplied by the worker count. We now process one + file at a time, accumulating only small bounded result sets (capped URL, + domain, and library-hint collections). + """ + info: dict[str, Any] = { + "zip_entries": 0, + "dex_files": [], + "native_libs": [], + "interesting_files": [], + "urls": [], + "domains": [], + "library_hints": {}, + "manifest_string_hints": {}, + } + manifest_bytes = b"" + url_set: set[str] = set() + domain_set: set[str] = set() + lib_hits: dict[str, set[str]] = {label: set() for label in HIGH_VALUE_LIB_HINTS} + url_cap, domain_cap, hit_cap = 200, 300, 8 + needles_lc = { + label: [n.lower().encode("latin1") for n in needles] + for label, needles in HIGH_VALUE_LIB_HINTS.items() + } + + def absorb(buf: bytes) -> None: + # Cheap: enrich bounded sets, stop scanning if both caps reached. + if len(url_set) < url_cap: + for m in URL_RE.findall(buf): + if len(url_set) >= url_cap: + break + url_set.add(m.decode("utf-8", "ignore")) + if len(domain_set) < domain_cap: + for m in DOMAIN_RE.findall(buf): + if len(domain_set) >= domain_cap: + break + domain_set.add(m.decode("utf-8", "ignore")) + lc = buf.lower() + for label, needles in needles_lc.items(): + if len(lib_hits[label]) >= hit_cap: + continue + for needle in needles: + if needle in lc: + lib_hits[label].add(needle.decode("latin1")) + + try: + with zipfile.ZipFile(apk) as zf: + names = zf.namelist() + info["zip_entries"] = len(names) + info["dex_files"] = [ + n for n in names if re.fullmatch(r"classes\d*\.dex", Path(n).name) + ] + info["native_libs"] = [ + n for n in names if n.startswith("lib/") and n.endswith(".so") + ][:500] + interesting_suffixes = ( + ".xml", + ".json", + ".properties", + ".txt", + ".html", + ".js", + ".dex", + ) + interesting_names = [ + n + for n in names + if n.endswith(interesting_suffixes) + or "firebase" in n.lower() + or "google" in n.lower() + ] + info["interesting_files"] = interesting_names[:500] + if "AndroidManifest.xml" in names: + try: + manifest_bytes = zf.read("AndroidManifest.xml")[:max_bytes_per_file] + absorb(manifest_bytes) + except Exception: + manifest_bytes = b"" + for n in interesting_names[:80]: + try: + with zf.open(n) as fh: + data = fh.read(max_bytes_per_file) + absorb(data) + del data + except Exception: + continue + except zipfile.BadZipFile: + info["error"] = "bad_zip" + return info + + info["urls"] = sorted(url_set) + info["domains"] = sorted(domain_set) + if manifest_bytes: + info["manifest_string_hints"] = manifest_string_hints(manifest_bytes) + info["library_hints"] = { + label: sorted(hits) for label, hits in lib_hits.items() if hits + } + return info + + +def parse_aapt_badging(output: str | None) -> dict[str, Any]: + parsed: dict[str, Any] = {"permissions": [], "sdk": {}} + if not output: + return parsed + m = PACKAGE_RE.search(output) + if m: + parsed["package"] = m.group(1) + parsed["version_name"] = m.group(2) + parsed["permissions"] = sorted(set(PERM_RE.findall(output))) + for m in SDK_RE.finditer(output): + if m.group(1): + parsed["sdk"]["min"] = m.group(1) + if m.group(2): + parsed["sdk"]["target"] = m.group(2) + app_label = re.search(r"application-label(?:-[a-zA-Z0-9_]+)?:'([^']+)'", output) + if app_label: + parsed["app_label"] = app_label.group(1) + return parsed + + +def parse_aapt_xmltree(output: str | None) -> dict[str, Any]: + parsed: dict[str, Any] = { + "schemes": [], + "hosts": [], + "manifest_xmltree_available": bool(output), + } + if not output: + return parsed + schemes = [] + for m in CUSTOM_SCHEME_RE.finditer(output): + schemes.append(next(g for g in m.groups() if g)) + hosts = [] + for m in HOST_RE.finditer(output): + hosts.append(next(g for g in m.groups() if g)) + parsed["schemes"] = sorted(set(schemes)) + parsed["hosts"] = sorted(set(hosts)) + parsed["browsable_mentions"] = output.count("android.intent.category.BROWSABLE") + parsed["view_action_mentions"] = output.count("android.intent.action.VIEW") + parsed["exported_true_mentions"] = output.count("exported") and output.count( + "0xffffffff" + ) + return parsed + + +# APKiD signal categories. Anything in HEAVY blocks static review almost entirely +# (encrypted strings, reflective dispatch, packed dex). Anything in MEDIUM +# (DexGuard 5-8 generations) reduces but does not prevent useful triage. +# The capability prefers reading source over reading bytecode, so we down-weight +# both, harder for HEAVY. +APKID_HEAVY_PACKERS = { + "DexProtector", + "Bangcle", + "Tencent's Legu", + "Tencent Legu", + "ApkProtect", + "APKProtect", + "Promon SHIELD", + "Liapp", + "AppSolid", + "Qihoo 360", + "Baidu", + "NQShield", + "Ijiami", + "Kiro", + "Jiagu", +} +APKID_MEDIUM_PACKERS = { + "DexGuard", # historically 5.x-8.x are visible + "DexGuard 9.x", + "Allatori", + "ProGuard", + "Stringer Java Obfuscator", +} +# Signals that something protector-ish is going on without naming it +APKID_AMBIGUOUS_HINTS = { + "unreadable field names", + "unreadable method names", + "illegal class name", + "anti_disassembly", +} + + +def summarize_packers(apkid_result: dict[str, Any] | None) -> dict[str, Any]: + """Reduce an APKiD result blob to a small, comparable shape. + + Returns a dict with: + hits: sorted unique list of non-noise APKiD strings + heavy: True if a known anti-static commercial packer is present + medium: True if a name-mangling obfuscator is present + ambiguous: True if APKiD flagged class/method-name oddities + tier: "heavy" | "medium" | "ambiguous" | "clean" + """ + out = { + "hits": [], + "heavy": False, + "medium": False, + "ambiguous": False, + "tier": "clean", + } + if not apkid_result or not isinstance(apkid_result, dict): + return out + result = apkid_result.get("result") or {} + files = result.get("files") or [] + hits: set[str] = set() + NOISE = { + "android sdk (dx)", + "android sdk (dx since v35)", + "android sdk (r8)", + "android sdk (dexlib 2.x)", + "android sdk (dexlib 1.x)", + } + for fmatch in files: + m = fmatch.get("matches") or {} + for cat in ( + "packer", + "obfuscator", + "protector", + "anti_disassembly", + "anti_vm", + "anti_debug", + ): + for h in m.get(cat) or []: + if h and h not in NOISE: + hits.add(h) + out["hits"] = sorted(hits) + + def _matches(needle: str, haystack: set[str]) -> bool: + nl = needle.lower() + return any(nl in h.lower() for h in haystack) + + for known in APKID_HEAVY_PACKERS: + if _matches(known, hits): + out["heavy"] = True + break + for known in APKID_MEDIUM_PACKERS: + if _matches(known, hits): + out["medium"] = True + break + for ambig in APKID_AMBIGUOUS_HINTS: + if _matches(ambig, hits): + out["ambiguous"] = True + break + if out["heavy"]: + out["tier"] = "heavy" + elif out["medium"]: + out["tier"] = "medium" + elif out["ambiguous"]: + out["tier"] = "ambiguous" + return out + + +def rank(record: dict[str, Any]) -> dict[str, Any]: + """Score an inventory record for static-analysis priority. + + Higher is better. Negative reasons (packers, obfuscators) are recorded as + `reasons` entries prefixed with ``penalty:`` and subtract from the score. + """ + score = 0 + reasons: list[str] = [] + manifest_hints = record.get("manifest_string_hints", {}) or {} + schemes = record.get("schemes") or manifest_hints.get("scheme_hints") or [] + hosts = record.get("hosts") or manifest_hints.get("host_hints") or [] + if schemes: + score += 3 + reasons.append("custom_or_app_link_schemes") + if hosts: + score += 2 + reasons.append("declared_or_manifest_hint_link_hosts") + if record.get("browsable_mentions", 0) or manifest_hints.get("mentions_browsable"): + score += 3 + reasons.append("browsable_entrypoints") + if manifest_hints.get("component_like_strings"): + score += min(4, len(manifest_hints["component_like_strings"]) // 5 + 1) + reasons.append("manifest_component_string_hints") + hints = record.get("library_hints", {}) + for key, weight in [ + ("oauth", 4), + ("payment", 4), + ("webview_bridge", 4), + ("firebase", 2), + ("react_native", 1), + ("flutter", 1), + ]: + if key in hints: + score += weight + reasons.append(f"library_hint:{key}") + if len(record.get("urls", [])) > 20: + score += 2 + reasons.append("many_embedded_urls") + if len(record.get("domains", [])) > 20: + score += 2 + reasons.append("many_embedded_domains") + + # Packer / obfuscator penalty. Applied here when the runner has already + # merged an `apkid_summary` block into the record. Default behaviour is + # unchanged for records without packer data — the rank stays comparable + # with prior runs. + pk = record.get("apkid_summary") or {} + tier = pk.get("tier") + if tier == "heavy": + score -= 12 + reasons.append("penalty:heavy_packer") + elif tier == "medium": + score -= 5 + reasons.append("penalty:medium_obfuscator") + elif tier == "ambiguous": + score -= 2 + reasons.append("penalty:ambiguous_obfuscation") + + record["semantic_priority"] = {"score": score, "reasons": sorted(set(reasons))} + return record + + +def analyze_apk(apk: Path) -> dict[str, Any]: + rec: dict[str, Any] = { + "apk": str(apk), + "file_name": apk.name, + "size": apk.stat().st_size, + "sha256": sha256_file(apk), + } + rec.update(zip_inventory(apk)) + aapt = shutil.which("aapt") or shutil.which("aapt2") + if aapt: + rec.update(parse_aapt_badging(run_cmd([aapt, "dump", "badging", str(apk)]))) + rec.update( + parse_aapt_xmltree( + run_cmd([aapt, "dump", "xmltree", str(apk), "AndroidManifest.xml"]) + ) + ) + else: + rec["tool_warnings"] = [ + "aapt/aapt2 not found; package/component/link metadata uses binary-manifest string-pool fallback and is partial" + ] + return rank(rec) + + +def main() -> int: + ap = argparse.ArgumentParser( + description="Extract semantic attack-surface inventory from APKs" + ) + ap.add_argument( + "paths", nargs="+", type=Path, help="APK files or directories containing APKs" + ) + ap.add_argument( + "--out", type=Path, help="Write JSONL output to this path; defaults to stdout" + ) + ap.add_argument( + "--json", action="store_true", help="Emit a single JSON array instead of JSONL" + ) + args = ap.parse_args() + + apks = iter_apks(args.paths) + # Stream one record at a time so memory stays at O(one record), not O(corpus). + # JSON-array mode still buffers (it has to write a single array), but the JSONL + # path — which is what the corpus runner uses — is fully streaming. + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + out_fh = args.out.open("w") if args.out else sys.stdout + try: + if args.json: + out_fh.write("[\n") + for idx, apk in enumerate(apks): + if idx: + out_fh.write(",\n") + out_fh.write(json.dumps(analyze_apk(apk), sort_keys=True)) + out_fh.write("\n]\n") + else: + for apk in apks: + out_fh.write(json.dumps(analyze_apk(apk), sort_keys=True)) + out_fh.write("\n") + finally: + if args.out: + out_fh.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/capabilities/android-apk-research/scripts/extract_corpus_components.py b/capabilities/android-apk-research/scripts/extract_corpus_components.py new file mode 100755 index 0000000..9b2fb41 --- /dev/null +++ b/capabilities/android-apk-research/scripts/extract_corpus_components.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +"""Stream every per-APK androguard.json under a corpus inventory dir and emit +one JSONL row per component, with the manifest facts joined to APK-level info. + +Falls back to `aapt2 dump xmltree` for APKs whose androguard.json errored out +(empirically: multi-dex APKs with >=22 classes*.dex break Androguard 4.1.3 with +`unpack requires a buffer of 2 bytes`). + +Output schema (one JSON object per line): + + { + "apk_sha256": "...", # SHA256 of the .apk + "apk_path": "...", # corpus relative path (if known) + "package": "com.example", + "version_name": "...", + "version_code": 42, + "min_sdk": 29, + "target_sdk": 34, + "apkid_tier": "clean|ambiguous|medium|heavy", + "runtime_kind": "native|react_native_hermes|...", # if available + "impact_class": "C_wallet", # if joined from triage manifest + "type": "activity|service|receiver|provider", + "name": "com.example.X", + "exported": true, + "permission": "...", # the android:permission on the component + "perm_protection": "signature|normal|...", # joined from manifest perm decls + "browsable": true, + "schemes": ["dashlane", "otpauth"], + "hosts": ["*"], + "paths": ["/vault", "/mplesslogin", ...], + "actions": ["android.intent.action.VIEW", ...], + "categories": ["android.intent.category.BROWSABLE", ...], + "mime_types": ["image/*"], + "grant_uri": null, # provider-only + "source": "androguard|aapt2" + } + +Usage: + python3 extract_corpus_components.py \ + --inventory-dir findings/corpus-2/inventory/apks \ + --triage-manifest findings/corpus-2/triage/manifest.jsonl \ + --out findings/corpus-2/triage/components.jsonl +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import Any, Iterator + +ANS = "http://schemas.android.com/apk/res/android:" + + +def strip_ns(k: str) -> str: + return k.replace(ANS, "android:") + + +def parse_aapt2_manifest(apk_path: Path) -> dict[str, Any] | None: + """Return a dict shaped like androguard.json from `aapt2 dump xmltree`. + + Fallback for APKs that broke Androguard. Best-effort; only fills the + fields this script consumes. + """ + try: + proc = subprocess.run( + [ + "aapt2", + "dump", + "xmltree", + "--file", + "AndroidManifest.xml", + str(apk_path), + ], + capture_output=True, + text=True, + timeout=60, + ) + except (FileNotFoundError, subprocess.TimeoutExpired) as exc: + print(f"aapt2 failed on {apk_path}: {exc}", file=sys.stderr) + return None + if proc.returncode != 0: + print(f"aapt2 nonzero on {apk_path}: {proc.stderr[:200]}", file=sys.stderr) + return None + out = proc.stdout + + elem_re = re.compile(r"^(\s*)E: (\w+)") + attr_re = re.compile(r"^(\s*)A: (\S+?)(?:\([^)]*\))?=(.*?)$") + raw_re = re.compile(r'\(Raw: "([^"]*)"\)') + + stack: list[list[Any]] = [] + events: list[list[Any]] = [] + for line in out.splitlines(): + em = elem_re.match(line) + if em: + indent, name = len(em.group(1)), em.group(2) + while stack and stack[-1][0] >= indent: + stack.pop() + node = [indent, name, {}] + stack.append(node) + events.append(node) + continue + am = attr_re.match(line) + if am and stack: + indent = len(am.group(1)) + key = strip_ns(am.group(2)) + value = am.group(3).strip() + rm = raw_re.search(value) + value = rm.group(1) if rm else value.strip('"') + for s in reversed(stack): + if s[0] < indent: + s[2][key] = value + break + + # package + package = None + for ev in events: + if ev[1] == "manifest": + package = ev[2].get("package") + break + + # permissions declared at top level → name -> protectionLevel + declared_perms: dict[str, str] = {} + for ev in events: + if ev[1] == "permission": + n = ev[2].get("android:name") + if n: + declared_perms[n] = ev[2].get("android:protectionLevel", "") + + components: list[dict[str, Any]] = [] + for idx, ev in enumerate(events): + if ev[1] not in ("activity", "service", "receiver", "provider"): + continue + base = ev[0] + exported_raw = ev[2].get("android:exported", "") + exported_norm = None + if exported_raw in ("0xffffffff", "true", "-1"): + exported_norm = True + elif exported_raw in ("0x0", "false", "0"): + exported_norm = False + intent_filters: list[dict[str, Any]] = [] + cur_filter: dict[str, Any] | None = None + j = idx + 1 + while j < len(events) and events[j][0] > base: + child = events[j] + if child[1] == "intent-filter": + cur_filter = { + "actions": [], + "categories": [], + "data": [], + "browsable": False, + "view": False, + } + intent_filters.append(cur_filter) + elif cur_filter is not None and child[1] == "action": + name = child[2].get("android:name") + if name: + cur_filter["actions"].append(name) + if name == "android.intent.action.VIEW": + cur_filter["view"] = True + elif cur_filter is not None and child[1] == "category": + name = child[2].get("android:name") + if name: + cur_filter["categories"].append(name) + if name == "android.intent.category.BROWSABLE": + cur_filter["browsable"] = True + elif cur_filter is not None and child[1] == "data": + d: dict[str, Any] = {} + for k in ( + "android:scheme", + "android:host", + "android:port", + "android:path", + "android:pathPrefix", + "android:pathPattern", + "android:mimeType", + ): + if k in child[2]: + d[k.split(":", 1)[1]] = child[2][k] + if d: + cur_filter["data"].append(d) + j += 1 + any_browsable = any(f["browsable"] for f in intent_filters) + components.append( + { + "type": ev[1], + "name": ev[2].get("android:name", ""), + "raw_name": ev[2].get("android:name", ""), + "exported": exported_norm, + "permission": ev[2].get("android:permission"), + "browsable": any_browsable, + "view": any(f["view"] for f in intent_filters), + "grantUriPermissions": ev[2].get("android:grantUriPermissions"), + "intent_filters": intent_filters, + } + ) + + return { + "package": package, + "components": components, + "_declared_perms": declared_perms, + "_source": "aapt2", + } + + +def normalize_androguard_data(g: dict[str, Any]) -> dict[str, Any]: + """Normalize an androguard.json into the same shape the consumer expects. + + Adds `_declared_perms` as empty (androguard didn't capture protection + levels here; we keep the field for shape parity). + """ + g.setdefault("_declared_perms", {}) + g["_source"] = "androguard" + return g + + +def gather_intent_filter_facts(comp: dict[str, Any]) -> dict[str, Any]: + schemes: list[str] = [] + hosts: list[str] = [] + paths: list[str] = [] + actions: list[str] = [] + categories: list[str] = [] + mimes: list[str] = [] + for f in comp.get("intent_filters", []) or []: + for a in f.get("actions") or []: + if a not in actions: + actions.append(a) + for c in f.get("categories") or []: + if c not in categories: + categories.append(c) + for d in f.get("data") or []: + if "scheme" in d and d["scheme"] not in schemes: + schemes.append(d["scheme"]) + if "host" in d and d["host"] not in hosts: + hosts.append(d["host"]) + for pk in ("path", "pathPrefix", "pathPattern"): + if pk in d: + paths.append(f"{pk}={d[pk]}") + if "mimeType" in d and d["mimeType"] not in mimes: + mimes.append(d["mimeType"]) + return { + "schemes": schemes, + "hosts": hosts, + "paths": paths, + "actions": actions, + "categories": categories, + "mime_types": mimes, + } + + +def load_triage_index(path: Path | None) -> dict[str, dict[str, Any]]: + if not path or not path.exists(): + return {} + idx: dict[str, dict[str, Any]] = {} + for line in path.read_text().splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + sha = obj.get("sha256") + if sha: + idx[sha] = obj + return idx + + +def load_runtime_kind_index(path: Path | None) -> dict[str, str]: + if not path or not path.exists(): + return {} + idx: dict[str, str] = {} + for line in path.read_text().splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + # The runtime_kind sweep is keyed by apk path; we keep package + kind. + apk = obj.get("apk", "") + # Strip dir + stem = Path(apk).stem + # Map package-version stem -> kind. We'll join via apk_path basename match. + idx[stem] = obj.get("runtime_kind", "") + return idx + + +def iter_components( + inventory_dir: Path, + triage_idx: dict[str, dict[str, Any]], + runtime_idx: dict[str, str], +) -> Iterator[dict[str, Any]]: + for sha_dir in sorted(inventory_dir.iterdir()): + if not sha_dir.is_dir(): + continue + sha = sha_dir.name + ag_path = sha_dir / "androguard.json" + if not ag_path.exists(): + continue + try: + ag = json.loads(ag_path.read_text()) + except json.JSONDecodeError as exc: + print(f"bad androguard.json for {sha}: {exc}", file=sys.stderr) + continue + + triage = triage_idx.get(sha, {}) + apk_path = triage.get("apk_path") + used_fallback = False + + if ag.get("status") == "error" or not ag.get("components"): + if apk_path and Path(apk_path).exists(): + fb = parse_aapt2_manifest(Path(apk_path)) + if fb: + ag = fb + used_fallback = True + else: + continue + else: + continue + else: + ag = normalize_androguard_data(ag) + + package = ag.get("package") or triage.get("package") + declared_perms = ag.get("_declared_perms") or {} + + # APK-level metadata + apkid_tier = triage.get("apkid_tier") + impact_class = triage.get("impact_class") + version_name = ag.get("version_name") or triage.get("version_name") + version_code = ag.get("version_code") or triage.get("version_code") + min_sdk = ag.get("min_sdk") + target_sdk = ag.get("target_sdk") + + # Runtime kind via apk basename stem (best-effort) + runtime_kind = "" + if apk_path: + runtime_kind = runtime_idx.get(Path(apk_path).stem, "") + + for comp in ag.get("components", []) or []: + facts = gather_intent_filter_facts(comp) + perm = comp.get("permission") + yield { + "apk_sha256": sha, + "apk_path": apk_path, + "package": package, + "version_name": version_name, + "version_code": version_code, + "min_sdk": min_sdk, + "target_sdk": target_sdk, + "apkid_tier": apkid_tier, + "runtime_kind": runtime_kind, + "impact_class": impact_class, + "type": comp.get("type"), + "name": comp.get("name"), + "exported": comp.get("exported"), + "permission": perm, + "perm_protection": declared_perms.get(perm or "", ""), + "browsable": comp.get("browsable", False), + "view": comp.get("view", False), + "grant_uri": comp.get("grantUriPermissions"), + "source": ag.get("_source") + or ("aapt2" if used_fallback else "androguard"), + **facts, + } + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--inventory-dir", required=True, type=Path) + ap.add_argument( + "--triage-manifest", + type=Path, + default=None, + help="optional triage manifest JSONL keyed by sha256 for apk_path / impact_class / apkid_tier joins", + ) + ap.add_argument( + "--runtime-kind", + type=Path, + default=None, + help="optional runtime_kind JSONL from detect_runtime_kind.sh sweep", + ) + ap.add_argument("--out", required=True, type=Path) + args = ap.parse_args() + + triage_idx = load_triage_index(args.triage_manifest) + runtime_idx = load_runtime_kind_index(args.runtime_kind) + + args.out.parent.mkdir(parents=True, exist_ok=True) + n_rows = 0 + n_fallback = 0 + seen_sha: set[str] = set() + with args.out.open("w") as f: + for row in iter_components(args.inventory_dir, triage_idx, runtime_idx): + f.write(json.dumps(row, sort_keys=True) + "\n") + n_rows += 1 + if row["source"] == "aapt2": + n_fallback += 1 + seen_sha.add(row["apk_sha256"]) + print(f"wrote {n_rows} components from {len(seen_sha)} APKs to {args.out}") + if n_fallback: + print(f" ({n_fallback} rows used aapt2 fallback)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/capabilities/android-apk-research/scripts/gplaydl_bulk.py b/capabilities/android-apk-research/scripts/gplaydl_bulk.py new file mode 100755 index 0000000..cc24b56 --- /dev/null +++ b/capabilities/android-apk-research/scripts/gplaydl_bulk.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +"""Bulk APK downloader on top of `gplaydl` (PyPI). + +`gplaydl` is a one-package-at-a-time CLI tool with anonymous-token auth via +Aurora Store's dispenser. Empirically it sustains ~20–25 MB/s against Google's +CDN, vs. AndroZoo's ~440 KB/s per connection. For a 5 GB corpus that's the +difference between an afternoon and 4+ hours. + +What this wrapper adds: + +- Bounded parallelism (default 8) — gplaydl's internal parallelism is per-app + splits/extras, not across packages. We want to download many packages at once. +- JSONL selection input compatible with our existing corpus_design flow + (`{"package": "com.foo", ...}` lines). +- Resume-by-prefix: detects packages already present in the output directory and + skips re-download. Useful when corpus selection grows incrementally. +- Per-package download manifest (JSONL) compatible with `run_corpus_inventory` + via plain filesystem path; gplaydl names files `-.apk`. +- Honest failure tracking: gplaydl exits non-zero on delisted packages and + region-locked ones; we record the failure and keep going. + +Limitations to know: + +- Anonymous token = "delisted from Google Play" means we cannot fetch it (we + fall back to AndroZoo in that case — see ``androzoo_download.py``). +- gplaydl downloads the *current* version. For older versions, pass --version. +- gplaydl writes split APKs and OBB extras by default; we pass --no-splits + --no-extras for first-pass research corpora unless ``--include-splits`` or + ``--include-extras`` is set. + +Usage: + + gplaydl_bulk.py corpus/selection.jsonl --out-dir corpus/apks \\ + --manifest-out corpus/download_manifest.jsonl --jobs 8 +""" + +from __future__ import annotations + +import argparse +import concurrent.futures as cf +import json +import os +import shutil +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Any, Iterator + + +def iter_jsonl(path: Path) -> Iterator[dict[str, Any]]: + with path.open("r", errors="ignore") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, dict) and obj.get("package"): + yield obj + + +def ensure_token(arch: str) -> None: + """gplaydl caches tokens at ~/.config/gplaydl/auth-.json. + + Run `gplaydl auth` once up front so concurrent workers don't race on it. + """ + token_path = Path.home() / ".config" / "gplaydl" / f"auth-{arch}.json" + if token_path.exists(): + return + proc = subprocess.run( + ["gplaydl", "auth", "--arch", arch], capture_output=True, text=True, check=False + ) + if proc.returncode != 0: + raise SystemExit( + f"gplaydl auth failed: {proc.stderr.strip() or proc.stdout.strip()}" + ) + + +def find_existing_apk(out_dir: Path, package: str) -> Path | None: + # gplaydl names files -.apk; pick the largest if multiple exist. + candidates = sorted( + out_dir.glob(f"{package}-*.apk"), + key=lambda p: p.stat().st_size if p.exists() else 0, + reverse=True, + ) + return candidates[0] if candidates else None + + +def download_one( + row: dict[str, Any], + out_dir: Path, + arch: str, + timeout: int, + include_splits: bool, + include_extras: bool, + version: int | None, + force: bool, +) -> dict[str, Any]: + package = str(row["package"]) + result: dict[str, Any] = dict(row) + result["arch"] = arch + result["source"] = "gplaydl" + + if not force: + existing = find_existing_apk(out_dir, package) + if existing is not None: + result["download_status"] = "exists" + result["path"] = str(existing) + result["size_bytes"] = existing.stat().st_size + return result + + cmd = ["gplaydl", "download", package, "-o", str(out_dir), "-a", arch] + if not include_splits: + cmd.append("--no-splits") + if not include_extras: + cmd.append("--no-extras") + if version is not None: + cmd += ["-v", str(version)] + started = time.time() + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout, check=False + ) + except subprocess.TimeoutExpired: + result["download_status"] = "timeout" + result["error"] = f"gplaydl timed out after {timeout}s" + result["duration_sec"] = round(time.time() - started, 2) + return result + result["duration_sec"] = round(time.time() - started, 2) + if proc.returncode != 0: + result["download_status"] = "error" + result["returncode"] = proc.returncode + # gplaydl prints "App not found" / "Region locked" / "Premium app" to stdout. + result["error"] = (proc.stdout or proc.stderr).strip()[-2000:] + return result + apk = find_existing_apk(out_dir, package) + if apk is None: + result["download_status"] = "error" + result["error"] = "gplaydl exit 0 but no APK on disk" + return result + result["download_status"] = "downloaded" + result["path"] = str(apk) + result["size_bytes"] = apk.stat().st_size + return result + + +def main() -> int: + ap = argparse.ArgumentParser( + description="Bulk-download APKs via gplaydl with bounded parallelism" + ) + ap.add_argument( + "selection_jsonl", + type=Path, + help='JSONL with one {"package":"com.foo"} object per line', + ) + ap.add_argument("--out-dir", type=Path, required=True) + ap.add_argument( + "--manifest-out", + type=Path, + help="Output manifest JSONL (default /download_manifest.jsonl)", + ) + ap.add_argument( + "--jobs", + type=int, + default=8, + help="Parallel downloads (default 8). gplaydl's own download streams " + "are already chunked, so 8 against the CDN is a safe sweet spot.", + ) + ap.add_argument("--arch", default="arm64", choices=["arm64", "armv7"]) + ap.add_argument( + "--timeout", + type=int, + default=600, + help="Per-package timeout in seconds (default 600)", + ) + ap.add_argument( + "--include-splits", + action="store_true", + help="Also download split APKs (config splits / language splits)", + ) + ap.add_argument( + "--include-extras", + action="store_true", + help="Also download OBB files and Play Asset Delivery packs", + ) + ap.add_argument( + "--force", + action="store_true", + help="Re-download even if an APK already exists for the package", + ) + ap.add_argument("--limit", type=int) + args = ap.parse_args() + + if not shutil.which("gplaydl"): + raise SystemExit("gplaydl not on PATH; pip install gplaydl") + + args.out_dir.mkdir(parents=True, exist_ok=True) + manifest_path = args.manifest_out or (args.out_dir / "download_manifest.jsonl") + manifest_path.parent.mkdir(parents=True, exist_ok=True) + tmp_manifest = manifest_path.with_suffix( + manifest_path.suffix + f".tmp.{os.getpid()}" + ) + + ensure_token(args.arch) + + rows = list(iter_jsonl(args.selection_jsonl)) + if args.limit is not None: + rows = rows[: args.limit] + total = len(rows) + + counters = {"downloaded": 0, "exists": 0, "errored": 0, "timeout": 0} + started = time.time() + manifest_lock = threading.Lock() + + def _work(row: dict[str, Any]) -> dict[str, Any]: + return download_one( + row, + args.out_dir, + args.arch, + args.timeout, + args.include_splits, + args.include_extras, + row.get("version_code") + if isinstance(row.get("version_code"), int) + else None, + args.force, + ) + + with ( + tmp_manifest.open("w") as manifest_fh, + cf.ThreadPoolExecutor(max_workers=max(1, args.jobs)) as pool, + ): + futures = {pool.submit(_work, row): row for row in rows} + completed = 0 + for fut in cf.as_completed(futures): + row = futures[fut] + try: + res = fut.result() + except Exception as exc: # noqa: BLE001 + res = dict(row) + res["download_status"] = "error" + res["error"] = str(exc) + status = res.get("download_status", "error") + if status == "downloaded": + counters["downloaded"] += 1 + elif status == "exists": + counters["exists"] += 1 + elif status == "timeout": + counters["timeout"] += 1 + else: + counters["errored"] += 1 + with manifest_lock: + manifest_fh.write(json.dumps(res, sort_keys=True)) + manifest_fh.write("\n") + manifest_fh.flush() + completed += 1 + elapsed = time.time() - started + rate = completed / elapsed if elapsed > 0 else 0 + eta = (total - completed) / rate if rate > 0 else float("inf") + print( + json.dumps( + { + "completed": completed, + "total": total, + "rate_per_sec": round(rate, 3), + "eta_sec": round(eta, 1) if eta != float("inf") else None, + "package": row["package"], + "status": status, + "path": res.get("path"), + "size_mb": round((res.get("size_bytes") or 0) / 1e6, 2), + "error": res.get("error"), + }, + sort_keys=True, + ), + flush=True, + ) + tmp_manifest.replace(manifest_path) + summary = { + "requested": total, + **counters, + "duration_sec": round(time.time() - started, 1), + "manifest": str(manifest_path), + "jobs": args.jobs, + "throughput_apks_per_min": round(total / (time.time() - started) * 60, 2) + if (time.time() - started) > 0 + else 0, + } + print(json.dumps(summary, sort_keys=True)) + return 0 if counters["errored"] + counters["timeout"] == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/capabilities/android-apk-research/scripts/normalize_findings.py b/capabilities/android-apk-research/scripts/normalize_findings.py new file mode 100755 index 0000000..534fe9b --- /dev/null +++ b/capabilities/android-apk-research/scripts/normalize_findings.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 +"""Normalize semantic Android finding hypotheses into JSONL/CSV/Markdown.""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path +from typing import Any + +RISK_ORDER = {"critical": 5, "high": 4, "medium": 3, "low": 2, "info": 1, "unknown": 0} +CONF_ORDER = { + "confirmed_dynamic": 5, + "strong_static_chain": 4, + "high": 3, + "needs_backend_validation": 3, + "needs_route_map_validation": 3, + "medium": 2, + "hardening_only": 1, + "low": 1, + "generic_library_noise": 0, + "unknown": 0, +} +REQUIRED = [ + "title", + "apk", + "package", + "entrypoint", + "source", + "trust_boundary", + "sink", + "impact", + "evidence", + "validation_plan", + "confidence", +] +CONFIDENCE_TIERS = { + "confirmed_dynamic", + "strong_static_chain", + "needs_backend_validation", + "needs_route_map_validation", + "hardening_only", + "generic_library_noise", +} + +# Default CWE / MASWE / MASVS mappings per `class` value, so an LLM that +# emits only `class` still produces well-tagged reports. The taxonomy lives +# in `skills/android-semantic-vuln-hunting/references/output-schema.md`; +# update both when a new class is introduced. +# +# MASWE ID grounding (verified against https://mas.owasp.org/MASWE/, beta): +# MASWE-0058 Insecure Deep Links (MASVS-PLATFORM) +# MASWE-0064 Insecure Content Providers (MASVS-PLATFORM) +# MASWE-0066 Insecure Intents (MASVS-PLATFORM) +# MASWE-0068 JavaScript Bridges in WebViews (MASVS-PLATFORM) +# Where no current MASWE cleanly maps (Dirty Stream, client-side trust, +# backend API abuse, request-signing replay, leaked host gates), leave +# `maswe` empty rather than asserting an unrelated ID — the CWE and MASVS +# columns carry the grounding instead. OWASP API Security Top 10 is the +# right backend-side framework for `apk_discovered_backend_*` once those +# get a separate `api_top10` column. +CLASS_TAXONOMY: dict[str, dict[str, list[str]]] = { + "deep_link_to_authenticated_webview": { + "cwe": ["CWE-939", "CWE-749"], + "maswe": ["MASWE-0058"], + "masvs": ["MASVS-PLATFORM", "MASVS-NETWORK"], + }, + "deep_link_to_js_bridge": { + "cwe": ["CWE-749", "CWE-829"], + "maswe": ["MASWE-0058", "MASWE-0068"], + "masvs": ["MASVS-PLATFORM"], + }, + "custom_scheme_arbitrary_webview": { + "cwe": ["CWE-939", "CWE-079"], + "maswe": ["MASWE-0058"], + "masvs": ["MASVS-PLATFORM"], + }, + "intent_redirection_private_component": { + "cwe": ["CWE-926", "CWE-940"], + "maswe": ["MASWE-0066"], + "masvs": ["MASVS-PLATFORM"], + }, + "intent_redirection_uri_grant_leak": { + "cwe": ["CWE-926", "CWE-200"], + "maswe": ["MASWE-0066"], + "masvs": ["MASVS-PLATFORM"], + }, + "dirty_stream_file_overwrite": { + "cwe": ["CWE-22", "CWE-73"], + "maswe": [], + "masvs": ["MASVS-PLATFORM", "MASVS-STORAGE"], + }, + "share_target_path_traversal": { + "cwe": ["CWE-22"], + "maswe": [], + "masvs": ["MASVS-PLATFORM", "MASVS-STORAGE"], + }, + "exported_provider_sqli": { + "cwe": ["CWE-89", "CWE-926"], + "maswe": ["MASWE-0064"], + "masvs": ["MASVS-PLATFORM"], + }, + "exported_provider_private_file_read": { + "cwe": ["CWE-200", "CWE-926"], + "maswe": ["MASWE-0064"], + "masvs": ["MASVS-PLATFORM", "MASVS-STORAGE"], + }, + "provider_uri_grant_confusion": { + "cwe": ["CWE-441", "CWE-926"], + "maswe": ["MASWE-0064", "MASWE-0066"], + "masvs": ["MASVS-PLATFORM"], + }, + "deep_link_auto_account_state_change": { + "cwe": ["CWE-352", "CWE-862"], + "maswe": ["MASWE-0058"], + "masvs": ["MASVS-AUTH", "MASVS-PLATFORM"], + }, + "client_state_auth_bypass": { + "cwe": ["CWE-602", "CWE-287"], + "maswe": [], + "masvs": ["MASVS-AUTH"], + }, + "apk_discovered_backend_bola": { + "cwe": ["CWE-639"], + "maswe": [], + "masvs": ["MASVS-AUTH", "MASVS-NETWORK"], + }, + "apk_discovered_backend_workflow_bypass": { + "cwe": ["CWE-841", "CWE-863"], + "maswe": [], + "masvs": ["MASVS-AUTH", "MASVS-NETWORK"], + }, + "apk_discovered_backend_mass_assignment": { + "cwe": ["CWE-915"], + "maswe": [], + "masvs": ["MASVS-NETWORK"], + }, + "apk_discovered_backend_ssrf_or_open_redirect": { + "cwe": ["CWE-918", "CWE-601"], + "maswe": [], + "masvs": ["MASVS-NETWORK"], + }, + "apk_discovered_graphql_operation_abuse": { + "cwe": ["CWE-639", "CWE-863"], + "maswe": [], + "masvs": ["MASVS-NETWORK"], + }, + "apk_discovered_grpc_operation_abuse": { + "cwe": ["CWE-639", "CWE-863"], + "maswe": [], + "masvs": ["MASVS-NETWORK"], + }, + "webview_bridge_to_mobile_api_action": { + "cwe": ["CWE-749", "CWE-829"], + "maswe": ["MASWE-0068"], + "masvs": ["MASVS-PLATFORM"], + }, + "mobile_request_signing_replay_or_confusion": { + "cwe": ["CWE-345", "CWE-294"], + "maswe": [], + "masvs": ["MASVS-NETWORK", "MASVS-CRYPTO"], + }, + "leaked_host_feature_flag_gated": { + "cwe": ["CWE-1188"], + "maswe": [], + "masvs": ["MASVS-NETWORK", "MASVS-CODE"], + }, + "leaked_host_intent_extra_gated": { + "cwe": ["CWE-1188", "CWE-926"], + "maswe": ["MASWE-0066"], + "masvs": ["MASVS-NETWORK", "MASVS-PLATFORM"], + }, +} + + +def load_records(paths: list[Path]) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for path in paths: + text = path.read_text() + try: + obj = json.loads(text) + if isinstance(obj, list): + records.extend(x for x in obj if isinstance(x, dict)) + elif isinstance(obj, dict): + records.append(obj) + continue + except json.JSONDecodeError: + pass + for line in text.splitlines(): + if not line.strip(): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError as exc: + print(f"skip invalid JSONL line in {path}: {exc}", file=sys.stderr) + continue + if isinstance(obj, dict): + records.append(obj) + return records + + +def normalize(rec: dict[str, Any]) -> dict[str, Any]: + out = {k: rec.get(k) for k in REQUIRED} + for k, v in rec.items(): + out.setdefault(k, v) + out["confidence"] = str(out.get("confidence") or "unknown").lower() + out["risk"] = str( + out.get("risk") or infer_risk(str(out.get("impact") or "")) + ).lower() + out["confidence_tier"] = str( + out.get("confidence_tier") or infer_confidence_tier(out) + ).lower() + out["validation_tier"] = str( + out.get("validation_tier") or infer_validation_tier(out) + ).lower() + # Tag classes against MASVS / CWE / MASWE. If the record carries explicit + # tags, those win; otherwise we fill defaults from CLASS_TAXONOMY so an + # LLM that emits only `class` still produces well-tagged output. + cls = str(out.get("class") or "").strip() + defaults = CLASS_TAXONOMY.get(cls, {}) + out["masvs"] = listify(out.get("masvs")) or list(defaults.get("masvs", [])) + out["cwe"] = listify(out.get("cwe")) or list(defaults.get("cwe", [])) + out["maswe"] = listify(out.get("maswe")) or list(defaults.get("maswe", [])) + out["evidence"] = listify(out.get("evidence")) + out["validation_plan"] = listify(out.get("validation_plan")) + out["missing_evidence"] = listify(out.get("missing_evidence")) + out["dedupe_key"] = out.get("dedupe_key") or "|".join( + str(out.get(k) or "") + for k in ["package", "entrypoint", "source", "sink", "impact"] + ) + out["missing_fields"] = [k for k in REQUIRED if not out.get(k)] + return out + + +def listify(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(x) for x in value] + return [str(value)] + + +def infer_confidence_tier(rec: dict[str, Any]) -> str: + text = " ".join( + str(rec.get(k) or "") + for k in ["impact", "trust_boundary", "scanner_gap", "source", "sink"] + ).lower() + if str(rec.get("confirmed") or "").lower() in {"true", "yes", "1"}: + return "confirmed_dynamic" + if ( + rec.get("needs_backend_validation") is True + or "backend" in text + or "server" in text + ): + return "needs_backend_validation" + if "route" in text or "deeplink" in text or "deep link" in text: + return "needs_route_map_validation" + if "generic" in text or "library" in text: + return "generic_library_noise" + if rec.get("evidence") and rec.get("source") and rec.get("sink"): + return "strong_static_chain" + return "hardening_only" + + +def infer_validation_tier(rec: dict[str, Any]) -> str: + text = " ".join( + listify(rec.get("validation_plan")) + + [str(rec.get("impact") or ""), str(rec.get("trust_boundary") or "")] + ).lower() + if "production" in text or "prod" in text: + return "tier3_explicit_production_authorization" + if "test account" in text or "qa" in text or "backend" in text or "server" in text: + return "tier2_test_account_or_qa_backend" + if "adb" in text or "device" in text or "emulator" in text: + return "tier1_local_device_no_live_backend" + return "tier0_static_only" + + +def infer_risk(impact: str) -> str: + impact_l = impact.lower() + if any( + x in impact_l + for x in [ + "account takeover", + "auth bypass", + "token theft", + "password reset", + "privilege escalation", + ] + ): + return "high" + if any( + x in impact_l + for x in ["data exposure", "file disclosure", "private component", "pii"] + ): + return "medium" + return "unknown" + + +def dedupe(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + chosen: dict[str, dict[str, Any]] = {} + for rec in records: + key = rec["dedupe_key"] + old = chosen.get(key) + if old is None or score(rec) > score(old): + chosen[key] = rec + return list(chosen.values()) + + +def score(rec: dict[str, Any]) -> tuple[int, int, int]: + return ( + RISK_ORDER.get(rec.get("risk", "unknown"), 0), + CONF_ORDER.get(rec.get("confidence", "unknown"), 0), + -len(rec.get("missing_fields", [])), + ) + + +def write_jsonl(records: list[dict[str, Any]], out: Path | None) -> None: + data = "".join(json.dumps(r, sort_keys=True) + "\n" for r in records) + (out.write_text(data) if out else sys.stdout.write(data)) + + +def write_csv(records: list[dict[str, Any]], out: Path | None) -> None: + fields = [ + "risk", + "confidence", + "confidence_tier", + "validation_tier", + "title", + "package", + "apk", + "class", + "masvs", + "cwe", + "maswe", + "entrypoint", + "source", + "sink", + "impact", + "scanner_gap", + "missing_evidence", + "missing_fields", + ] + fh = out.open("w", newline="") if out else sys.stdout + try: + writer = csv.DictWriter(fh, fieldnames=fields, extrasaction="ignore") + writer.writeheader() + for r in records: + row = dict(r) + row["missing_fields"] = ",".join(r.get("missing_fields", [])) + row["missing_evidence"] = ",".join(r.get("missing_evidence", [])) + row["masvs"] = ",".join(r.get("masvs", [])) + row["cwe"] = ",".join(r.get("cwe", [])) + row["maswe"] = ",".join(r.get("maswe", [])) + writer.writerow(row) + finally: + if out: + fh.close() + + +def write_markdown(records: list[dict[str, Any]], out: Path | None) -> None: + lines = [ + "# Android semantic vulnerability hypotheses", + "", + f"Total deduplicated findings: {len(records)}", + "", + ] + for i, r in enumerate(records, start=1): + lines.append(f"## {i}. {r.get('title') or 'Untitled finding'}") + lines.append("") + lines.append( + f"- Risk / confidence: **{r.get('risk')} / {r.get('confidence')}**" + ) + lines.append(f"- Confidence tier: `{r.get('confidence_tier')}`") + lines.append(f"- Validation tier: `{r.get('validation_tier')}`") + lines.append(f"- APK/package: `{r.get('apk')}` / `{r.get('package')}`") + lines.append(f"- MASVS: {', '.join(r.get('masvs') or []) or 'unmapped'}") + lines.append(f"- CWE: {', '.join(r.get('cwe') or []) or 'unmapped'}") + lines.append(f"- MASWE: {', '.join(r.get('maswe') or []) or 'unmapped'}") + lines.append(f"- Entrypoint: `{r.get('entrypoint')}`") + lines.append(f"- Source: `{r.get('source')}`") + lines.append(f"- Trust boundary: {r.get('trust_boundary')}") + lines.append(f"- Sink: `{r.get('sink')}`") + lines.append(f"- Impact: {r.get('impact')}") + if r.get("scanner_gap"): + lines.append(f"- Scanner gap: {r.get('scanner_gap')}") + if r.get("missing_fields"): + lines.append(f"- Missing fields: {', '.join(r.get('missing_fields'))}") + lines.append("") + if r.get("missing_evidence"): + lines.append("Missing evidence:") + for e in r.get("missing_evidence", []): + lines.append(f"- {e}") + lines.append("") + lines.append("Evidence:") + for e in r.get("evidence", []): + lines.append(f"- {e}") + lines.append("") + lines.append("Validation plan:") + for step in r.get("validation_plan", []): + lines.append(f"- {step}") + lines.append("") + data = "\n".join(lines) + (out.write_text(data) if out else sys.stdout.write(data)) + + +def main() -> int: + ap = argparse.ArgumentParser( + description="Normalize Android semantic vulnerability hypotheses" + ) + ap.add_argument( + "inputs", + nargs="+", + type=Path, + help="JSON or JSONL files containing finding hypotheses", + ) + ap.add_argument( + "--format", choices=["jsonl", "csv", "markdown"], default="markdown" + ) + ap.add_argument("--out", type=Path) + args = ap.parse_args() + records = [normalize(r) for r in load_records(args.inputs)] + records = dedupe(records) + records.sort(key=score, reverse=True) + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + if args.format == "jsonl": + write_jsonl(records, args.out) + elif args.format == "csv": + write_csv(records, args.out) + else: + write_markdown(records, args.out) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/capabilities/android-apk-research/scripts/protector_detect.py b/capabilities/android-apk-research/scripts/protector_detect.py new file mode 100644 index 0000000..8625e07 --- /dev/null +++ b/capabilities/android-apk-research/scripts/protector_detect.py @@ -0,0 +1,626 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +"""Detect Android protectors (DexProtector and Promon Shield today). + +Tier-1 of the protector-triage capability. Given an APK or an inventory +artifact tree produced by run_corpus_inventory, emit a normalized +`protector.json` summarizing protector signals and recommending a +triage strategy. + +DexProtector signals (any one is enough; multiple raise confidence): + - manifest -- the bootstrap + class injected next to the original Application class + - presence of libdpboot.so + libdexprotector.so (or libdexprotector_h.so) + in any lib// + - `DPLF` magic inside libdexprotector.so (start of the packed payload + in modern versions) OR a last PT_LOAD whose contents start with DPLF + - the protected-asset filenames the post calls out by name: + se.dat, classes.dex.dat, mm.dat, dp.mp3, resources.dat, + ic.dat, ct.dat, rcdb.dat, dp.arm-*.so.dat + +Promon Shield signals: + - APKiD packer hit for "Promon Shield" on a native library + - a native library named libshield.so or random-looking lib[a-z]{10,12}.so + with Promon ELF sections + - at least two of the ELF sections .ncu, .ncc, .ncd in the same library + - historical encrypted config assets: config-encrypt.txt, mappings.bin, pbi.bin + +This script is intentionally cheap: it never parses the manifest beyond +a string match and never executes any of the decoded payload. It's the +fast pre-filter before the heavier unpacker (dexprotector_unpack.py) runs. +""" + +from __future__ import annotations + +import argparse +import io +import json +import re +import struct +import zipfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +PROTECTED_ASSETS = ( + "se.dat", + "classes.dex.dat", + "mm.dat", + "dp.mp3", + "resources.dat", + "ic.dat", + "ct.dat", + "rcdb.dat", + "ict.dat", +) + +NATIVE_LIB_NAMES = ( + "libdpboot.so", + "libdexprotector.so", + "libdexprotector_h.so", + "libdp.so", +) + +PROMON_HISTORICAL_ASSETS = ("config-encrypt.txt", "mappings.bin", "pbi.bin") +PROMON_SECTION_NAMES = (".ncu", ".ncc", ".ncd") +PROMON_RANDOM_LIB_PATTERN = re.compile(r"^lib[a-z]{10,12}\.so$") + +# Permissive: matches "Protected", "ProtectedFoo", "ProtectedLiveNetTV", but +# not "Foo.Protected" lookalikes elsewhere in the manifest binary. +PROTECTED_BOOT_PATTERN = re.compile(rb"Protected[A-Za-z0-9_]*") + + +CONFIDENCE_RANK = {"none": 0, "low": 1, "medium": 2, "high": 3} + + +@dataclass +class ProtectorReport: + target: str + package: str | None = None + is_apk: bool = False + protector: str = "unknown" + confidence: str = "none" + signals: dict[str, Any] = field(default_factory=dict) + artifacts: dict[str, Any] = field(default_factory=dict) + triage_strategy: str = "default" + notes: list[str] = field(default_factory=list) + + def to_json(self) -> str: + return json.dumps(self.__dict__, indent=2, sort_keys=True) + + +def _find_dplf_in_so(blob: bytes) -> dict[str, Any] | None: + if b"DPLF" not in blob: + return None + idx = blob.find(b"DPLF") + info: dict[str, Any] = {"dplf_file_offset": idx, "size": len(blob)} + if blob[:4] != b"\x7fELF": + info["host"] = "non-elf" + return info + try: + e_phoff = struct.unpack_from(" float: + if not data: + return 0.0 + counts = [0] * 256 + for b in data: + counts[b] += 1 + entropy = 0.0 + length = len(data) + for c in counts: + if not c: + continue + p = c / length + entropy -= p * __import__("math").log2(p) + return round(entropy, 4) + + +def _parse_elf_sections(blob: bytes) -> dict[str, dict[str, Any]]: + """Return ELF sections keyed by name for 32/64-bit little/big endian ELFs. + + This intentionally parses only the section table fields needed by the + protector detector. It is safe for zipped APK bytes and never executes code. + """ + if len(blob) < 0x34 or blob[:4] != b"\x7fELF": + return {} + elf_class = blob[4] + endian_id = blob[5] + if elf_class not in (1, 2) or endian_id not in (1, 2): + return {} + endian = "<" if endian_id == 1 else ">" + try: + if elf_class == 1: + if len(blob) < 0x34: + return {} + e_shoff = struct.unpack_from(endian + "I", blob, 0x20)[0] + e_shentsize = struct.unpack_from(endian + "H", blob, 0x2E)[0] + e_shnum = struct.unpack_from(endian + "H", blob, 0x30)[0] + e_shstrndx = struct.unpack_from(endian + "H", blob, 0x32)[0] + fmt = endian + "10I" + else: + if len(blob) < 0x40: + return {} + e_shoff = struct.unpack_from(endian + "Q", blob, 0x28)[0] + e_shentsize = struct.unpack_from(endian + "H", blob, 0x3A)[0] + e_shnum = struct.unpack_from(endian + "H", blob, 0x3C)[0] + e_shstrndx = struct.unpack_from(endian + "H", blob, 0x3E)[0] + fmt = endian + "IIQQQQIIQQ" + + if not e_shoff or not e_shentsize or not e_shnum or e_shstrndx >= e_shnum: + return {} + if e_shoff + e_shentsize * e_shnum > len(blob): + return {} + + raw_sections: list[tuple[int, int, int, int, int, int]] = [] + for i in range(e_shnum): + off = e_shoff + i * e_shentsize + fields = struct.unpack_from(fmt, blob, off) + if elf_class == 1: + sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size = fields[:6] + else: + sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size = fields[:6] + raw_sections.append( + (sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size) + ) + + shstr = raw_sections[e_shstrndx] + shstr_off, shstr_size = shstr[4], shstr[5] + if shstr_off + shstr_size > len(blob): + return {} + shstrtab = blob[shstr_off : shstr_off + shstr_size] + + sections: dict[str, dict[str, Any]] = {} + for idx, (sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size) in enumerate( + raw_sections + ): + if sh_name >= len(shstrtab): + name = f"" + else: + end = shstrtab.find(b"\x00", sh_name) + if end == -1: + end = len(shstrtab) + name = shstrtab[sh_name:end].decode("utf-8", errors="replace") + sample = b"" + if sh_size and sh_offset < len(blob): + sample = blob[ + sh_offset : min(len(blob), sh_offset + min(sh_size, 65536)) + ] + sections[name] = { + "index": idx, + "type": sh_type, + "flags": sh_flags, + "addr": sh_addr, + "offset": sh_offset, + "size": sh_size, + "entropy_sample": _shannon_entropy(sample), + } + return sections + except (struct.error, OverflowError): + return {} + + +def _promon_lib_name_kind(base: str) -> str | None: + if base == "libshield.so": + return "libshield" + if base in NATIVE_LIB_NAMES: + return None + if PROMON_RANDOM_LIB_PATTERN.match(base): + return "random_libname" + return None + + +def _extract_apkid_packers(apkid_path: Path) -> dict[str, list[str]]: + if not apkid_path.exists(): + return {} + try: + data = json.loads(apkid_path.read_text()) + except json.JSONDecodeError: + return {} + hits: dict[str, list[str]] = {} + for f in data.get("result", {}).get("files", []): + packers = f.get("matches", {}).get("packer", []) + promon = [p for p in packers if "promon" in p.lower()] + if not promon: + continue + filename = f.get("filename", "") + rel = filename.split("!", 1)[-1] if "!" in filename else filename + hits[rel] = promon + return hits + + +def _merge_promon_apkid_hits( + report: ProtectorReport, hits: dict[str, list[str]] +) -> None: + if not hits: + return + report.signals["apkid_promon"] = hits + libs = report.signals.setdefault("promon_native_libs", {}) + for rel in hits: + if not rel.startswith("lib/"): + continue + parts = rel.split("/", 2) + if len(parts) < 3: + continue + abi = parts[1] + libs.setdefault(abi, []) + if rel not in libs[abi]: + libs[abi].append(rel) + for abi, paths in list(libs.items()): + libs[abi] = sorted(paths) + + +def _apply_dexprotector_report(report: ProtectorReport) -> None: + libs = report.signals.get("native_libs", {}) + abis_present = sorted({n.split("/")[1] for ns in libs.values() for n in ns}) + report.artifacts["abis"] = abis_present + report.artifacts["dexprotector_unpack_supported"] = "arm64-v8a" in abis_present + report.triage_strategy = "protector_aware" + report.notes += [ + "JADX output is incomplete: encrypted classes are loaded from assets/classes.dex.dat at runtime", + "ripgrep/Semgrep on JADX sources will miss the protected first-party code", + "asset-derived flows (config, certs, keystores) are encrypted on disk; trust JADX strings only for unencrypted classes", + "if libdexprotector.so contains a DPLF payload in the arm64 build, scripts/dexprotector_unpack.py can recover libdp.so without an Android device", + "frida hooks against libdp.so trigger the master-key corruption path described in the source post; static unpack avoids this", + ] + + +def _apply_promon_report(report: ProtectorReport) -> None: + libs = report.signals.get("promon_native_libs", {}) + abis_present = sorted( + { + p.split("/")[1] + for paths in libs.values() + for p in paths + if p.startswith("lib/") + } + ) + report.artifacts["abis"] = abis_present + report.artifacts["promon_recovery_supported"] = "research" + report.artifacts["java_string_recovery_supported"] = True + report.artifacts["static_native_unpack_supported"] = False + report.triage_strategy = "protector_aware_native_rasp" + report.notes += [ + "Promon Shield is primarily native RASP/app shielding: do not assume all first-party DEX is encrypted", + "run normal manifest/JADX attack-surface analysis, but expect strings/class bindings and runtime policy to be hidden behind the shield", + "first recovery milestone is smali string/constant recovery, followed by Java/native binding maps and native RASP triage", + "dynamic validation may trigger anti-debug, anti-hook, root/emulator, repackaging, and shield self-integrity checks; establish a clean baseline before instrumentation", + ] + + +def _choose_best_report(reports: list[ProtectorReport]) -> ProtectorReport: + return max(reports, key=lambda r: CONFIDENCE_RANK[r.confidence]) + + +def _merge_reports( + best: ProtectorReport, reports: list[ProtectorReport], target: str +) -> ProtectorReport: + for r in reports: + if r is best: + continue + for k, v in r.signals.items(): + if k not in best.signals: + best.signals[k] = v + elif isinstance(v, dict) and isinstance(best.signals[k], dict): + merged = dict(best.signals[k]) + for subk, subv in v.items(): + if subk not in merged: + merged[subk] = subv + elif isinstance(subv, list) and isinstance(merged[subk], list): + merged[subk] = sorted(set(merged[subk] + subv)) + elif isinstance(subv, dict) and isinstance(merged[subk], dict): + merged[subk] = {**merged[subk], **subv} + else: + merged[subk] = subv + best.signals[k] = merged + elif isinstance(v, list) and isinstance(best.signals[k], list): + best.signals[k] = sorted(set(best.signals[k] + v)) + best.target = target + return best + + +def _scan_apk(path: Path) -> ProtectorReport: + report = ProtectorReport(target=str(path), is_apk=True) + with zipfile.ZipFile(path) as z: + names = z.namelist() + + # manifest probe (do not parse binary AXML; just match string) + try: + am = z.read("AndroidManifest.xml") + except KeyError: + am = b"" + m = PROTECTED_BOOT_PATTERN.search(am) + if m: + report.signals["protected_application_class"] = m.group(0).decode( + "ascii", errors="replace" + ) + + # native libs + found_libs: dict[str, list[str]] = {} + for n in names: + if not n.startswith("lib/"): + continue + base = n.rsplit("/", 1)[-1] + if base in NATIVE_LIB_NAMES: + found_libs.setdefault(base, []).append(n) + if found_libs: + report.signals["native_libs"] = found_libs + + # assets + protected_assets: list[str] = [] + for n in names: + if not n.startswith("assets/"): + continue + base = n.rsplit("/", 1)[-1] + if base in PROTECTED_ASSETS: + protected_assets.append(n) + elif base.startswith("dp.arm-") and base.endswith(".so.dat"): + protected_assets.append(n) + if protected_assets: + report.signals["protected_assets"] = protected_assets + + # DPLF watermark scan inside libdexprotector.so (arm64 first, then any) + dplf_info: dict[str, Any] = {} + for abi_pref in ("arm64-v8a", "armeabi-v7a", "x86_64", "x86"): + n = f"lib/{abi_pref}/libdexprotector.so" + if n in names: + blob = z.read(n) + info = _find_dplf_in_so(blob) + if info: + dplf_info[abi_pref] = info + break + if dplf_info: + report.signals["dplf"] = dplf_info + + # Promon Shield: random/libshield native library plus protected ELF + # sections. Scan all native libs because Promon often renames libshield. + promon_libs: dict[str, list[str]] = {} + promon_name_hints: dict[str, str] = {} + promon_sections: dict[str, dict[str, Any]] = {} + for n in names: + if not (n.startswith("lib/") and n.endswith(".so")): + continue + base = n.rsplit("/", 1)[-1] + name_kind = _promon_lib_name_kind(base) + if name_kind: + promon_name_hints[n] = name_kind + # Section parsing is cheap enough for native libs and gives the + # strongest Promon signal. Avoid reading non-candidates only if a + # name hint is absent? No: APKiD has a section-only promon_a rule. + blob = z.read(n) + sections = _parse_elf_sections(blob) + section_hits = { + s: sections[s] for s in PROMON_SECTION_NAMES if s in sections + } + if section_hits: + promon_sections[n] = section_hits + # A random-looking lib[a-z]{10,12}.so name alone is too noisy in + # modern APKs (e.g. React Native's libjscexecutor.so). Treat + # libshield.so as a candidate by name, but require Promon section + # pairs for random names. + if name_kind == "libshield" or len(section_hits) >= 2: + abi = n.split("/", 2)[1] + promon_libs.setdefault(abi, []).append(n) + if promon_libs: + report.signals["promon_native_libs"] = { + abi: sorted(paths) for abi, paths in promon_libs.items() + } + if promon_name_hints: + candidate_paths = {p for paths in promon_libs.values() for p in paths} + report.signals["promon_name_hints"] = { + p: kind + for p, kind in promon_name_hints.items() + if p in candidate_paths or kind == "libshield" + } + if promon_sections: + report.signals["promon_elf_sections"] = promon_sections + + promon_assets = [] + for n in names: + if not n.startswith("assets/"): + continue + base = n.rsplit("/", 1)[-1] + if base in PROMON_HISTORICAL_ASSETS: + promon_assets.append(n) + if promon_assets: + report.signals["promon_assets"] = sorted(promon_assets) + + apkid_sidecar = path.with_suffix(".apkid.json") + _merge_promon_apkid_hits(report, _extract_apkid_packers(apkid_sidecar)) + + dex_score = sum( + bool(report.signals.get(k)) + for k in ( + "protected_application_class", + "native_libs", + "protected_assets", + "dplf", + ) + ) + + promon_section_pair = any( + len(section_hits) >= 2 + for section_hits in report.signals.get("promon_elf_sections", {}).values() + ) + promon_apkid = bool(report.signals.get("apkid_promon")) + promon_has_lib = bool(report.signals.get("promon_native_libs")) + promon_has_assets = bool(report.signals.get("promon_assets")) + if promon_apkid or promon_section_pair: + promon_confidence = "high" + elif promon_has_lib and promon_has_assets: + promon_confidence = "medium" + elif promon_has_lib or promon_has_assets: + promon_confidence = "low" + else: + promon_confidence = "none" + + dex_confidence = "none" + if dex_score >= 3: + dex_confidence = "high" + elif dex_score >= 1: + dex_confidence = "medium" if dex_score == 2 else "low" + + if CONFIDENCE_RANK[promon_confidence] > CONFIDENCE_RANK[dex_confidence]: + report.protector = "promon_shield" + report.confidence = promon_confidence + elif dex_confidence != "none": + report.protector = "dexprotector" + report.confidence = dex_confidence + + if report.protector == "dexprotector": + _apply_dexprotector_report(report) + elif report.protector == "promon_shield": + _apply_promon_report(report) + + return report + + +def _scan_inventory_dir(path: Path) -> ProtectorReport: + """Use an existing run_corpus_inventory artifact directory. + + Expected layout: + /androguard.json + /inventory.json (optional) + """ + androguard = path / "androguard.json" + if not androguard.exists(): + raise SystemExit(f"no androguard.json under {path}") + data = json.loads(androguard.read_text()) + report = ProtectorReport(target=str(path), is_apk=False) + pkg = data.get("package") + report.package = pkg + app_class = data.get("application_class") or "" + apkid_hits = _extract_apkid_packers(path / "apkid.json") + promon_hits = { + k: v for k, v in apkid_hits.items() if any("promon" in p.lower() for p in v) + } + _merge_promon_apkid_hits(report, promon_hits) + inv = path / "inventory.json" + if inv.exists(): + try: + inv_data = json.loads(inv.read_text()) + except json.JSONDecodeError: + inv_data = {} + native_libs = inv_data.get("native_libs") or [] + promon_libs: dict[str, list[str]] = {} + promon_name_hints: dict[str, str] = {} + for n in native_libs: + if not isinstance(n, str) or not n.startswith("lib/"): + continue + base = n.rsplit("/", 1)[-1] + kind = _promon_lib_name_kind(base) + if kind != "libshield" and n not in promon_hits: + continue + if kind: + promon_name_hints[n] = kind + abi = n.split("/", 2)[1] + promon_libs.setdefault(abi, []).append(n) + if promon_libs: + libs = report.signals.setdefault("promon_native_libs", {}) + for abi, paths in promon_libs.items(): + libs.setdefault(abi, []) + libs[abi] = sorted(set(libs[abi] + paths)) + if promon_name_hints: + report.signals["promon_name_hints"] = promon_name_hints + if "Protected" in app_class: + report.signals["protected_application_class"] = app_class + # androguard.json from the existing inventory should already list files. + if promon_hits: + report.protector = "promon_shield" + report.confidence = "high" + _apply_promon_report(report) + report.notes.append( + "inventory-only Promon scan used APKiD/native-lib metadata; re-run with the APK path for ELF section entropy and asset evidence" + ) + elif app_class and "Protected" in app_class: + report.protector = "dexprotector" + report.confidence = "low" + report.notes.append( + "inventory-only scan; re-run with the APK path for native-lib / DPLF / asset evidence" + ) + _apply_dexprotector_report(report) + return report + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument( + "input", + type=Path, + help="APK or directory produced by run_corpus_inventory (apks//)", + ) + ap.add_argument( + "-o", + "--output", + type=Path, + default=None, + help="path to write protector.json (default: alongside input)", + ) + args = ap.parse_args() + + if args.input.is_dir(): + report = _scan_inventory_dir(args.input) + default_out = args.input / "protector.json" + elif args.input.suffix.lower() in {".apk", ".xapk"}: + if args.input.suffix.lower() == ".xapk": + # XAPK: scan every embedded APK and merge signals + reports: list[ProtectorReport] = [] + tmpdir = args.input.with_suffix(".__xapk") + tmpdir.mkdir(exist_ok=True) + with zipfile.ZipFile(args.input) as z: + for n in z.namelist(): + if not n.lower().endswith(".apk"): + continue + blob = z.read(n) + tmp = tmpdir / n.replace("/", "_") + tmp.write_bytes(blob) + reports.append(_scan_apk(tmp)) + tmp.unlink() + tmpdir.rmdir() + if not reports: + raise SystemExit("no APKs inside XAPK") + report = _merge_reports( + _choose_best_report(reports), reports, str(args.input) + ) + else: + report = _scan_apk(args.input) + default_out = args.input.with_suffix(".protector.json") + else: + raise SystemExit("input must be an APK/XAPK or inventory directory") + + out = args.output or default_out + out.write_text(report.to_json()) + print(report.to_json()) + + +if __name__ == "__main__": + main() diff --git a/capabilities/android-apk-research/scripts/rank_backend_richness.py b/capabilities/android-apk-research/scripts/rank_backend_richness.py new file mode 100644 index 0000000..19a14e8 --- /dev/null +++ b/capabilities/android-apk-research/scripts/rank_backend_richness.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Rank APK/source targets by backend richness summaries. + +Accepts one or more `backend_richness.json` files from extract_api_map.py and +emits a sorted JSONL/Markdown inbox. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +def load_summary(path: Path) -> dict[str, Any] | None: + try: + obj = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + obj["summary_path"] = str(path) + # Best-effort package/target inference from parent dir. + obj.setdefault("target", path.parent.name) + return obj + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument( + "summaries", nargs="+", type=Path, help="backend_richness.json files" + ) + ap.add_argument("--out-jsonl", type=Path, required=True) + ap.add_argument("--out-md", type=Path, default=None) + args = ap.parse_args() + + rows = [s for p in args.summaries if (s := load_summary(p))] + rows.sort(key=lambda r: (-int(r.get("total_score") or 0), r.get("target") or "")) + + args.out_jsonl.parent.mkdir(parents=True, exist_ok=True) + with args.out_jsonl.open("w") as f: + for r in rows: + f.write(json.dumps(r, sort_keys=True) + "\n") + + if args.out_md: + args.out_md.parent.mkdir(parents=True, exist_ok=True) + lines = ["# Backend-rich APK ranking\n"] + lines.append( + f"Ranked {len(rows)} targets by `extract_api_map.py` summary score.\n" + ) + for i, r in enumerate(rows, 1): + lines.append( + f"{i}. **{r.get('target')}** — score={r.get('total_score')} " + f"richness={r.get('backend_richness')} rows={r.get('row_count')} " + f"summary=`{r.get('summary_path')}`" + ) + counts = r.get("unique_value_counts") or {} + if counts: + compact = ", ".join(f"{k}:{v}" for k, v in sorted(counts.items())) + lines.append(f" - unique: {compact}") + flags = r.get("synergy_flags") or {} + hot = [k for k, v in sorted(flags.items()) if v] + if hot: + lines.append(f" - synergy: {', '.join(hot)}") + args.out_md.write_text("\n".join(lines) + "\n") + + print(f"wrote {len(rows)} rows to {args.out_jsonl}") + if args.out_md: + print(f"wrote markdown to {args.out_md}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/capabilities/android-apk-research/scripts/rank_components.py b/capabilities/android-apk-research/scripts/rank_components.py new file mode 100755 index 0000000..12e671f --- /dev/null +++ b/capabilities/android-apk-research/scripts/rank_components.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +"""Score and rank components from `extract_corpus_components.py` output. + +Applies risk priors to each component row and emits: + - components_ranked.jsonl (full schema + score + reasons + read-budget tag) + - components_ranked.md (operator inbox — top N grouped by app) + +Priors (tuned 2026-05-18 against corpus-2; revise as more empirics come in): + + +5 exported + no permission + BROWSABLE + +3 host wildcard `*` or empty/missing + +3 path/name matches "(join|accept|invite|redirect|callback|sso|oauth| + mpless|transfer|register|complete|recover|magic| + consume|reset|enroll|claim|activate|verify|next| + return|continue|target|intent|router|proxy|dispatch| + open|import|share)" + +3 action matches "(IMPORT_|EXPORT_|RECOVER_|MIGRATE_|RESET_)" + or scheme in {otpauth, smsto, smsmms} + +3 exported/no-perm share/import target (SEND, SEND_MULTIPLE, GET_CONTENT, + OPEN_DOCUMENT, PICK) + +2 broad MIME share/import target (*/*, application/octet-stream) + +2 exported AutofillService / AccessibilityService / CredentialProviderService + +2 exported ContentProvider with grantUriPermissions=true + +2 exported BROADCAST receiver with no perm + action not in {INSTALL_REFERRER, + BOOT_COMPLETED} + +1 scheme in {http, https} (App Link surface) with non-wildcard host + (lower than wildcard host but worth surfacing) + + Corpus-3 adjustments (2026-05-19): + + +2 router-shape component name (Router/Dispatcher/Resolver/DeepLink/AppLink/ + NavHandler) with >=3 distinct schemes and >=3 distinct hosts. Empirically + this is the *current* high-impact deep-link shape after the post-MSRC-2024 + Dirty Stream fix campaign. Examples surfaced in corpus-3: Robinhood + DeeplinkResolverActivity, Chase DeepLinkHandlerActivity, Mint RouterActivity, + Citi SplashScreenActivity, Teams SplashActivity. + +1 runtime_kind in hybrid set (react_native_js, react_native_hermes, flutter_aot, + capacitor, cordova). Hybrid surfaces have an extra JS/Dart trust boundary + that scanners can't reach, so worth promoting these in the inbox to offset + the JS/Dart follow-up cost. + -2 file/content scheme share/import target on a component file with <=80 lines + (delegating stub). Empirically the Dirty-Stream-shaped top tier is dominated + by short subclasses that immediately delegate to a base; only the base class + has any real logic, so the manifest-only rank over-weights every subclass. + NOTE: requires --src to be set; otherwise the rule is silent. + -1 receiver/activity name matches "(Splash|Launcher|Receiver)$" with no scheme + and no high-risk action — these score 7-9 on host_wildcard alone but rarely + lead anywhere. + -2 permission with protectionLevel=normal + -5 permission with protectionLevel=signature (defanged unless attacker has same sig) + -12 apkid_tier=heavy + -1 declared MAIN/LAUNCHER only and nothing else + -1 component name matches "Hilt_|_GeneratedInjector|_MembersInjector|Dagger|_Factory$" + +Read-budget tag (informational): + read_budget = "5m" if score >= 7 + = "1m" if score in [3, 6] + = "skip" if score < 3 + +Usage: + python3 rank_components.py \ + --components findings/corpus-2/triage/components.jsonl \ + --out-jsonl findings/corpus-2/triage/components_ranked.jsonl \ + --out-md findings/corpus-2/triage/components_ranked.md \ + --top-md 100 +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any + +HIGH_RISK_PATH_RE = re.compile( + r"(join|accept|invite|redirect|callback|sso|oauth|mpless|transfer|register|" + r"complete|recover|magic|consume|reset|enroll|claim|activate|verify|next|" + r"return|continue|target|intent|router|proxy|dispatch|open|import|share)", + re.IGNORECASE, +) +HIGH_RISK_ACTION_RE = re.compile(r"(IMPORT_|EXPORT_|RECOVER_|MIGRATE_|RESET_)") +HIGH_RISK_SCHEMES = {"otpauth", "smsto", "smsmms"} +ROUTER_NAME_RE = re.compile( + r"(Router|Dispatcher|Resolver|DeepLink|AppLink|NavHandler|UriHandler|" + r"IntentHandler|LinkRedirect|RedirectUri)", + re.IGNORECASE, +) +HYBRID_RUNTIME_KINDS = { + "react_native_js", + "react_native_hermes", + "flutter_aot", + "capacitor", + "cordova", +} +LOW_SIGNAL_TAIL_NAME_RE = re.compile(r"(Splash|Launcher|Receiver)Activity?$") +SHARE_IMPORT_ACTIONS = { + "android.intent.action.SEND", + "android.intent.action.SEND_MULTIPLE", + "android.intent.action.GET_CONTENT", + "android.intent.action.OPEN_DOCUMENT", + "android.intent.action.PICK", +} +BROAD_MIME_TYPES = {"*/*", "application/octet-stream"} +GENERATED_NAME_RE = re.compile( + r"(Hilt_|_GeneratedInjector|_MembersInjector|Dagger|_Factory$|_Provide|Hilt$)" +) +BENIGN_RECEIVER_ACTIONS = { + "com.android.vending.INSTALL_REFERRER", + "android.intent.action.BOOT_COMPLETED", + "android.intent.action.MY_PACKAGE_REPLACED", + "android.intent.action.PACKAGE_REPLACED", +} +AUTOFILL_LIKE_SERVICE_ACTIONS = { + "android.service.autofill.AutofillService", + "android.service.credentials.CredentialProviderService", + "android.accessibilityservice.AccessibilityService", +} + + +def host_wildcard(hosts: list[str]) -> bool: + if not hosts: + return True # implicit any-host + return any(h in ("", "*") for h in hosts) + + +def score_component(row: dict[str, Any]) -> tuple[int, list[str]]: + score = 0 + reasons: list[str] = [] + if row.get("exported") is False: + # Definitely-internal — skip unless we want to surface for completeness. + return 0, ["not_exported"] + + exported = row.get("exported") is True or row.get("browsable") is True + perm = row.get("permission") or "" + perm_protection = (row.get("perm_protection") or "").lower() + browsable = bool(row.get("browsable")) + actions = row.get("actions") or [] + schemes = row.get("schemes") or [] + hosts = row.get("hosts") or [] + paths = row.get("paths") or [] + typ = row.get("type") + name = row.get("name") or "" + + if exported and not perm and browsable: + score += 5 + reasons.append("exported_browsable_no_perm") + + if browsable and host_wildcard(hosts): + score += 3 + reasons.append("host_wildcard") + + if any(HIGH_RISK_PATH_RE.search(p) for p in paths): + score += 3 + reasons.append("high_risk_path") + + if exported and not perm and HIGH_RISK_PATH_RE.search(name): + score += 2 + reasons.append("high_risk_component_name") + + if any(HIGH_RISK_ACTION_RE.search(a) for a in actions): + score += 3 + reasons.append("high_risk_action") + if any(s in HIGH_RISK_SCHEMES for s in schemes): + score += 3 + reasons.append("high_risk_scheme") + + if exported and not perm and any(a in SHARE_IMPORT_ACTIONS for a in actions): + score += 3 + reasons.append("share_import_target_no_perm") + + mime_types = row.get("mime_types") or [] + if exported and any(m in BROAD_MIME_TYPES or m.endswith("/*") for m in mime_types): + score += 2 + reasons.append("broad_mime_share_import") + + if typ == "service" and any(a in AUTOFILL_LIKE_SERVICE_ACTIONS for a in actions): + score += 2 + reasons.append("auth_critical_service") + + if typ == "provider" and ( + row.get("grant_uri") in (True, "true", "0xffffffff", "-1") + ): + score += 2 + reasons.append("provider_grant_uri") + + if ( + typ == "receiver" + and exported + and not perm + and actions + and not any(a in BENIGN_RECEIVER_ACTIONS for a in actions) + ): + # Custom action on an exported receiver with no perm is a smell. + score += 2 + reasons.append("exported_custom_receiver_no_perm") + + if ( + browsable + and any(s in ("http", "https") for s in schemes) + and not host_wildcard(hosts) + ): + score += 1 + reasons.append("app_link_surface") + + # Corpus-3: reward concentrated deep-link routers (the post-MSRC-2024 bug shape). + if ( + ROUTER_NAME_RE.search(name) + and len({s for s in schemes if s}) >= 3 + and len({h for h in hosts if h and h != "*"}) >= 3 + ): + score += 2 + reasons.append("router_shape_multi_scheme_host") + + # Corpus-3: hybrid runtimes have an extra JS/Dart trust boundary scanners miss. + if row.get("runtime_kind") in HYBRID_RUNTIME_KINDS: + score += 1 + reasons.append(f"hybrid_runtime:{row['runtime_kind']}") + + # Corpus-3: low-signal Splash/Launcher/Receiver tail without scheme+action specifics + # is rarely a real chain; penalize so they don't crowd out router hits. + if ( + LOW_SIGNAL_TAIL_NAME_RE.search(name) + and not schemes + and not any(HIGH_RISK_ACTION_RE.search(a) for a in actions) + ): + score -= 1 + reasons.append("low_signal_tail_name") + + if perm_protection == "normal": + score -= 2 + reasons.append("perm_normal") + elif perm_protection == "signature" or perm_protection == "0x2": + score -= 5 + reasons.append("perm_signature") + elif perm_protection in ("signatureOrSystem", "0x3", "signatureorsystem"): + score -= 4 + reasons.append("perm_signature_or_system") + + if row.get("apkid_tier") == "heavy": + score -= 12 + reasons.append("apkid_heavy_packer") + elif row.get("apkid_tier") == "medium": + score -= 3 + reasons.append("apkid_medium_packer") + elif row.get("apkid_tier") == "ambiguous": + score -= 1 + reasons.append("apkid_ambiguous_packer") + + # MAIN/LAUNCHER-only is launcher boilerplate + if actions == [ + "android.intent.action.MAIN" + ] and "android.intent.category.LAUNCHER" in (row.get("categories") or []): + score -= 1 + reasons.append("launcher_only") + + if GENERATED_NAME_RE.search(name): + score -= 1 + reasons.append("generated_class") + + return score, reasons + + +def read_budget(score: int) -> str: + if score >= 7: + return "5m" + if score >= 3: + return "1m" + return "skip" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--components", required=True, type=Path) + ap.add_argument("--out-jsonl", required=True, type=Path) + ap.add_argument("--out-md", required=True, type=Path) + ap.add_argument("--top-md", type=int, default=100) + ap.add_argument( + "--min-score", + type=int, + default=3, + help="suppress rows below this score from the markdown view (jsonl keeps all)", + ) + args = ap.parse_args() + + rows: list[dict[str, Any]] = [] + for line in args.components.read_text().splitlines(): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + score, reasons = score_component(row) + row["score"] = score + row["score_reasons"] = reasons + row["read_budget"] = read_budget(score) + rows.append(row) + + rows.sort(key=lambda r: (-r["score"], r.get("package") or "", r.get("name") or "")) + + args.out_jsonl.parent.mkdir(parents=True, exist_ok=True) + with args.out_jsonl.open("w") as f: + for r in rows: + f.write(json.dumps(r, sort_keys=True) + "\n") + + # Markdown view: keep score >= min-score, cap at top-md, group by package. + eligible = [r for r in rows if r["score"] >= args.min_score][: args.top_md] + by_pkg: dict[str, list[dict[str, Any]]] = defaultdict(list) + for r in eligible: + by_pkg[r.get("package") or "(unknown)"].append(r) + + lines: list[str] = [] + lines.append("# Corpus components — ranked\n") + lines.append( + f"Total components scored: {len(rows)} (across {len({r['apk_sha256'] for r in rows})} APKs). " + f"Showing top {len(eligible)} with score >= {args.min_score}.\n" + ) + lines.append( + "Score legend: see `rank_components.py` docstring. Read-budget: " + "`5m` = full deep-read; `1m` = grep + skim; `skip` = below threshold.\n" + ) + lines.append("## Top entries by impact class\n") + by_class: dict[str, list[dict[str, Any]]] = defaultdict(list) + for r in eligible: + by_class[r.get("impact_class") or "(unclassified)"].append(r) + for cls in sorted(by_class): + rs = by_class[cls] + lines.append(f"\n### {cls} — {len(rs)} components in top-{len(eligible)}\n") + for r in rs: + lines.append( + f"- **{r['score']}** `{r.get('package')}` / {r.get('type')} " + f"`{r.get('name')}` — " + f"budget={r['read_budget']} " + f"reasons=[{', '.join(r['score_reasons'])}]" + ) + sc = r.get("schemes") or [] + ho = r.get("hosts") or [] + pa = r.get("paths") or [] + ac = [ + a + for a in (r.get("actions") or []) + if a + not in ( + "android.intent.action.VIEW", + "android.intent.action.MAIN", + ) + ] + if sc: + lines.append(f" - schemes: {sc}") + if ho: + lines.append(f" - hosts: {ho}") + if pa: + lines.append(f" - paths: {pa[:10]}") + if ac: + lines.append(f" - actions: {ac[:5]}") + + lines.append("\n## Full ranked list grouped by package\n") + for pkg in sorted(by_pkg): + rs = by_pkg[pkg] + top_score = max(r["score"] for r in rs) + lines.append(f"\n### {pkg} (top score {top_score})\n") + for r in rs: + lines.append( + f"- **{r['score']}** {r.get('type')} `{r.get('name')}` " + f"reasons=[{', '.join(r['score_reasons'])}]" + ) + + args.out_md.parent.mkdir(parents=True, exist_ok=True) + args.out_md.write_text("\n".join(lines) + "\n") + print(f"wrote {len(rows)} rows to {args.out_jsonl}") + print(f"wrote top-{len(eligible)} markdown view to {args.out_md}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/capabilities/android-apk-research/scripts/research/promon/README.md b/capabilities/android-apk-research/scripts/research/promon/README.md new file mode 100644 index 0000000..8825a3d --- /dev/null +++ b/capabilities/android-apk-research/scripts/research/promon/README.md @@ -0,0 +1,25 @@ +# Promon Shield static recovery (research-grade) + +These scripts are **not wired into the routine `android-protector-triage` flow**. They are first-pass static evaluators of Promon Shield's bootstrap, smali string indirection, native ELF layout, and Java/native binding callsites. Validation across more than the one anchor sample (`in.org.npci.upiapp`) is still pending. + +Use them only when the operator has explicitly chosen the Promon research path. The skill workflow's mainline detection lives in [`scripts/protector_detect.py`](../../protector_detect.py); these scripts pick up *after* detection confirms a Promon-shielded APK. + +Runbook (mirrors `skills/android-protector-triage/references/promon-shield.md`): + +```bash +python3 capabilities/android-apk-research/scripts/research/promon/promon_recover.py path/to.apk -o findings//promon +``` + +Outputs are documented in the reference. Treat all findings as `scanner_gap = adjacent` or `not found` — do not promote to `strong_static_chain` based on these artifacts alone. + +## Files + +- `promon_recover.py` — orchestrator (detect → optional apktool decode → string recovery → ELF triage → roll-up). +- `promon_string_recover.py` — smali char-array / `String.intern()` deobfuscator. +- `promon_java_triage.py` — Java/smali integration map (native methods, load-library callsites, framework hints). +- `promon_binding_triage.py` — focused extraction of native string/class binding IDs. +- `promon_elf_triage.py` — APK/ELF profiler (section metadata, entropy, `.init_array`, RASP strings, AArch64 `SVC #0` sites). + +## Why not in `scripts/` proper? + +These are evaluator-grade, not production-grade — moving to `scripts/research/promon/` keeps the mainline `scripts/` directory aligned with the MCP tool surface and the skill workflow. When a script here graduates (validated across multiple samples, wired into an MCP tool), promote it to `scripts/` and update the references. diff --git a/capabilities/android-apk-research/scripts/research/promon/promon_binding_triage.py b/capabilities/android-apk-research/scripts/research/promon/promon_binding_triage.py new file mode 100755 index 0000000..b51a9c2 --- /dev/null +++ b/capabilities/android-apk-research/scripts/research/promon/promon_binding_triage.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 +"""Extract Promon Java/native binding callsites from smali. + +This script specializes the Java triage result into a binding map for methods +such as: + Lgms/e;->a(I)Ljava/lang/String; + Lgms/e;->a(Ljava/lang/Class;I)V + +It does not recover plaintext values. It maps hidden IDs to callsites and +immediate use context so the next recovery step can target the right native +lookup table or dynamic dump points. +""" + +from __future__ import annotations + +import argparse +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +CLASS_RE = re.compile(r"^\.class\s+.*?\s+(L[^;]+;)") +METHOD_RE = re.compile(r"^\.method\s+.*?\s+([^\s]+)$") +INVOKE_RE = re.compile( + r"^(?Pinvoke-\S+)\s+\{(?P[^}]*)\},\s+(?P\S+;->\S+)$" +) +CONST_RE = re.compile( + r"^const(?:/\S+)?\s+(?P[vp]\d+),\s+(?P[-+]?0x[0-9a-fA-F]+|[-+]?\d+)" +) +CONST_CLASS_RE = re.compile(r"^const-class\s+(?P[vp]\d+),\s+(?PL[^;]+;)") +MOVE_RESULT_RE = re.compile(r"^move-result-object\s+(?P[vp]\d+)") + +DEFAULT_STRING_TARGET_RE = "" +DEFAULT_CLASS_TARGET_RE = "" +METHOD_RE_FULL = re.compile(r"^\.method\s+(?P.+)$") +NATIVE_DECL_RE = re.compile(r"(?P[^\s(]+)\((?P[^)]*)\)(?P\S+)$") + +SINK_PATTERNS = [ + ( + "url_or_uri", + [ + "Ljava/net/URL;", + "Ljava/net/URI;", + "Landroid/net/Uri;", + "parse(Ljava/lang/String;)", + ], + ), + ( + "webview", + [ + "Landroid/webkit/WebView;", + "loadUrl", + "evaluateJavascript", + "addJavascriptInterface", + ], + ), + ("intent", ["Landroid/content/Intent;", "setAction", "putExtra", "getStringExtra"]), + ("json", ["Lorg/json/", "put(Ljava/lang/String;", "optString", "getString"]), + ("shared_preferences", ["SharedPreferences", "getString", "putString"]), + ("file_path", ["Ljava/io/File;", "FileInputStream", "FileOutputStream"]), + ("crypto", ["Ljavax/crypto/", "MessageDigest", "KeyStore", "Certificate"]), + ( + "react_native", + [ + "Lcom/facebook/react/", + "WritableMap", + "ReadableMap", + "Promise", + "ReactMethod", + ], + ), + ( + "log", + ["Landroid/util/Log;", "Ljava/lang/Exception;->(Ljava/lang/String;)"], + ), +] + + +@dataclass +class BindingCall: + file: str + class_name: str | None + method_name: str | None + line: int + binding_type: str + target: str + id_value: int | None + id_hex: str | None + class_argument: str | None + argument_registers: list[str] + result_register: str | None + immediate_uses: list[dict[str, Any]] + sink_hints: list[str] + context: list[str] + + def to_json(self) -> str: + return json.dumps(self.__dict__, sort_keys=True) + + +def smali_files(root: Path) -> list[Path]: + if root.is_file() and root.suffix == ".smali": + return [root] + return sorted(p for p in root.rglob("*.smali") if p.is_file()) + + +def parse_regs(text: str) -> list[str]: + if ".." in text: + return [] + return [x.strip() for x in text.split(",") if x.strip()] + + +def parse_int(s: str) -> int | None: + try: + return int(s, 0) + except (TypeError, ValueError): + return None + + +def class_at(lines: list[str]) -> str | None: + for line in lines: + m = CLASS_RE.match(line.strip()) + if m: + return m.group(1) + return None + + +def method_at(lines: list[str], idx: int) -> str | None: + for j in range(idx, -1, -1): + s = lines[j].strip() + if s == ".end method": + return None + m = METHOD_RE.match(s) + if m: + return m.group(1) + return None + + +def collect_native_binding_targets( + files: list[Path], root: Path +) -> tuple[set[str], set[str]]: + string_targets: set[str] = set() + class_targets: set[str] = set() + for path in files: + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + cls = class_at(lines) + if not cls: + continue + for raw in lines: + m = METHOD_RE_FULL.match(raw.strip()) + if not m: + continue + parts = m.group("decl").split() + if "native" not in parts or not parts: + continue + sig = parts[-1] + sm = NATIVE_DECL_RE.match(sig) + if not sm: + continue + args = sm.group("args") + ret = sm.group("ret") + full = f"{cls}->{sig}" + if ret == "Ljava/lang/String;" and args in {"I", "II"}: + string_targets.add(full) + if ret == "V" and args == "Ljava/lang/Class;I": + class_targets.add(full) + return string_targets, class_targets + + +def previous_const_int( + lines: list[str], idx: int, reg: str, max_back: int = 40 +) -> int | None: + for j in range(idx - 1, max(-1, idx - max_back - 1), -1): + m = CONST_RE.match(lines[j].strip()) + if m and m.group("reg") == reg: + return parse_int(m.group("value")) + return None + + +def previous_const_class( + lines: list[str], idx: int, reg: str, max_back: int = 40 +) -> str | None: + for j in range(idx - 1, max(-1, idx - max_back - 1), -1): + m = CONST_CLASS_RE.match(lines[j].strip()) + if m and m.group("reg") == reg: + return m.group("class") + return None + + +def result_register_after(lines: list[str], idx: int) -> str | None: + for j in range(idx + 1, min(len(lines), idx + 5)): + m = MOVE_RESULT_RE.match(lines[j].strip()) + if m: + return m.group("reg") + if ( + lines[j].strip() + and not lines[j].strip().startswith(":") + and not lines[j].strip().startswith(".line") + ): + # keep looking through blank/label/line noise only + continue + return None + + +def context_window(lines: list[str], idx: int, radius: int = 8) -> list[str]: + start = max(0, idx - radius) + end = min(len(lines), idx + radius + 1) + return [f"{i+1}: {lines[i]}" for i in range(start, end)] + + +def classify_sinks(texts: list[str]) -> list[str]: + hints = [] + joined = "\n".join(texts) + for name, pats in SINK_PATTERNS: + if any(p in joined for p in pats): + hints.append(name) + return sorted(set(hints)) + + +def immediate_uses( + lines: list[str], idx: int, result_reg: str | None, max_forward: int = 16 +) -> list[dict[str, Any]]: + if not result_reg: + return [] + uses: list[dict[str, Any]] = [] + for j in range(idx + 1, min(len(lines), idx + max_forward + 1)): + s = lines[j].strip() + if not s or s.startswith(".line") or s.startswith(":"): + continue + if result_reg not in s: + continue + inv = INVOKE_RE.match(s) + uses.append( + { + "line": j + 1, + "text": s, + "invoke_target": inv.group("target") if inv else None, + "invoke_op": inv.group("op") if inv else None, + } + ) + if len(uses) >= 8: + break + return uses + + +def scan_file( + path: Path, + root: Path, + string_targets: set[str], + class_targets: set[str], + string_re: re.Pattern[str] | None, + class_re: re.Pattern[str] | None, +) -> list[BindingCall]: + rel = str(path.relative_to(root)) + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + cls = class_at(lines) + calls: list[BindingCall] = [] + for i, raw in enumerate(lines): + s = raw.strip() + inv = INVOKE_RE.match(s) + if not inv: + continue + target = inv.group("target") + regs = parse_regs(inv.group("regs")) + binding_type = None + id_value = None + class_arg = None + if target in string_targets or ( + string_re is not None and string_re.search(target) + ): + binding_type = "string" + if regs: + id_value = previous_const_int(lines, i, regs[-1]) + elif target in class_targets or ( + class_re is not None and class_re.search(target) + ): + binding_type = "class_or_field" + if regs: + id_value = previous_const_int(lines, i, regs[-1]) + if len(regs) >= 2: + class_arg = previous_const_class(lines, i, regs[-2]) + if class_arg is None and regs[-2].startswith("p"): + class_arg = cls + else: + continue + result_reg = ( + result_register_after(lines, i) if binding_type == "string" else None + ) + uses = immediate_uses(lines, i, result_reg) + ctx = context_window(lines, i) + sink_hints = classify_sinks([u["text"] for u in uses] + ctx) + calls.append( + BindingCall( + file=rel, + class_name=cls, + method_name=method_at(lines, i), + line=i + 1, + binding_type=binding_type, + target=target, + id_value=id_value, + id_hex=hex(id_value) if id_value is not None else None, + class_argument=class_arg, + argument_registers=regs, + result_register=result_reg, + immediate_uses=uses, + sink_hints=sink_hints, + context=ctx, + ) + ) + return calls + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument( + "input", type=Path, help="apktool-decoded smali directory or .smali file" + ) + ap.add_argument( + "-o", "--out", type=Path, required=True, help="bindings JSONL output" + ) + ap.add_argument("--summary", type=Path, default=None, help="markdown summary path") + ap.add_argument( + "--json", type=Path, default=None, help="aggregate JSON output path" + ) + ap.add_argument( + "--string-target-regex", + default=DEFAULT_STRING_TARGET_RE, + help="regex for string binding native target", + ) + ap.add_argument( + "--class-target-regex", + default=DEFAULT_CLASS_TARGET_RE, + help="regex for class/field binding native target", + ) + args = ap.parse_args() + + root = args.input.resolve() + scan_root = root if root.is_dir() else root.parent + files = smali_files(root) + native_string_targets, native_class_targets = collect_native_binding_targets( + files, scan_root + ) + string_re = ( + re.compile(args.string_target_regex) if args.string_target_regex else None + ) + class_re = re.compile(args.class_target_regex) if args.class_target_regex else None + calls: list[BindingCall] = [] + for f in files: + calls.extend( + scan_file( + f, + scan_root, + native_string_targets, + native_class_targets, + string_re, + class_re, + ) + ) + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text( + "\n".join(c.to_json() for c in calls) + ("\n" if calls else ""), + encoding="utf-8", + ) + + by_type: dict[str, int] = {} + by_target: dict[str, int] = {} + by_sink: dict[str, int] = {} + ids_by_target: dict[str, set[int]] = {} + for c in calls: + by_type[c.binding_type] = by_type.get(c.binding_type, 0) + 1 + by_target[c.target] = by_target.get(c.target, 0) + 1 + if c.id_value is not None: + ids_by_target.setdefault(c.target, set()).add(c.id_value) + for hint in c.sink_hints: + by_sink[hint] = by_sink.get(hint, 0) + 1 + + aggregate = { + "input": str(args.input), + "files_scanned": len(files), + "binding_calls": len(calls), + "native_string_targets": sorted(native_string_targets), + "native_class_targets": sorted(native_class_targets), + "by_type": by_type, + "by_target_top20": dict( + sorted(by_target.items(), key=lambda kv: kv[1], reverse=True)[:20] + ), + "unique_ids_by_target": {k: len(v) for k, v in sorted(ids_by_target.items())}, + "by_sink_hint": dict( + sorted(by_sink.items(), key=lambda kv: kv[1], reverse=True) + ), + "examples": [c.__dict__ for c in calls[:50]], + } + if args.json: + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text( + json.dumps(aggregate, indent=2, sort_keys=True), encoding="utf-8" + ) + + if args.summary: + lines = [ + "# Promon binding triage summary", + "", + f"Input: `{args.input}`", + f"Smali files scanned: {len(files)}", + f"Binding callsites: {len(calls)}", + "", + "## By type", + "", + ] + for k, v in sorted(by_type.items(), key=lambda kv: kv[1], reverse=True): + lines.append(f"- {k}: {v}") + lines += ["", "## Top targets", ""] + for target, count in sorted( + by_target.items(), key=lambda kv: kv[1], reverse=True + )[:20]: + uid = len(ids_by_target.get(target, set())) + lines.append(f"- `{target}`: {count} calls, {uid} unique IDs") + lines += ["", "## Sink hints", ""] + if by_sink: + for sink, count in sorted( + by_sink.items(), key=lambda kv: kv[1], reverse=True + ): + lines.append(f"- {sink}: {count}") + else: + lines.append("- none") + lines += ["", "## Examples", ""] + for c in calls[:50]: + sinks = ",".join(c.sink_hints) if c.sink_hints else "none" + lines.append( + f"- `{c.file}:{c.line}` {c.binding_type} `{c.target}` id={c.id_hex or 'unknown'} result={c.result_register or 'n/a'} sinks={sinks}" + ) + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text("\n".join(lines) + "\n", encoding="utf-8") + + print( + json.dumps( + { + "files_scanned": len(files), + "binding_calls": len(calls), + "out": str(args.out), + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/capabilities/android-apk-research/scripts/research/promon/promon_elf_triage.py b/capabilities/android-apk-research/scripts/research/promon/promon_elf_triage.py new file mode 100755 index 0000000..89fc5d6 --- /dev/null +++ b/capabilities/android-apk-research/scripts/research/promon/promon_elf_triage.py @@ -0,0 +1,769 @@ +#!/usr/bin/env python3 +"""Static ELF triage for Promon Shield candidate native libraries. + +Accepts either an APK/XAPK or a bare ELF shared object. For APKs, candidate +Promon libraries are selected using the same static indicators as the detector: +libshield.so, random-looking lib[a-z]{10,12}.so with Promon sections, or any ELF +with at least two of .ncu/.ncc/.ncd. + +Outputs are intended for vuln-research coverage restoration, not bypassing: + - elf-triage.json: aggregate library summaries + - elf-sections.jsonl: section table with entropy samples + - syscall-sites.jsonl: AArch64 SVC #0 sites and nearby syscall-number hints + - native-imports.txt: visible dynamic imports / needed libraries + - promon-elf-summary.md: human-readable summary + - libs//.so: extracted candidate libraries when input is an APK +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +import shutil +import struct +import zipfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable + +PROMON_SECTION_NAMES = (".ncu", ".ncc", ".ncd") +PROMON_RANDOM_LIB_PATTERN = re.compile(r"^lib[a-z]{10,12}\.so$") +RASP_TERMS = ( + b"frida", + b"Frida", + b"xposed", + b"Xposed", + b"substrate", + b"Substrate", + b"ptrace", + b"TracerPid", + b"/proc/self/maps", + b"/proc/self/status", + b"/proc/self/mounts", + b"magisk", + b"Magisk", + b"/su", + b"/system/xbin/su", + b"base.apk", + b"ro.kernel.qemu", + b"ro.debuggable", + b"ro.secure", +) +AARCH64_SVC0 = b"\x01\x00\x00\xd4" +AARCH64_RET = b"\xc0\x03\x5f\xd6" + +AARCH64_SYSCALL_NAMES = { + 56: "openat", + 57: "close", + 63: "read", + 64: "write", + 93: "exit", + 94: "exit_group", + 129: "kill", + 134: "rt_sigaction", + 172: "getpid", + 178: "gettid", + 198: "socket", + 215: "munmap", + 220: "clone", + 221: "execve", + 222: "mmap", + 226: "mprotect", + 260: "wait4", + 270: "process_vm_readv", + 271: "process_vm_writev", + 101: "ptrace", + 167: "prctl", +} + +ELF_MACHINES = { + 3: "x86", + 40: "ARM", + 62: "x86_64", + 183: "AArch64", +} + + +@dataclass +class Section: + index: int + name: str + type: int + flags: int + addr: int + offset: int + size: int + entsize: int + link: int + info: int + addralign: int + entropy_sample: float + + +@dataclass +class Segment: + index: int + type: int + flags: int + offset: int + vaddr: int + filesz: int + memsz: int + align: int + + +@dataclass +class ElfInfo: + path: str + abi: str | None + source: str + size: int + sha256: str + elf_class: str + endian: str + machine: str + machine_id: int + entry: int + sections: list[Section] = field(default_factory=list) + segments: list[Segment] = field(default_factory=list) + needed: list[str] = field(default_factory=list) + imported_symbols: list[str] = field(default_factory=list) + init_array: list[int] = field(default_factory=list) + promon_sections: list[str] = field(default_factory=list) + candidate_reasons: list[str] = field(default_factory=list) + rasp_strings: dict[str, list[int]] = field(default_factory=dict) + syscall_sites: list[dict[str, Any]] = field(default_factory=list) + + +def entropy(data: bytes) -> float: + if not data: + return 0.0 + counts = [0] * 256 + for b in data: + counts[b] += 1 + ent = 0.0 + for c in counts: + if c: + p = c / len(data) + ent -= p * math.log2(p) + return round(ent, 4) + + +def cstring(buf: bytes, off: int) -> str: + if off < 0 or off >= len(buf): + return "" + end = buf.find(b"\x00", off) + if end == -1: + end = len(buf) + return buf[off:end].decode("utf-8", errors="replace") + + +def read_cstrings( + buf: bytes, min_len: int = 5, limit: int = 2000 +) -> list[tuple[int, str]]: + out: list[tuple[int, str]] = [] + i = 0 + while i < len(buf) and len(out) < limit: + if 32 <= buf[i] <= 126: + j = i + while j < len(buf) and 32 <= buf[j] <= 126: + j += 1 + if j - i >= min_len: + out.append((i, buf[i:j].decode("ascii", errors="replace"))) + i = j + 1 + else: + i += 1 + return out + + +class ElfParser: + def __init__(self, blob: bytes): + if len(blob) < 0x34 or blob[:4] != b"\x7fELF": + raise ValueError("not an ELF") + self.blob = blob + self.elf_class_id = blob[4] + self.endian_id = blob[5] + if self.elf_class_id not in (1, 2) or self.endian_id not in (1, 2): + raise ValueError("unsupported ELF class/endian") + self.is64 = self.elf_class_id == 2 + self.endian = "<" if self.endian_id == 1 else ">" + self.elf_class = "ELF64" if self.is64 else "ELF32" + self.endian_name = "little" if self.endian == "<" else "big" + self.machine_id = struct.unpack_from(self.endian + "H", blob, 0x12)[0] + if self.is64: + self.entry = struct.unpack_from(self.endian + "Q", blob, 0x18)[0] + self.phoff = struct.unpack_from(self.endian + "Q", blob, 0x20)[0] + self.shoff = struct.unpack_from(self.endian + "Q", blob, 0x28)[0] + self.phentsize = struct.unpack_from(self.endian + "H", blob, 0x36)[0] + self.phnum = struct.unpack_from(self.endian + "H", blob, 0x38)[0] + self.shentsize = struct.unpack_from(self.endian + "H", blob, 0x3A)[0] + self.shnum = struct.unpack_from(self.endian + "H", blob, 0x3C)[0] + self.shstrndx = struct.unpack_from(self.endian + "H", blob, 0x3E)[0] + else: + self.entry = struct.unpack_from(self.endian + "I", blob, 0x18)[0] + self.phoff = struct.unpack_from(self.endian + "I", blob, 0x1C)[0] + self.shoff = struct.unpack_from(self.endian + "I", blob, 0x20)[0] + self.phentsize = struct.unpack_from(self.endian + "H", blob, 0x2A)[0] + self.phnum = struct.unpack_from(self.endian + "H", blob, 0x2C)[0] + self.shentsize = struct.unpack_from(self.endian + "H", blob, 0x2E)[0] + self.shnum = struct.unpack_from(self.endian + "H", blob, 0x30)[0] + self.shstrndx = struct.unpack_from(self.endian + "H", blob, 0x32)[0] + self.sections = self._parse_sections() + self.segments = self._parse_segments() + + def _parse_sections(self) -> list[Section]: + if ( + not self.shoff + or not self.shentsize + or not self.shnum + or self.shstrndx >= self.shnum + ): + return [] + if self.shoff + self.shentsize * self.shnum > len(self.blob): + return [] + raw = [] + if self.is64: + fmt = self.endian + "IIQQQQIIQQ" + else: + fmt = self.endian + "10I" + for i in range(self.shnum): + off = self.shoff + i * self.shentsize + fields = struct.unpack_from(fmt, self.blob, off) + if self.is64: + ( + sh_name, + sh_type, + sh_flags, + sh_addr, + sh_offset, + sh_size, + sh_link, + sh_info, + sh_addralign, + sh_entsize, + ) = fields + else: + ( + sh_name, + sh_type, + sh_flags, + sh_addr, + sh_offset, + sh_size, + sh_link, + sh_info, + sh_addralign, + sh_entsize, + ) = fields + raw.append( + ( + sh_name, + sh_type, + sh_flags, + sh_addr, + sh_offset, + sh_size, + sh_link, + sh_info, + sh_addralign, + sh_entsize, + ) + ) + shstr = raw[self.shstrndx] + shstr_off, shstr_size = shstr[4], shstr[5] + if shstr_off + shstr_size > len(self.blob): + return [] + shstrtab = self.blob[shstr_off : shstr_off + shstr_size] + sections: list[Section] = [] + for idx, ( + sh_name, + sh_type, + sh_flags, + sh_addr, + sh_offset, + sh_size, + sh_link, + sh_info, + sh_addralign, + sh_entsize, + ) in enumerate(raw): + name = ( + cstring(shstrtab, sh_name) + if sh_name < len(shstrtab) + else f"" + ) + sample = b"" + if sh_size and sh_offset < len(self.blob): + sample = self.blob[ + sh_offset : min(len(self.blob), sh_offset + min(sh_size, 65536)) + ] + sections.append( + Section( + idx, + name, + sh_type, + sh_flags, + sh_addr, + sh_offset, + sh_size, + sh_entsize, + sh_link, + sh_info, + sh_addralign, + entropy(sample), + ) + ) + return sections + + def _parse_segments(self) -> list[Segment]: + if not self.phoff or not self.phentsize or not self.phnum: + return [] + if self.phoff + self.phentsize * self.phnum > len(self.blob): + return [] + segs: list[Segment] = [] + for i in range(self.phnum): + off = self.phoff + i * self.phentsize + if self.is64: + p_type, p_flags = struct.unpack_from(self.endian + "II", self.blob, off) + p_offset, p_vaddr, _p_paddr, p_filesz, p_memsz, p_align = ( + struct.unpack_from(self.endian + "6Q", self.blob, off + 8) + ) + else: + ( + p_type, + p_offset, + p_vaddr, + _p_paddr, + p_filesz, + p_memsz, + p_flags, + p_align, + ) = struct.unpack_from(self.endian + "8I", self.blob, off) + segs.append( + Segment( + i, p_type, p_flags, p_offset, p_vaddr, p_filesz, p_memsz, p_align + ) + ) + return segs + + def section_by_name(self, name: str) -> Section | None: + for s in self.sections: + if s.name == name: + return s + return None + + def section_data(self, s: Section) -> bytes: + if not s.size or s.offset >= len(self.blob): + return b"" + return self.blob[s.offset : min(len(self.blob), s.offset + s.size)] + + def vaddr_to_offset(self, vaddr: int) -> int | None: + for seg in self.segments: + if seg.type != 1: # PT_LOAD + continue + if seg.vaddr <= vaddr < seg.vaddr + seg.filesz: + return seg.offset + (vaddr - seg.vaddr) + for s in self.sections: + if s.addr <= vaddr < s.addr + s.size: + return s.offset + (vaddr - s.addr) + return None + + def parse_dynamic(self) -> tuple[list[str], list[str]]: + dyn = self.section_by_name(".dynamic") + dynstr = self.section_by_name(".dynstr") + dynsym = self.section_by_name(".dynsym") + needed: list[str] = [] + imports: list[str] = [] + dynstr_data = self.section_data(dynstr) if dynstr else b"" + if dyn and dynstr_data: + data = self.section_data(dyn) + ent = 16 if self.is64 else 8 + for off in range(0, len(data) - ent + 1, ent): + if self.is64: + tag, val = struct.unpack_from(self.endian + "qQ", data, off) + else: + tag, val = struct.unpack_from(self.endian + "iI", data, off) + if tag == 0: + break + if tag == 1: # DT_NEEDED + needed.append(cstring(dynstr_data, val)) + if dynsym and dynstr_data and dynsym.entsize: + data = self.section_data(dynsym) + for off in range(0, len(data) - dynsym.entsize + 1, dynsym.entsize): + if self.is64: + st_name, st_info, _st_other, st_shndx, st_value, _st_size = ( + struct.unpack_from(self.endian + "IBBHQQ", data, off) + ) + else: + st_name, st_value, _st_size, st_info, _st_other, st_shndx = ( + struct.unpack_from(self.endian + "IIIBBH", data, off) + ) + if not st_name: + continue + # Undefined symbols are imports. + if st_shndx == 0: + name = cstring(dynstr_data, st_name) + if name: + imports.append(name) + return sorted(set(needed)), sorted(set(imports)) + + def parse_init_array(self) -> list[int]: + s = self.section_by_name(".init_array") + if not s: + return [] + data = self.section_data(s) + ptr_size = 8 if self.is64 else 4 + fmt = self.endian + ("Q" if self.is64 else "I") + vals = [] + for off in range(0, len(data) - ptr_size + 1, ptr_size): + val = struct.unpack_from(fmt, data, off)[0] + if val: + vals.append(val) + return vals + + +def find_rasp_strings(blob: bytes) -> dict[str, list[int]]: + hits: dict[str, list[int]] = {} + low = blob.lower() + for term in RASP_TERMS: + needle = term.lower() + positions = [] + start = 0 + while True: + idx = low.find(needle, start) + if idx == -1: + break + positions.append(idx) + start = idx + 1 + if len(positions) >= 20: + break + if positions: + hits[term.decode("utf-8", errors="replace")] = positions + return hits + + +def decode_movz_w8(insn: int) -> int | None: + # MOVZ Wd, #imm16, LSL #shift. We only need W8 syscall materialization. + if (insn & 0x7F800000) != 0x52800000: + return None + rd = insn & 0x1F + if rd != 8: + return None + hw = (insn >> 21) & 0x3 + imm16 = (insn >> 5) & 0xFFFF + return imm16 << (16 * hw) + + +def decode_movn_w8(insn: int) -> int | None: + if (insn & 0x7F800000) != 0x12800000: + return None + rd = insn & 0x1F + if rd != 8: + return None + hw = (insn >> 21) & 0x3 + imm16 = (insn >> 5) & 0xFFFF + val = ~(imm16 << (16 * hw)) & 0xFFFFFFFF + return val + + +def find_aarch64_syscalls(parser: ElfParser, lib_path: str) -> list[dict[str, Any]]: + if parser.machine_id != 183: + return [] + sites: list[dict[str, Any]] = [] + executable_ranges: list[tuple[str, int, int, int]] = [] + for sec in parser.sections: + # SHF_EXECINSTR catches normal .text and Promon's .ncc when the section + # table is still meaningful. + if sec.flags & 0x4: + executable_ranges.append( + (sec.name or f"section_{sec.index}", sec.offset, sec.size, sec.addr) + ) + if not executable_ranges: + # Some protected ELFs hide useful section flags. Fall back to executable + # PT_LOAD segments so SVC sites are still visible in packed/protected + # layouts. + for seg in parser.segments: + if seg.type == 1 and (seg.flags & 0x1): # PT_LOAD + PF_X + executable_ranges.append( + (f"PT_LOAD_{seg.index}", seg.offset, seg.filesz, seg.vaddr) + ) + + seen_offsets: set[int] = set() + for name, file_offset, size, vaddr_base in executable_ranges: + if not size or file_offset >= len(parser.blob): + continue + data = parser.blob[file_offset : min(len(parser.blob), file_offset + size)] + start = 0 + while True: + idx = data.find(AARCH64_SVC0, start) + if idx == -1: + break + file_off = file_offset + idx + if file_off in seen_offsets: + start = idx + 4 + continue + seen_offsets.add(file_off) + vaddr = vaddr_base + idx + syscall_no = None + syscall_source = None + # Look back up to 8 instructions for MOVZ/MOVN W8, #imm. + for back in range(4, min(36, idx + 4), 4): + insn = struct.unpack_from(" str | None: + if path.startswith("lib/"): + parts = path.split("/", 2) + if len(parts) >= 3: + return parts[1] + return None + + +def candidate_reasons(zip_path: str, parser: ElfParser) -> list[str]: + base = zip_path.rsplit("/", 1)[-1] + promon_sections = [ + s.name for s in parser.sections if s.name in PROMON_SECTION_NAMES + ] + reasons: list[str] = [] + if base == "libshield.so": + reasons.append("libshield.so") + if PROMON_RANDOM_LIB_PATTERN.match(base) and len(promon_sections) >= 2: + reasons.append("random-libname-with-promon-sections") + if len(promon_sections) >= 2: + reasons.append("promon-section-pair") + return reasons + + +def triage_blob(blob: bytes, source_path: str, abi: str | None, source: str) -> ElfInfo: + parser = ElfParser(blob) + needed, imports = parser.parse_dynamic() + init_array = parser.parse_init_array() + promon_sections = [ + s.name for s in parser.sections if s.name in PROMON_SECTION_NAMES + ] + info = ElfInfo( + path=source_path, + abi=abi, + source=source, + size=len(blob), + sha256=hashlib.sha256(blob).hexdigest(), + elf_class=parser.elf_class, + endian=parser.endian_name, + machine=ELF_MACHINES.get(parser.machine_id, f"machine_{parser.machine_id}"), + machine_id=parser.machine_id, + entry=parser.entry, + sections=parser.sections, + segments=parser.segments, + needed=needed, + imported_symbols=imports, + init_array=init_array, + promon_sections=promon_sections, + candidate_reasons=candidate_reasons(source_path, parser), + rasp_strings=find_rasp_strings(blob), + syscall_sites=find_aarch64_syscalls(parser, source_path), + ) + return info + + +def info_summary(info: ElfInfo) -> dict[str, Any]: + return { + "path": info.path, + "abi": info.abi, + "source": info.source, + "size": info.size, + "sha256": info.sha256, + "elf_class": info.elf_class, + "endian": info.endian, + "machine": info.machine, + "machine_id": info.machine_id, + "entry": info.entry, + "section_count": len(info.sections), + "segment_count": len(info.segments), + "needed": info.needed, + "imported_symbol_count": len(info.imported_symbols), + "init_array": info.init_array, + "promon_sections": info.promon_sections, + "candidate_reasons": info.candidate_reasons, + "rasp_string_terms": sorted(info.rasp_strings), + "syscall_site_count": len(info.syscall_sites), + } + + +def section_record(info: ElfInfo, s: Section) -> dict[str, Any]: + return { + "library": info.path, + "abi": info.abi, + "index": s.index, + "name": s.name, + "type": s.type, + "flags": s.flags, + "addr": s.addr, + "offset": s.offset, + "size": s.size, + "entsize": s.entsize, + "link": s.link, + "info": s.info, + "addralign": s.addralign, + "entropy_sample": s.entropy_sample, + "promon_section": s.name in PROMON_SECTION_NAMES, + } + + +def collect_inputs( + input_path: Path, out_dir: Path +) -> list[tuple[str, str | None, bytes, str]]: + entries: list[tuple[str, str | None, bytes, str]] = [] + suffix = input_path.suffix.lower() + if suffix in {".apk", ".xapk"}: + with zipfile.ZipFile(input_path) as z: + for name in z.namelist(): + if not (name.startswith("lib/") and name.endswith(".so")): + continue + blob = z.read(name) + if not blob.startswith(b"\x7fELF"): + continue + try: + parser = ElfParser(blob) + except ValueError: + continue + reasons = candidate_reasons(name, parser) + if not reasons: + continue + abi = abi_from_zip_path(name) + extract_path = ( + out_dir / "libs" / (abi or "unknown") / name.rsplit("/", 1)[-1] + ) + extract_path.parent.mkdir(parents=True, exist_ok=True) + extract_path.write_bytes(blob) + entries.append((name, abi, blob, f"apk:{input_path}")) + else: + blob = input_path.read_bytes() + entries.append((input_path.name, None, blob, str(input_path))) + return entries + + +def write_outputs(infos: list[ElfInfo], out_dir: Path) -> None: + out_dir.mkdir(parents=True, exist_ok=True) + triage = { + "library_count": len(infos), + "libraries": [info_summary(i) for i in infos], + } + (out_dir / "elf-triage.json").write_text( + json.dumps(triage, indent=2, sort_keys=True), encoding="utf-8" + ) + + with (out_dir / "elf-sections.jsonl").open("w", encoding="utf-8") as f: + for info in infos: + for s in info.sections: + f.write(json.dumps(section_record(info, s), sort_keys=True) + "\n") + + with (out_dir / "syscall-sites.jsonl").open("w", encoding="utf-8") as f: + for info in infos: + for site in info.syscall_sites: + f.write(json.dumps(site, sort_keys=True) + "\n") + + with (out_dir / "native-imports.txt").open("w", encoding="utf-8") as f: + for info in infos: + f.write(f"## {info.path}\n") + if info.needed: + f.write("NEEDED:\n") + for n in info.needed: + f.write(f" {n}\n") + if info.imported_symbols: + f.write("IMPORTS:\n") + for sym in info.imported_symbols: + f.write(f" {sym}\n") + f.write("\n") + + lines = ["# Promon ELF triage summary", "", f"Libraries triaged: {len(infos)}", ""] + for info in infos: + lines += [ + f"## `{info.path}`", + "", + f"- ABI: `{info.abi or 'n/a'}`", + f"- SHA-256: `{info.sha256}`", + f"- Size: `{info.size}` bytes", + f"- Machine: `{info.machine}` / {info.elf_class} / {info.endian}-endian", + f"- Candidate reasons: {', '.join(info.candidate_reasons) if info.candidate_reasons else 'bare input'}", + f"- Promon sections: {', '.join(info.promon_sections) if info.promon_sections else 'none'}", + f"- Init array entries: {', '.join(hex(x) for x in info.init_array) if info.init_array else 'none'}", + f"- Visible imports: {len(info.imported_symbols)} symbols; NEEDED: {', '.join(info.needed) if info.needed else 'none'}", + f"- AArch64 SVC #0 sites: {len(info.syscall_sites)}", + f"- RASP string terms: {', '.join(info.rasp_strings) if info.rasp_strings else 'none'}", + "", + ] + promon_secs = [s for s in info.sections if s.name in PROMON_SECTION_NAMES] + if promon_secs: + lines += [ + "| Section | Offset | Size | Flags | Entropy sample |", + "| --- | ---: | ---: | ---: | ---: |", + ] + for s in promon_secs: + lines.append( + f"| `{s.name}` | `{s.offset:#x}` | {s.size} | `{s.flags:#x}` | {s.entropy_sample} |" + ) + lines.append("") + syscall_named = [s for s in info.syscall_sites if s.get("syscall_name_hint")] + if syscall_named: + lines += ["First syscall hints:", ""] + for site in syscall_named[:20]: + lines.append( + f"- `{site['section']}` `{site['file_offset']:#x}` -> {site['syscall_number_hint']} / {site['syscall_name_hint']}" + ) + lines.append("") + (out_dir / "promon-elf-summary.md").write_text( + "\n".join(lines) + "\n", encoding="utf-8" + ) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("input", type=Path, help="APK/XAPK or bare ELF .so") + ap.add_argument( + "-o", "--out-dir", type=Path, required=True, help="output directory" + ) + args = ap.parse_args() + + out_dir = args.out_dir + out_dir.mkdir(parents=True, exist_ok=True) + entries = collect_inputs(args.input, out_dir) + infos: list[ElfInfo] = [] + for source_path, abi, blob, source in entries: + try: + infos.append(triage_blob(blob, source_path, abi, source)) + except ValueError as e: + print(f"skip {source_path}: {e}") + write_outputs(infos, out_dir) + print(json.dumps({"libraries": len(infos), "out_dir": str(out_dir)}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/capabilities/android-apk-research/scripts/research/promon/promon_java_triage.py b/capabilities/android-apk-research/scripts/research/promon/promon_java_triage.py new file mode 100755 index 0000000..32b393d --- /dev/null +++ b/capabilities/android-apk-research/scripts/research/promon/promon_java_triage.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +"""Java/smali-side triage for Promon-protected APKs. + +Input is an apktool-decoded smali directory (or one .smali file). The goal is +not deobfuscation; it is to map how Java code interacts with native code and +whether Promon-style string/class binding appears to be present. + +Outputs: + - java-triage.json: aggregate counters and examples + - native-methods.jsonl: native declarations + - native-call-sites.jsonl: calls into declared native methods + - load-library-sites.jsonl: System.loadLibrary callsites + local context + - promon-java-summary.md: human-readable summary +""" + +from __future__ import annotations + +import argparse +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +CLASS_RE = re.compile(r"^\.class\s+(?P.*?)\s*(?PL[^;]+;)") +SUPER_RE = re.compile(r"^\.super\s+(?PL[^;]+;)") +METHOD_RE = re.compile(r"^\.method\s+(?P.+)$") +FIELD_RE = re.compile(r"^\.field\s+(?P.+)$") +INVOKE_RE = re.compile( + r"^(?Pinvoke-\S+)\s+\{(?P[^}]*)\},\s+(?P\S+;->\S+)$" +) +CONST_STRING_RE = re.compile( + r"^const-string(?:/jumbo)?\s+(?P[vp]\d+),\s+\"(?P.*)\"$" +) +CONST_RE = re.compile( + r"^const(?:/\S+)?\s+(?P[vp]\d+),\s+(?P[-+]?0x[0-9a-fA-F]+|[-+]?\d+)" +) +NATIVE_DECL_RE = re.compile(r"(?P[^\s(]+)\((?P[^)]*)\)(?P\S+)$") + +FRAMEWORK_HINTS = { + "react_native": [ + "Lcom/facebook/react/", + "Lcom/facebook/soloader/SoLoader;", + "Lcom/facebook/hermes/", + "libreactnativejni", + "libjscexecutor", + ], + "flutter": ["Lio/flutter/", "libflutter.so", "FlutterActivity"], + "cordova": ["Lorg/apache/cordova/", "CordovaActivity"], + "webview": ["Landroid/webkit/WebView;", "WebViewClient", "addJavascriptInterface"], + "xamarin": ["Lmono/", "libmonodroid", "Xamarin"], +} + +STARTUP_CLASS_HINTS = ( + "Application;", + "Activity;", + "Service;", + "BroadcastReceiver;", + "ContentProvider;", +) + + +@dataclass +class MethodCtx: + class_name: str | None + method_decl: str | None + method_name: str | None + start_line: int + access_flags: list[str] = field(default_factory=list) + + +@dataclass +class NativeMethod: + file: str + class_name: str | None + method_name: str + descriptor: str + args: str + ret: str + flags: list[str] + line: int + namespace: str + likely_role: str + + def to_json(self) -> str: + return json.dumps(self.__dict__, sort_keys=True) + + @property + def full_ref(self) -> str: + return ( + f"{self.class_name}->{self.descriptor}" + if self.class_name + else self.descriptor + ) + + +@dataclass +class NativeCallSite: + file: str + class_name: str | None + method_name: str | None + line: int + invoke: str + target: str + target_role: str + target_namespace: str + argument_registers: list[str] + int_argument_hints: list[int] + context: list[str] + + def to_json(self) -> str: + return json.dumps(self.__dict__, sort_keys=True) + + +@dataclass +class LoadLibrarySite: + file: str + class_name: str | None + method_name: str | None + line: int + invoke: str + argument_registers: list[str] + recovered_library: str | None + recovery: str + context: list[str] + namespace: str + startup_related: bool + + def to_json(self) -> str: + return json.dumps(self.__dict__, sort_keys=True) + + +def smali_files(root: Path) -> list[Path]: + if root.is_file() and root.suffix == ".smali": + return [root] + return sorted(p for p in root.rglob("*.smali") if p.is_file()) + + +def parse_regs(text: str) -> list[str]: + if ".." in text: + return [] + return [x.strip() for x in text.split(",") if x.strip()] + + +def class_namespace(cls: str | None) -> str: + if not cls: + return "unknown" + s = cls.strip("L;").replace("/", ".") + parts = s.split(".") + return ".".join(parts[:3]) if len(parts) >= 3 else s + + +def likely_native_role(cls: str | None, method: str, ret: str, args: str) -> str: + c = cls or "" + if ( + c.startswith("Lcom/facebook/") + or c.startswith("Lcom/google/") + or c.startswith("Lokhttp") + or c.startswith("Lokio") + ): + return "third_party_framework" + if ret == "Ljava/lang/String;" and args in {"I", "II", "Ljava/lang/String;"}: + return "possible_string_binding" + if "Ljava/lang/Class;" in args or ret == "V" and "I" in args: + return "possible_class_or_field_binding" + if method.lower() in {"init", "initialize", "nativeinit", "register"}: + return "possible_native_bootstrap" + return "app_or_library_native" + + +def parse_method_decl(decl: str) -> tuple[list[str], str, str, str, str] | None: + parts = decl.split() + if not parts: + return None + sig = parts[-1] + flags = parts[:-1] + m = NATIVE_DECL_RE.match(sig) + if not m: + return None + name = m.group("name") + args = m.group("args") + ret = m.group("ret") + return flags, name, sig, args, ret + + +def previous_const_string( + lines: list[str], idx: int, reg: str, max_back: int = 40 +) -> tuple[str | None, str]: + for j in range(idx - 1, max(-1, idx - max_back - 1), -1): + s = lines[j].strip() + m = CONST_STRING_RE.match(s) + if m and m.group("reg") == reg: + return m.group("value"), "const-string" + # Common SoLoader pattern: const-string then invoke-static {vX}, SoLoader->loadLibrary + return None, "unresolved" + + +def previous_const_int( + lines: list[str], idx: int, reg: str, max_back: int = 20 +) -> int | None: + for j in range(idx - 1, max(-1, idx - max_back - 1), -1): + s = lines[j].strip() + m = CONST_RE.match(s) + if m and m.group("reg") == reg: + try: + return int(m.group("value"), 0) + except ValueError: + return None + return None + + +def find_method_at(lines: list[str], idx: int, cls: str | None) -> MethodCtx: + for j in range(idx, -1, -1): + s = lines[j].strip() + if s == ".end method": + return MethodCtx(cls, None, None, 0) + m = METHOD_RE.match(s) + if m: + parsed = parse_method_decl(m.group("decl")) + if parsed: + flags, name, sig, _args, _ret = parsed + return MethodCtx(cls, sig, name, j + 1, flags) + return MethodCtx(cls, m.group("decl"), None, j + 1) + return MethodCtx(cls, None, None, 0) + + +def context_window(lines: list[str], idx: int, radius: int = 8) -> list[str]: + start = max(0, idx - radius) + end = min(len(lines), idx + radius + 1) + return [f"{i+1}: {lines[i]}" for i in range(start, end)] + + +def is_startup_related( + cls: str | None, super_cls: str | None, method: str | None +) -> bool: + text = " ".join(x or "" for x in [cls, super_cls, method]) + return any(h in text for h in STARTUP_CLASS_HINTS) or ( + method + in { + "()V", + "onCreate()V", + "attachBaseContext(Landroid/content/Context;)V", + } + ) + + +def scan_file( + path: Path, root: Path +) -> tuple[list[NativeMethod], list[LoadLibrarySite], dict[str, Any]]: + rel = str(path.relative_to(root)) + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + cls = None + super_cls = None + class_flags: list[str] = [] + native_methods: list[NativeMethod] = [] + load_sites: list[LoadLibrarySite] = [] + counters = { + "string_intern": 0, + "new_array_char_files": 0, + "methods_returning_char_array_from_int": 0, + "native_string_int_methods": 0, + "load_library_calls": 0, + "so_loader_calls": 0, + "framework_hits": {k: 0 for k in FRAMEWORK_HINTS}, + } + full_text = "\n".join(lines) + for fw, hints in FRAMEWORK_HINTS.items(): + counters["framework_hits"][fw] = sum(1 for h in hints if h in full_text) + if "Ljava/lang/String;->intern()" in full_text: + counters["string_intern"] = full_text.count("Ljava/lang/String;->intern()") + if "new-array" in full_text and "[C" in full_text: + counters["new_array_char_files"] = 1 + + for i, raw in enumerate(lines): + s = raw.strip() + m = CLASS_RE.match(s) + if m: + cls = m.group("class") + class_flags = m.group("flags").split() + continue + m = SUPER_RE.match(s) + if m: + super_cls = m.group("super") + continue + m = METHOD_RE.match(s) + if m: + parsed = parse_method_decl(m.group("decl")) + if parsed: + flags, name, sig, args, ret = parsed + if "native" in flags: + role = likely_native_role(cls, name, ret, args) + native_methods.append( + NativeMethod( + file=rel, + class_name=cls, + method_name=name, + descriptor=sig, + args=args, + ret=ret, + flags=flags, + line=i + 1, + namespace=class_namespace(cls), + likely_role=role, + ) + ) + if ret == "Ljava/lang/String;" and args in {"I", "II"}: + counters["native_string_int_methods"] += 1 + if sig.endswith("(I)[C"): + counters["methods_returning_char_array_from_int"] += 1 + continue + + inv = INVOKE_RE.match(s) + if not inv: + continue + target = inv.group("target") + if ( + "Ljava/lang/System;->loadLibrary(Ljava/lang/String;)V" in target + or "Lcom/facebook/soloader/SoLoader;->loadLibrary" in target + ): + regs = parse_regs(inv.group("regs")) + arg_reg = regs[-1] if regs else None + lib = None + recovery = "unresolved" + if arg_reg: + lib, recovery = previous_const_string(lines, i, arg_reg) + method_ctx = find_method_at(lines, i, cls) + is_so_loader = "SoLoader;->loadLibrary" in target + counters["load_library_calls"] += 1 + if is_so_loader: + counters["so_loader_calls"] += 1 + load_sites.append( + LoadLibrarySite( + file=rel, + class_name=cls, + method_name=method_ctx.method_name, + line=i + 1, + invoke=target, + argument_registers=regs, + recovered_library=lib, + recovery=recovery, + context=context_window(lines, i), + namespace=class_namespace(cls), + startup_related=is_startup_related( + cls, super_cls, method_ctx.method_name + ), + ) + ) + return native_methods, load_sites, counters + + +def scan_native_calls( + path: Path, root: Path, native_map: dict[str, NativeMethod] +) -> list[NativeCallSite]: + rel = str(path.relative_to(root)) + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + cls = None + calls: list[NativeCallSite] = [] + for i, raw in enumerate(lines): + s = raw.strip() + m = CLASS_RE.match(s) + if m: + cls = m.group("class") + continue + inv = INVOKE_RE.match(s) + if not inv: + continue + target = inv.group("target") + native = native_map.get(target) + if not native: + continue + regs = parse_regs(inv.group("regs")) + int_hints = [] + for r in regs: + v = previous_const_int(lines, i, r) + if v is not None: + int_hints.append(v) + method_ctx = find_method_at(lines, i, cls) + calls.append( + NativeCallSite( + file=rel, + class_name=cls, + method_name=method_ctx.method_name, + line=i + 1, + invoke=inv.group("op"), + target=target, + target_role=native.likely_role, + target_namespace=native.namespace, + argument_registers=regs, + int_argument_hints=int_hints, + context=context_window(lines, i), + ) + ) + return calls + + +def merge_counters(total: dict[str, Any], add: dict[str, Any]) -> None: + for k, v in add.items(): + if isinstance(v, dict): + total.setdefault(k, {}) + for kk, vv in v.items(): + total[k][kk] = total[k].get(kk, 0) + vv + else: + total[k] = total.get(k, 0) + v + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument( + "input", type=Path, help="apktool-decoded smali directory or .smali file" + ) + ap.add_argument( + "-o", "--out", type=Path, required=True, help="aggregate JSON output path" + ) + ap.add_argument( + "--native-methods-out", + type=Path, + default=None, + help="native methods JSONL path", + ) + ap.add_argument( + "--native-calls-out", + type=Path, + default=None, + help="native callsites JSONL path", + ) + ap.add_argument( + "--load-sites-out", type=Path, default=None, help="loadLibrary sites JSONL path" + ) + ap.add_argument("--summary", type=Path, default=None, help="markdown summary path") + args = ap.parse_args() + + root = args.input.resolve() + scan_root = root if root.is_dir() else root.parent + files = smali_files(root) + all_native: list[NativeMethod] = [] + all_loads: list[LoadLibrarySite] = [] + counters: dict[str, Any] = {} + for f in files: + native, loads, c = scan_file(f, scan_root) + all_native.extend(native) + all_loads.extend(loads) + merge_counters(counters, c) + + native_map = {n.full_ref: n for n in all_native} + all_native_calls: list[NativeCallSite] = [] + for f in files: + all_native_calls.extend(scan_native_calls(f, scan_root, native_map)) + + native_by_role: dict[str, int] = {} + native_by_namespace: dict[str, int] = {} + for n in all_native: + native_by_role[n.likely_role] = native_by_role.get(n.likely_role, 0) + 1 + native_by_namespace[n.namespace] = native_by_namespace.get(n.namespace, 0) + 1 + + load_libs: dict[str, int] = {} + unresolved_loads = 0 + for site in all_loads: + if site.recovered_library: + load_libs[site.recovered_library] = ( + load_libs.get(site.recovered_library, 0) + 1 + ) + else: + unresolved_loads += 1 + + native_calls_by_target: dict[str, int] = {} + native_calls_by_role: dict[str, int] = {} + for call in all_native_calls: + native_calls_by_target[call.target] = ( + native_calls_by_target.get(call.target, 0) + 1 + ) + native_calls_by_role[call.target_role] = ( + native_calls_by_role.get(call.target_role, 0) + 1 + ) + + result = { + "input": str(args.input), + "files_scanned": len(files), + "counters": counters, + "native_methods": { + "count": len(all_native), + "by_role": native_by_role, + "by_namespace_top20": dict( + sorted(native_by_namespace.items(), key=lambda kv: kv[1], reverse=True)[ + :20 + ] + ), + "examples": [n.__dict__ for n in all_native[:50]], + }, + "native_call_sites": { + "count": len(all_native_calls), + "by_role": native_calls_by_role, + "by_target_top50": dict( + sorted( + native_calls_by_target.items(), key=lambda kv: kv[1], reverse=True + )[:50] + ), + "by_target_top50_list": [ + {"target": target, "count": count} + for target, count in sorted( + native_calls_by_target.items(), key=lambda kv: kv[1], reverse=True + )[:50] + ], + "examples": [c.__dict__ for c in all_native_calls[:50]], + }, + "load_library_sites": { + "count": len(all_loads), + "recovered_libraries": load_libs, + "unresolved": unresolved_loads, + "examples": [s.__dict__ for s in all_loads[:50]], + }, + "framework_hints": counters.get("framework_hits", {}), + } + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2, sort_keys=True), encoding="utf-8") + + native_out = args.native_methods_out or args.out.with_name("native-methods.jsonl") + native_out.parent.mkdir(parents=True, exist_ok=True) + native_out.write_text( + "\n".join(n.to_json() for n in all_native) + ("\n" if all_native else ""), + encoding="utf-8", + ) + + native_calls_out = args.native_calls_out or args.out.with_name( + "native-call-sites.jsonl" + ) + native_calls_out.parent.mkdir(parents=True, exist_ok=True) + native_calls_out.write_text( + "\n".join(c.to_json() for c in all_native_calls) + + ("\n" if all_native_calls else ""), + encoding="utf-8", + ) + + load_out = args.load_sites_out or args.out.with_name("load-library-sites.jsonl") + load_out.parent.mkdir(parents=True, exist_ok=True) + load_out.write_text( + "\n".join(s.to_json() for s in all_loads) + ("\n" if all_loads else ""), + encoding="utf-8", + ) + + if args.summary: + lines = [ + "# Promon Java/smali triage summary", + "", + f"Input: `{args.input}`", + f"Smali files scanned: {len(files)}", + "", + "## Coverage hints", + "", + f"- `String.intern()` calls: {counters.get('string_intern', 0)}", + f"- files with `new-array [C`: {counters.get('new_array_char_files', 0)}", + f"- methods returning `(I)[C`: {counters.get('methods_returning_char_array_from_int', 0)}", + f"- native String/int methods: {counters.get('native_string_int_methods', 0)}", + f"- native method callsites: {len(all_native_calls)}", + f"- loadLibrary calls: {counters.get('load_library_calls', 0)}", + f"- SoLoader loadLibrary calls: {counters.get('so_loader_calls', 0)}", + "", + "## Framework hints", + "", + ] + for fw, count in sorted((counters.get("framework_hits") or {}).items()): + lines.append(f"- {fw}: {count}") + lines += ["", "## Native methods by role", ""] + for role, count in sorted( + native_by_role.items(), key=lambda kv: kv[1], reverse=True + ): + lines.append(f"- {role}: {count}") + lines += ["", "## Native callsites by role", ""] + for role, count in sorted( + native_calls_by_role.items(), key=lambda kv: kv[1], reverse=True + ): + lines.append(f"- {role}: {count}") + lines += ["", "## Top native call targets", ""] + for target, count in sorted( + native_calls_by_target.items(), key=lambda kv: kv[1], reverse=True + )[:20]: + lines.append(f"- `{target}`: {count}") + lines += ["", "## Recovered load libraries", ""] + if load_libs: + for lib, count in sorted( + load_libs.items(), key=lambda kv: kv[1], reverse=True + ): + lines.append(f"- `{lib}`: {count}") + else: + lines.append("- none") + lines += ["", "## LoadLibrary examples", ""] + for site in all_loads[:25]: + lines.append( + f"- `{site.file}:{site.line}` `{site.class_name or 'unknown'}` `{site.method_name or 'unknown'}` -> `{site.recovered_library or 'unresolved'}` via {site.recovery}" + ) + lines += ["", "## Native method examples", ""] + for n in all_native[:25]: + lines.append( + f"- `{n.file}:{n.line}` `{n.class_name}->{n.descriptor}` role={n.likely_role}" + ) + lines += ["", "## Native callsite examples", ""] + for c in all_native_calls[:25]: + hints = ( + ", ".join(hex(x) for x in c.int_argument_hints) + if c.int_argument_hints + else "none" + ) + lines.append( + f"- `{c.file}:{c.line}` `{c.target}` role={c.target_role} int_hints={hints}" + ) + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text("\n".join(lines) + "\n", encoding="utf-8") + + print( + json.dumps( + { + "files_scanned": len(files), + "native_methods": len(all_native), + "native_call_sites": len(all_native_calls), + "load_library_sites": len(all_loads), + "out": str(args.out), + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/capabilities/android-apk-research/scripts/research/promon/promon_recover.py b/capabilities/android-apk-research/scripts/research/promon/promon_recover.py new file mode 100755 index 0000000..3551208 --- /dev/null +++ b/capabilities/android-apk-research/scripts/research/promon/promon_recover.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 +"""Run the static Promon Shield coverage-restoration workflow. + +This wrapper orchestrates the implemented static stages: + 1. protector_detect.py -> protector.json + 2. optional apktool decode -> smali-raw/ + 3. promon_string_recover.py -> strings.jsonl + patched smali + summary + 4. promon_elf_triage.py -> native ELF profile outputs + 5. RECOVERY_SUMMARY.md roll-up + +It is intentionally static. It does not bypass Promon, instrument a device, or +attempt native section decryption. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +SCRIPT_DIR = Path(__file__).resolve().parent +SCRIPTS_ROOT = SCRIPT_DIR.parents[1] +PROTECTOR_DETECT = SCRIPTS_ROOT / "protector_detect.py" +PROMON_STRING_RECOVER = SCRIPT_DIR / "promon_string_recover.py" +PROMON_ELF_TRIAGE = SCRIPT_DIR / "promon_elf_triage.py" +PROMON_JAVA_TRIAGE = SCRIPT_DIR / "promon_java_triage.py" +PROMON_BINDING_TRIAGE = SCRIPT_DIR / "promon_binding_triage.py" + + +def run( + cmd: list[str], *, cwd: Path | None = None, check: bool = True +) -> subprocess.CompletedProcess[str]: + print("$ " + " ".join(cmd)) + proc = subprocess.run( + cmd, + cwd=str(cwd) if cwd else None, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + if proc.stdout: + print(proc.stdout, end="" if proc.stdout.endswith("\n") else "\n") + if check and proc.returncode != 0: + raise SystemExit(f"command failed ({proc.returncode}): {' '.join(cmd)}") + return proc + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + return json.loads(path.read_text()) + except json.JSONDecodeError: + return {} + + +def count_jsonl(path: Path) -> int: + if not path.exists(): + return 0 + return sum( + 1 + for line in path.read_text(encoding="utf-8", errors="replace").splitlines() + if line.strip() + ) + + +def find_apktool(apktool_arg: str | None) -> str | None: + if apktool_arg: + return apktool_arg + return shutil.which("apktool") + + +def run_apktool_decode(apktool: str, apk: Path, out_dir: Path, force: bool) -> bool: + if out_dir.exists() and not force: + print(f"apktool output exists, reusing: {out_dir}") + return True + if out_dir.exists() and force: + shutil.rmtree(out_dir) + proc = run([apktool, "d", "-f", str(apk), "-o", str(out_dir)], check=False) + if proc.returncode != 0: + print("apktool decode failed; continuing with native/protector outputs only") + return False + return True + + +def write_summary( + out_dir: Path, input_path: Path, apktool_ran: bool, apktool_ok: bool +) -> None: + protector = load_json(out_dir / "protector.json") + elf_triage = load_json(out_dir / "elf-triage.json") + java_triage = load_json(out_dir / "java-triage.json") + binding_triage = load_json(out_dir / "promon-bindings.json") + strings_count = count_jsonl(out_dir / "strings.jsonl") + native_calls_count = count_jsonl(out_dir / "native-call-sites.jsonl") + binding_count = count_jsonl(out_dir / "promon-bindings.jsonl") + syscall_count = count_jsonl(out_dir / "syscall-sites.jsonl") + section_count = count_jsonl(out_dir / "elf-sections.jsonl") + libs = elf_triage.get("libraries", []) + + lines = [ + "# Promon static recovery summary", + "", + f"Input: `{input_path}`", + "", + "## Stage status", + "", + f"- Protector detection: {'ok' if protector else 'missing'}", + f"- apktool decode: {'skipped' if not apktool_ran else 'ok' if apktool_ok else 'failed'}", + f"- Java/smali triage: {'ok' if java_triage else 'missing'}", + f"- String recovery: {strings_count} strings recovered", + f"- Native method callsites mapped: {native_calls_count}", + f"- Promon binding callsites mapped: {binding_count}", + f"- ELF triage: {len(libs)} candidate libraries triaged", + f"- ELF sections recorded: {section_count}", + f"- AArch64 syscall sites recorded: {syscall_count}", + "", + ] + + if protector: + lines += [ + "## Protector routing", + "", + f"- Protector: `{protector.get('protector')}`", + f"- Confidence: `{protector.get('confidence')}`", + f"- Triage strategy: `{protector.get('triage_strategy')}`", + "", + ] + notes = protector.get("notes") or [] + if notes: + lines += ["Notes:", ""] + lines += [f"- {n}" for n in notes] + lines.append("") + + if libs: + lines += ["## Native candidates", ""] + for lib in libs: + lines += [ + f"### `{lib.get('path')}`", + "", + f"- ABI: `{lib.get('abi') or 'n/a'}`", + f"- SHA-256: `{lib.get('sha256')}`", + f"- Machine: `{lib.get('machine')}` / `{lib.get('elf_class')}`", + f"- Candidate reasons: {', '.join(lib.get('candidate_reasons') or []) or 'bare input'}", + f"- Promon sections: {', '.join(lib.get('promon_sections') or []) or 'none'}", + f"- Init array: {', '.join(hex(x) for x in lib.get('init_array') or []) or 'none'}", + f"- RASP string terms: {', '.join(lib.get('rasp_string_terms') or []) or 'none'}", + f"- Syscall sites: {lib.get('syscall_site_count')}", + "", + ] + + if java_triage: + counters = java_triage.get("counters", {}) + native_calls = java_triage.get("native_call_sites", {}) + lines += [ + "## Java/smali triage", + "", + f"- Smali files scanned: {java_triage.get('files_scanned')}", + f"- Framework hints: {java_triage.get('framework_hints')}", + f"- `String.intern()` calls: {counters.get('string_intern', 0)}", + f"- methods returning `(I)[C`: {counters.get('methods_returning_char_array_from_int', 0)}", + f"- native String/int methods: {counters.get('native_string_int_methods', 0)}", + f"- loadLibrary calls: {counters.get('load_library_calls', 0)}", + f"- native method callsites: {native_calls.get('count', 0)}", + "", + ] + top_targets = native_calls.get("by_target_top50", {}) + top_target_list = native_calls.get("by_target_top50_list") or [ + {"target": target, "count": count} for target, count in top_targets.items() + ] + if top_targets: + lines += ["Top native call targets:", ""] + for item in top_target_list[:10]: + lines.append(f"- `{item['target']}`: {item['count']}") + lines.append("") + + if binding_triage: + lines += [ + "## Promon binding triage", + "", + f"- Binding callsites: {binding_triage.get('binding_calls')}", + f"- By type: {binding_triage.get('by_type')}", + f"- Sink hints: {binding_triage.get('by_sink_hint')}", + "", + ] + top_targets = binding_triage.get("by_target_top20", {}) + if top_targets: + lines += ["Top binding targets:", ""] + for target, count in list(top_targets.items())[:10]: + unique = (binding_triage.get("unique_ids_by_target") or {}).get(target) + suffix = f", {unique} unique IDs" if unique is not None else "" + lines.append(f"- `{target}`: {count}{suffix}") + lines.append("") + + lines += [ + "## Artifacts", + "", + "- `protector.json`", + "- `smali-raw/` when apktool decode succeeds", + "- `strings.jsonl`", + "- `java-triage.json`", + "- `native-methods.jsonl`", + "- `native-call-sites.jsonl`", + "- `load-library-sites.jsonl`", + "- `promon-java-summary.md`", + "- `promon-bindings.jsonl`", + "- `promon-bindings.json`", + "- `promon-binding-summary.md`", + "- `smali-strings-recovered/` when string recovery runs", + "- `string-recovery-summary.md`", + "- `elf-triage.json`", + "- `elf-sections.jsonl`", + "- `syscall-sites.jsonl`", + "- `native-imports.txt`", + "- `promon-elf-summary.md`", + "- `libs//.so` for APK inputs with native candidates", + "", + "## Interpretation", + "", + "This is a static coverage-restoration pass. For Promon-protected APKs, use the union of manifest/JADX output, recovered strings, patched smali, and native ELF triage for vulnerability research. Native sections are not decrypted here; dynamic validation or native unpacking should be separate, explicitly authorized follow-up work.", + "", + ] + (out_dir / "RECOVERY_SUMMARY.md").write_text("\n".join(lines), encoding="utf-8") + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument( + "input", + type=Path, + help="APK/XAPK, bare ELF .so, inventory dir, or apktool-decoded smali dir", + ) + ap.add_argument( + "-o", "--out-dir", type=Path, required=True, help="output directory" + ) + ap.add_argument( + "--apktool", + default=None, + help="apktool executable path/name (default: PATH lookup)", + ) + ap.add_argument( + "--skip-apktool", + action="store_true", + help="skip apktool decode/string recovery", + ) + ap.add_argument("--skip-java", action="store_true", help="skip Java/smali triage") + ap.add_argument( + "--skip-bindings", + action="store_true", + help="skip Promon binding callsite triage", + ) + ap.add_argument( + "--skip-strings", + action="store_true", + help="skip string recovery even if smali exists", + ) + ap.add_argument("--skip-elf", action="store_true", help="skip native ELF triage") + ap.add_argument( + "--force", + action="store_true", + help="overwrite existing smali decode / patched smali outputs", + ) + args = ap.parse_args() + + inp = args.input.resolve() + out_dir = args.out_dir.resolve() + out_dir.mkdir(parents=True, exist_ok=True) + + # Stage 1: detector. It supports APK/XAPK and inventory dirs. Bare .so and + # decoded smali dirs are outside its contract, so skip those gracefully. + protector_path = out_dir / "protector.json" + if inp.suffix.lower() in {".apk", ".xapk"} or inp.is_dir(): + # Avoid running detector on apktool decoded dirs without androguard.json. + if inp.is_file() or (inp / "androguard.json").exists(): + run( + [ + sys.executable, + str(PROTECTOR_DETECT), + str(inp), + "-o", + str(protector_path), + ] + ) + else: + print("detector skipped: directory is not an inventory artifact") + else: + print("detector skipped: input is not APK/XAPK/inventory") + + # Stage 2: apktool decode if input is an APK/XAPK. + apktool_ran = False + apktool_ok = False + smali_raw = out_dir / "smali-raw" + if not args.skip_apktool and inp.suffix.lower() in {".apk", ".xapk"}: + apktool_ran = True + apktool = find_apktool(args.apktool) + if apktool is None: + print("apktool not found; skipping smali decode/string recovery") + else: + apktool_ok = run_apktool_decode(apktool, inp, smali_raw, args.force) + elif inp.is_dir() and any(inp.rglob("*.smali")): + smali_raw = inp + apktool_ok = True + + has_smali = apktool_ok and smali_raw.exists() and any(smali_raw.rglob("*.smali")) + + # Stage 3a: Java/smali integration triage. + if not args.skip_java and has_smali: + run( + [ + sys.executable, + str(PROMON_JAVA_TRIAGE), + str(smali_raw), + "-o", + str(out_dir / "java-triage.json"), + "--native-methods-out", + str(out_dir / "native-methods.jsonl"), + "--native-calls-out", + str(out_dir / "native-call-sites.jsonl"), + "--load-sites-out", + str(out_dir / "load-library-sites.jsonl"), + "--summary", + str(out_dir / "promon-java-summary.md"), + ] + ) + else: + print("Java/smali triage skipped") + + # Stage 3b: Promon binding callsite triage. This maps native string and + # class/field binding IDs even when plaintext recovery is not available. + if not args.skip_bindings and has_smali: + run( + [ + sys.executable, + str(PROMON_BINDING_TRIAGE), + str(smali_raw), + "-o", + str(out_dir / "promon-bindings.jsonl"), + "--json", + str(out_dir / "promon-bindings.json"), + "--summary", + str(out_dir / "promon-binding-summary.md"), + ] + ) + else: + print("Promon binding triage skipped") + + # Stage 3c: smali string recovery. + if not args.skip_strings and has_smali: + patched = out_dir / "smali-strings-recovered" + if patched.exists() and args.force: + shutil.rmtree(patched) + run( + [ + sys.executable, + str(PROMON_STRING_RECOVER), + str(smali_raw), + "-o", + str(out_dir / "strings.jsonl"), + "--patched-out", + str(patched), + "--summary", + str(out_dir / "string-recovery-summary.md"), + ] + ) + else: + print("string recovery skipped") + + # Stage 4: native ELF triage. Accepts APK/XAPK and bare ELF .so. It also + # tolerates non-Promon APKs by producing zero library records. + protector = load_json(protector_path) + if ( + protector + and protector.get("protector") not in {"promon_shield", "unknown"} + and inp.suffix.lower() in {".apk", ".xapk"} + ): + print( + f"ELF triage skipped: detected protector is {protector.get('protector')}, not promon_shield" + ) + elif not args.skip_elf and ( + inp.is_file() + and ( + inp.suffix.lower() in {".apk", ".xapk", ".so"} + or inp.read_bytes()[:4] == b"\x7fELF" + ) + ): + run([sys.executable, str(PROMON_ELF_TRIAGE), str(inp), "-o", str(out_dir)]) + else: + print("ELF triage skipped") + + write_summary(out_dir, inp, apktool_ran, apktool_ok) + print(f"wrote {out_dir / 'RECOVERY_SUMMARY.md'}") + + +if __name__ == "__main__": + main() diff --git a/capabilities/android-apk-research/scripts/research/promon/promon_string_recover.py b/capabilities/android-apk-research/scripts/research/promon/promon_string_recover.py new file mode 100755 index 0000000..3a860b5 --- /dev/null +++ b/capabilities/android-apk-research/scripts/research/promon/promon_string_recover.py @@ -0,0 +1,615 @@ +#!/usr/bin/env python3 +"""Recover Promon-style smali string constants without rebuilding an APK. + +This is a research-pipeline friendly port of the public Promon string +recovery approach: scan apktool-decoded smali, evaluate narrow char-array +obfuscation patterns, emit JSONL evidence, and optionally write patched smali. + +Supported first-pass patterns: + 1. Inline char-array construction followed by java.lang.String->intern(). + 2. Helper methods returning [C from an int parameter, followed by call sites + that pass an integer, create a String, then intern it. + +The evaluator intentionally supports only simple integer/char operations seen in +Promon-protected apps. It does not execute arbitrary smali or Python generated +from smali. +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +REG = r"[vp]\d+" +INT_LIT_RE = re.compile(r"[-+]?((0x[0-9a-fA-F]+)|\d+)") +CLASS_RE = re.compile(r"^\.class\s+.*?\s+(L[^;]+;)") +METHOD_RE = re.compile(r"^\.method\s+.*?\s+([^\s]+)$") +INVOKE_RE = re.compile(r"^(invoke-\S+)\s+\{([^}]*)\},\s+(\S+;->\S+)$") + + +@dataclass +class Recovery: + file: str + class_name: str | None + method_name: str | None + start_line: int + end_line: int + pattern: str + plaintext: str + replacement_register: str | None + confidence: str = "medium" + evidence: dict[str, Any] = field(default_factory=dict) + + def to_json(self) -> str: + return json.dumps(self.__dict__, ensure_ascii=False, sort_keys=True) + + +@dataclass +class CharArrayHelper: + file: str + class_name: str + method_name: str + descriptor: str + start_line: int + end_line: int + param_register: str + lines: list[str] + + @property + def full_ref(self) -> str: + return f"{self.class_name}->{self.method_name}" + + +def parse_int(token: str) -> int | None: + token = token.rstrip(",").strip() + m = INT_LIT_RE.fullmatch(token) + if not m: + return None + try: + return int(token, 0) + except ValueError: + return None + + +def parse_register_list(text: str) -> list[str]: + text = text.strip() + if not text: + return [] + # Minimal support for explicit register lists. Range invokes are uncommon in + # the targeted string patterns; leave them unresolved for now. + if ".." in text: + return [] + return [p.strip() for p in text.split(",") if p.strip()] + + +def java_escape(s: str) -> str: + return json.dumps(s, ensure_ascii=False)[1:-1] + + +def read_smali(path: Path) -> list[str]: + return path.read_text(encoding="utf-8", errors="replace").splitlines() + + +def write_smali(path: Path, lines: list[str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def find_class_name(lines: list[str]) -> str | None: + for line in lines: + m = CLASS_RE.match(line.strip()) + if m: + return m.group(1) + return None + + +def line_method_at(lines: list[str], idx: int) -> str | None: + for j in range(idx, -1, -1): + s = lines[j].strip() + m = METHOD_RE.match(s) + if m: + return m.group(1) + if s == ".end method": + return None + return None + + +def method_bounds_at(lines: list[str], idx: int) -> tuple[int, int] | None: + start = None + for j in range(idx, -1, -1): + s = lines[j].strip() + if METHOD_RE.match(s): + start = j + break + if s == ".end method": + return None + if start is None: + return None + for j in range(idx, len(lines)): + if lines[j].strip() == ".end method": + return start, j + return start, len(lines) - 1 + + +def method_ranges(lines: list[str]) -> list[tuple[int, int, str]]: + ranges: list[tuple[int, int, str]] = [] + start = None + name = None + for i, line in enumerate(lines): + s = line.strip() + m = METHOD_RE.match(s) + if m: + start = i + name = m.group(1) + elif s == ".end method" and start is not None and name is not None: + ranges.append((start, i, name)) + start = None + name = None + return ranges + + +def eval_char_array_lines( + lines: list[str], param_value: int | None = None +) -> tuple[str | None, dict[str, Any]]: + """Evaluate a narrow smali char-array string construction. + + Returns (plaintext, evidence). Supports const/new-array, xor/add/sub/rsub, + int-to-char, aput-char, aget-char, move-result-object from String ctor, and + return-object. If param_value is provided, p0/p1 are initialized with it so + helper methods returning [C can be evaluated at call sites. + """ + regs: dict[str, Any] = {} + arrays: dict[str, list[Any]] = {} + result: str | None = None + returned_array: list[Any] | None = None + ops_seen: list[str] = [] + if param_value is not None: + regs["p0"] = param_value + regs["p1"] = param_value + + def val(tok: str) -> Any: + tok = tok.rstrip(",") + if tok in regs: + return regs[tok] + lit = parse_int(tok) + if lit is not None: + return lit + raise KeyError(tok) + + for raw in lines: + line = raw.strip() + if not line or line.startswith("#") or line.startswith("."): + continue + parts = line.replace(",", "").split() + if not parts: + continue + op = parts[0] + try: + if ( + op.startswith("const") + and len(parts) >= 3 + and re.fullmatch(REG, parts[1]) + ): + parsed = parse_int(parts[2]) + if parsed is not None: + regs[parts[1]] = parsed + ops_seen.append(op) + elif op == "new-array" and len(parts) >= 4 and parts[3] == "[C": + size = int(val(parts[2])) + arrays[parts[1]] = [None] * size + regs[parts[1]] = arrays[parts[1]] + ops_seen.append(op) + elif op == "xor-int/lit16" and len(parts) >= 4: + regs[parts[1]] = int(val(parts[2])) ^ int(val(parts[3])) + ops_seen.append(op) + elif op == "xor-int" and len(parts) >= 4: + regs[parts[1]] = int(val(parts[2])) ^ int(val(parts[3])) + ops_seen.append(op) + elif op in {"add-int", "add-int/2addr"}: + if op == "add-int/2addr" and len(parts) >= 3: + regs[parts[1]] = int(val(parts[1])) + int(val(parts[2])) + elif len(parts) >= 4: + regs[parts[1]] = int(val(parts[2])) + int(val(parts[3])) + ops_seen.append(op) + elif op.startswith("add-int/lit") and len(parts) >= 4: + regs[parts[1]] = int(val(parts[2])) + int(val(parts[3])) + ops_seen.append(op) + elif op in {"sub-int", "sub-int/2addr"}: + if op == "sub-int/2addr" and len(parts) >= 3: + regs[parts[1]] = int(val(parts[1])) - int(val(parts[2])) + elif len(parts) >= 4: + regs[parts[1]] = int(val(parts[2])) - int(val(parts[3])) + ops_seen.append(op) + elif op.startswith("rsub-int") and len(parts) >= 4: + regs[parts[1]] = int(val(parts[3])) - int(val(parts[2])) + ops_seen.append(op) + elif op == "int-to-char" and len(parts) >= 3: + regs[parts[1]] = chr(int(val(parts[2])) & 0x10FFFF) + ops_seen.append(op) + elif op == "aput-char" and len(parts) >= 4: + ch = val(parts[1]) + arr = regs.get(parts[2]) + idx = int(val(parts[3])) + if isinstance(ch, int): + ch = chr(ch & 0x10FFFF) + if isinstance(arr, list) and 0 <= idx < len(arr): + arr[idx] = ch + ops_seen.append(op) + elif op == "aget-char" and len(parts) >= 4: + arr = regs.get(parts[2]) + idx = int(val(parts[3])) + if isinstance(arr, list) and 0 <= idx < len(arr): + regs[parts[1]] = arr[idx] + ops_seen.append(op) + elif ( + op.startswith("invoke-direct") + and "Ljava/lang/String;->([C)V" in line + ): + m = INVOKE_RE.match(line) + if not m: + continue + regs_list = parse_register_list(m.group(2)) + if len(regs_list) >= 2 and isinstance(regs.get(regs_list[1]), list): + arr = regs[regs_list[1]] + if all(isinstance(c, str) for c in arr): + regs[regs_list[0]] = "".join(arr) + ops_seen.append("String.([C)") + elif op == "move-result-object" and len(parts) >= 2: + # If the previous invoke was String.intern(), the receiver is + # already the string; the scan window resolves replacement via + # the move-result register. Keep existing result if present. + ops_seen.append(op) + elif op == "return-object" and len(parts) >= 2: + obj = regs.get(parts[1]) + if isinstance(obj, list) and all(isinstance(c, str) for c in obj): + returned_array = obj + result = "".join(obj) + elif isinstance(obj, str): + result = obj + ops_seen.append(op) + except (KeyError, ValueError, TypeError, IndexError, OverflowError): + continue + + if result is None: + # Fallback: any completed char array in the window is a likely inline + # string if String.intern() was seen by the caller. + for arr in arrays.values(): + if arr and all(isinstance(c, str) for c in arr): + result = "".join(arr) + returned_array = arr + break + evidence = { + "ops_seen": sorted(set(ops_seen)), + "returned_array_len": len(returned_array or []), + } + return result, evidence + + +def collect_helpers( + file_path: Path, root: Path, lines: list[str] +) -> dict[str, CharArrayHelper]: + helpers: dict[str, CharArrayHelper] = {} + cls = find_class_name(lines) + if not cls: + return helpers + rel = str(file_path.relative_to(root)) + for start, end, method_name in method_ranges(lines): + if not method_name.endswith("(I)[C"): + continue + window = lines[start : end + 1] + if not any("new-array" in l and "[C" in l for l in window): + continue + # p0 for static methods, p1 for instance methods. Promon examples are + # usually static, but keep both in evaluator. + param_register = ( + "p0" + if " static " in lines[start] or lines[start].startswith(".method static") + else "p1" + ) + helper = CharArrayHelper( + file=rel, + class_name=cls, + method_name=method_name, + descriptor=method_name, + start_line=start + 1, + end_line=end + 1, + param_register=param_register, + lines=window, + ) + helpers[helper.full_ref] = helper + return helpers + + +def find_inline_recoveries( + file_path: Path, root: Path, lines: list[str] +) -> list[Recovery]: + recs: list[Recovery] = [] + cls = find_class_name(lines) + rel = str(file_path.relative_to(root)) + helper_ranges = [ + (s, e) for s, e, name in method_ranges(lines) if name.endswith("(I)[C") + ] + for i, line in enumerate(lines): + if "Ljava/lang/String;->intern()" not in line: + continue + bounds = method_bounds_at(lines, i) + if bounds is None: + continue + method_start, _method_end = bounds + if any(s <= i <= e for s, e in helper_ranges): + continue + new_array_idx = i + while new_array_idx > method_start and i - new_array_idx < 80: + if "new-array" in lines[new_array_idx] and "[C" in lines[new_array_idx]: + break + new_array_idx -= 1 + if not ("new-array" in lines[new_array_idx] and "[C" in lines[new_array_idx]): + continue + # Include a few instructions before new-array so the size register's + # const assignment is available to the evaluator. + eval_start = max(method_start + 1, new_array_idx - 4) + patch_start = new_array_idx + if new_array_idx > method_start + 1 and lines[ + new_array_idx - 1 + ].strip().startswith("const"): + patch_start = new_array_idx - 1 + move_result_idx = i + for j in range(i + 1, min(len(lines), i + 4)): + s = lines[j].strip().replace(",", "") + parts = s.split() + if len(parts) >= 2 and parts[0] == "move-result-object": + move_result_idx = j + break + end = move_result_idx + plaintext, evidence = eval_char_array_lines(lines[eval_start : end + 1]) + if plaintext is None: + continue + repl_reg = None + for j in range(i + 1, min(len(lines), i + 4)): + s = lines[j].strip().replace(",", "") + parts = s.split() + if len(parts) >= 2 and parts[0] == "move-result-object": + repl_reg = parts[1] + break + recs.append( + Recovery( + file=rel, + class_name=cls, + method_name=line_method_at(lines, i), + start_line=patch_start + 1, + end_line=end + 1, + pattern="inline_char_array_intern", + plaintext=plaintext, + replacement_register=repl_reg, + confidence="high" if repl_reg else "medium", + evidence=evidence, + ) + ) + return recs + + +def find_helper_call_recoveries( + file_path: Path, + root: Path, + lines: list[str], + helpers: dict[str, CharArrayHelper], +) -> list[Recovery]: + recs: list[Recovery] = [] + cls = find_class_name(lines) + rel = str(file_path.relative_to(root)) + consts: dict[str, int] = {} + pending_helper: tuple[CharArrayHelper, int, int] | None = None + pending_string: tuple[str, int, int, str] | None = ( + None # plaintext,start,end,helper_ref + ) + + for i, raw in enumerate(lines): + line = raw.strip() + parts = line.replace(",", "").split() + if ( + parts + and parts[0].startswith("const") + and len(parts) >= 3 + and re.fullmatch(REG, parts[1]) + ): + parsed = parse_int(parts[2]) + if parsed is not None: + consts[parts[1]] = parsed + m = INVOKE_RE.match(line) + if m: + regs = parse_register_list(m.group(2)) + target = m.group(3) + helper = helpers.get(target) + if helper and regs: + arg_reg = regs[-1] + if arg_reg in consts: + plaintext, evidence = eval_char_array_lines( + helper.lines, consts[arg_reg] + ) + if plaintext is not None: + pending_helper = (helper, consts[arg_reg], i) + pending_string = (plaintext, i, i, helper.full_ref) + elif "Ljava/lang/String;->([C)V" in target and pending_string: + pending_string = ( + pending_string[0], + pending_string[1], + i, + pending_string[3], + ) + elif "Ljava/lang/String;->intern()" in target and pending_string: + move_result_idx = i + repl_reg = None + for j in range(i + 1, min(len(lines), i + 4)): + s = lines[j].strip().replace(",", "") + p = s.split() + if len(p) >= 2 and p[0] == "move-result-object": + repl_reg = p[1] + move_result_idx = j + break + end = move_result_idx + helper, arg_value, helper_call_line = ( + pending_helper if pending_helper else (None, None, None) + ) # type: ignore[misc] + recs.append( + Recovery( + file=rel, + class_name=cls, + method_name=line_method_at(lines, i), + start_line=(pending_string[1] + 1), + end_line=end + 1, + pattern="helper_char_array_intern", + plaintext=pending_string[0], + replacement_register=repl_reg, + confidence="high" if repl_reg else "medium", + evidence={ + "helper": pending_string[3], + "helper_file": helper.file if helper else None, + "helper_line": helper.start_line if helper else None, + "arg_value": arg_value, + "helper_call_line": (helper_call_line + 1) + if helper_call_line is not None + else None, + }, + ) + ) + pending_helper = None + pending_string = None + if line.startswith(".end method"): + consts = {} + pending_helper = None + pending_string = None + return recs + + +def patch_lines(lines: list[str], recs: list[Recovery]) -> list[str]: + patched = list(lines) + for rec in sorted( + [r for r in recs if r.replacement_register], + key=lambda r: r.start_line, + reverse=True, + ): + indent = "" + if 0 <= rec.start_line - 1 < len(lines): + indent = lines[rec.start_line - 1][ + : len(lines[rec.start_line - 1]) + - len(lines[rec.start_line - 1].lstrip()) + ] + replacement = f'{indent}const-string {rec.replacement_register}, "{java_escape(rec.plaintext)}"' + start = rec.start_line - 1 + end = rec.end_line + patched[start:end] = [replacement] + return patched + + +def iter_smali_files(root: Path) -> list[Path]: + if root.is_file() and root.suffix == ".smali": + return [root] + return sorted(p for p in root.rglob("*.smali") if p.is_file()) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument( + "input", type=Path, help="apktool-decoded directory or a .smali file" + ) + ap.add_argument("-o", "--out", type=Path, required=True, help="JSONL output path") + ap.add_argument( + "--patched-out", + type=Path, + default=None, + help="optional directory for patched smali tree", + ) + ap.add_argument( + "--summary", type=Path, default=None, help="optional markdown summary path" + ) + args = ap.parse_args() + + root = args.input.resolve() + files = iter_smali_files(root) + all_lines: dict[Path, list[str]] = {} + helpers: dict[str, CharArrayHelper] = {} + for f in files: + lines = read_smali(f) + all_lines[f] = lines + helpers.update( + collect_helpers(f, root if root.is_dir() else root.parent, lines) + ) + + scan_root = root if root.is_dir() else root.parent + recoveries: list[Recovery] = [] + by_file: dict[Path, list[Recovery]] = {} + for f, lines in all_lines.items(): + recs = find_inline_recoveries(f, scan_root, lines) + recs.extend(find_helper_call_recoveries(f, scan_root, lines, helpers)) + # Dedupe same line/plaintext/pattern in case an inline helper body is + # also seen as a call-site pattern. + seen: set[tuple[int, str, str]] = set() + unique: list[Recovery] = [] + for r in recs: + key = (r.start_line, r.pattern, r.plaintext) + if key in seen: + continue + seen.add(key) + unique.append(r) + recoveries.extend(unique) + if unique: + by_file[f] = unique + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text( + "\n".join(r.to_json() for r in recoveries) + ("\n" if recoveries else ""), + encoding="utf-8", + ) + + if args.patched_out: + out_root = args.patched_out + if out_root.exists(): + shutil.rmtree(out_root) + if root.is_dir(): + shutil.copytree(root, out_root) + for f, recs in by_file.items(): + rel = f.relative_to(scan_root) + write_smali(out_root / rel, patch_lines(all_lines[f], recs)) + else: + out_root.mkdir(parents=True, exist_ok=True) + out_file = out_root / root.name + write_smali(out_file, patch_lines(all_lines[root], by_file.get(root, []))) + + if args.summary: + args.summary.parent.mkdir(parents=True, exist_ok=True) + top = sorted(recoveries, key=lambda r: (r.file, r.start_line))[:50] + body = [ + "# Promon string recovery summary", + "", + f"Input: `{args.input}`", + f"Smali files scanned: {len(files)}", + f"Helpers discovered: {len(helpers)}", + f"Strings recovered: {len(recoveries)}", + "", + "## First recovered strings", + "", + ] + for r in top: + body.append(f"- `{r.file}:{r.start_line}` `{r.pattern}` -> `{r.plaintext}`") + args.summary.write_text("\n".join(body) + "\n", encoding="utf-8") + + print( + json.dumps( + { + "files_scanned": len(files), + "helpers": len(helpers), + "strings_recovered": len(recoveries), + "out": str(args.out), + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/capabilities/android-apk-research/scripts/run_class_rg.sh b/capabilities/android-apk-research/scripts/run_class_rg.sh new file mode 100755 index 0000000..cba50f8 --- /dev/null +++ b/capabilities/android-apk-research/scripts/run_class_rg.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# Run the focused-rg profile for one impact class against a JADX source tree. +# Patterns mirror references/android-semantic-vuln-hunting/impact-class-rg-profiles.md. +# +# Usage: +# run_class_rg.sh A_ingress /path/to/jadx/sources/ /path/to/out_dir +# +# Writes: +# /rg-class-.txt one line per hit +# /rg-class-.summary.txt per-file hit counts, descending +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "usage: $0 " >&2 + echo " = A_ingress|B_remote|C_wallet|D_secret|E_file_cloud|F_family|G_messenger|H_email|I_browser|J_iot" >&2 + exit 64 +fi +CLASS="$1" +SRC="$2" +OUT="$3" +mkdir -p "$OUT" + +declare -a PATTERNS=() + +case "$CLASS" in + A_ingress|A) + SHORT=A + PATTERNS+=( + 'decodeQrCode|parseQrPayload|decodeBarcode|onBarcodeDetected|onQrCodeDetected' + 'BarcodeFormat|BarcodeReader|MultiFormatReader|Result\.getText\(' + 'ZXing|com\.google\.mlkit\.vision\.barcode|BarcodeAnalyzer' + 'Uri\.parse\([^)]*(result|barcode|qr|payload|scanned)' + 'WifiConfig|MeCard|vCard|VCard|geo:|tel:|mailto:|sms:' + 'Intent\(.*Uri\.parse\(|startActivity\(.*Uri\.parse\(' + 'loadUrl\([^)]*(result|scanned|payload)' + 'onActivityResult.*(RequestCode.*QR|Scan|Barcode)' + ) + ;; + B_remote|B) + SHORT=B + PATTERNS+=( + 'AccessibilityService|onAccessibilityEvent|performGlobalAction' + 'MediaProjection|VirtualDisplay|ScreenCapture|createScreenCaptureIntent' + 'pairingCode|partnerId|sessionToken|sessionId|connectionId' + 'startSession|joinSession|acceptInvite|inviteCode' + 'getQueryParameter\(.*(code|token|session|partner|invite|pairing)' + 'addJavascriptInterface|loadUrl' + 'TYPE_APPLICATION_OVERLAY|TYPE_SYSTEM_ALERT|SYSTEM_ALERT_WINDOW' + 'Settings\.canDrawOverlays|requestPermission.*overlay' + ) + ;; + C_wallet|C) + SHORT=C + PATTERNS+=( + 'wc:|walletconnect|WalletConnect|WCSession|WCClient|RelayClient' + 'signTransaction|signMessage|signTypedData|personal_sign|eth_send|eth_sign' + 'sendTransaction|sendRawTransaction|broadcastTransaction' + 'getActiveSession|approveSession|rejectSession' + 'dappUrl|originUrl|requestUrl|metadata\.url|peerMetadata' + 'Web3|Web3Modal|RPC|provider\.send' + 'mnemonic|seed_phrase|seedPhrase|privateKey|keystore' + 'BiometricPrompt|FingerprintManager|KeyguardManager' + 'addJavascriptInterface|setJavaScriptEnabled' + 'getQueryParameter\(.*(address|amount|chain|callback|tx)' + ) + ;; + D_secret|D) + SHORT=D + PATTERNS+=( + 'AutofillService|onFillRequest|FillResponse|AutofillId|AssistStructure' + 'AccessibilityService|onAccessibilityEvent|AccessibilityNodeInfo' + 'getInstalledApplications|getPackagesForUid|getApplicationInfo' + 'getRunningAppProcesses|getRunningTasks|UsageStatsManager' + 'ClipboardManager|setPrimaryClip|getPrimaryClip' + 'BiometricPrompt|BiometricManager|KeyGenParameterSpec' + 'KeyStore|MasterKey|EncryptedSharedPreferences|Cipher\.getInstance' + 'addJavascriptInterface|@JavascriptInterface' + 'getQueryParameter\(.*(totp|otp|secret|seed|token|code)' + 'TOTP|HOTP|generateOTP' + ) + ;; + E_file_cloud|E) + SHORT=E + PATTERNS+=( + 'ContentProvider|getContentResolver|openInputStream|openOutputStream' + 'FileProvider|getUriForFile|FLAG_GRANT_READ_URI_PERMISSION|FLAG_GRANT_WRITE_URI_PERMISSION' + 'grantUriPermission|revokeUriPermission' + 'ACTION_SEND|ACTION_SEND_MULTIPLE|EXTRA_STREAM|EXTRA_TEXT' + 'OpenableColumns|MediaStore|DocumentsContract' + 'StorageVolume|Environment\.getExternalStorage' + 'webdav|WebDAV|nextcloud|owncloud' + 'getQueryParameter\(.*(path|file|url|src|download)' + 'startActivity\(.*VIEW.*Uri' + ) + ;; + F_family|F) + SHORT=F + PATTERNS+=( + 'DeviceAdminReceiver|DevicePolicyManager|onPasswordChanged' + 'FusedLocationProvider|LocationManager|requestLocationUpdates' + 'AccessibilityService|onAccessibilityEvent' + 'pairingCode|invitationCode|familyCode|joinCode' + 'getQueryParameter\(.*(code|invite|family|child|parent|pair)' + 'AdminPin|adminPin|PARENT_PIN|parentPin' + 'Geofence|addGeofences|GeofencingClient' + 'BackgroundService|JobScheduler|WorkManager.*Periodic' + 'sendTextMessage|SmsManager' + ) + ;; + G_messenger|G) + SHORT=G + PATTERNS+=( + 'LinkPreview|OpenGraph|ogImage|ogDescription|fetchPreview' + 'addJavascriptInterface|setJavaScriptEnabled|shouldOverrideUrlLoading' + 'CookieManager|setCookie' + 'invitationLink|inviteLink|joinLink|tg:|sgnl:|threema:|line:|wickr:' + 'getParcelableExtra\(.*(EXTRA_STREAM|EXTRA_INTENT)' + 'startActivity\(.*Intent.*data' + 'getQueryParameter\(.*(invite|token|chat|user|room|server)' + 'Notification.*setContentIntent|PendingIntent\.getActivity' + 'Linkify|spannable|URLSpan' + ) + ;; + H_email|H) + SHORT=H + PATTERNS+=( + 'MimeMessage|MimeMultipart|MimeBodyPart|MimeUtility|MimeType' + 'WebView.*loadDataWithBaseURL|loadData|loadUrl' + 'setJavaScriptEnabled\s*\(\s*true|addJavascriptInterface' + 'MailTo|mailto:|message/rfc822' + 'CalendarContract|Events\.CONTENT_URI' + 'X-Originating-IP|Received: from|Return-Path' + 'S/MIME|PGP|PgpKey|Mailvelope' + 'getParcelableExtra\(.*(EXTRA_STREAM|EXTRA_EMAIL)' + 'FileProvider|getUriForFile' + 'getQueryParameter\(.*(subject|body|cc|bcc|to|attach)' + ) + ;; + I_browser|I) + SHORT=I + PATTERNS+=( + 'CustomTabsIntent|CustomTabsClient|CustomTabsSession|CustomTabsCallback' + 'addJavascriptInterface|@JavascriptInterface|setJavaScriptEnabled' + 'shouldOverrideUrlLoading|WebViewClient|WebChromeClient' + 'intent://|intent:.*S\.browser_fallback_url' + 'Intent\.parseUri|parseUri' + 'getCookie|setCookie|CookieManager' + 'getQueryParameter\(.*(url|src|next|return|redirect)' + 'TrustedWebActivity|TWA|setNavigationBarColor' + 'about:|chrome:|content:|javascript:' + 'PWA|service-worker|manifest\.webmanifest' + ) + ;; + J_iot|J) + SHORT=J + PATTERNS+=( + 'mDNS|NsdManager|MulticastSocket|SSDP|UPnP|Bonjour' + 'WifiManager|WifiNetworkSpecifier|WifiConfiguration|connectToWifi' + 'BluetoothGatt|BluetoothLeScanner|ScanCallback' + 'addJavascriptInterface|setJavaScriptEnabled' + 'OAuth|oauth_token|authorize|redirect_uri' + 'mqtt|Mqtt|MqttClient|MqttAndroidClient' + 'PushToken|FCM_TOKEN|registrationToken|onTokenRefresh' + 'deviceId|deviceUuid|deviceSecret|pairingToken|provisioningToken' + 'getQueryParameter\(.*(device|token|home|account|server)' + 'http://(192|10|172)\.' + ) + ;; + *) + echo "unknown class: $CLASS" >&2 + exit 64 + ;; +esac + +OUT_FILE="$OUT/rg-class-$SHORT.txt" +SUMMARY="$OUT/rg-class-$SHORT.summary.txt" + +# Build the rg invocation: -n line numbers, -P PCRE2, multiple -e patterns +ARGS=( -n -P ) +for pat in "${PATTERNS[@]}"; do + ARGS+=( -e "$pat" ) +done +ARGS+=( "$SRC" ) + +rg "${ARGS[@]}" > "$OUT_FILE" 2>/dev/null || true + +# Summary: file -> hit count, descending. Helps prioritise files to read. +awk -F: '{print $1}' "$OUT_FILE" | sort | uniq -c | sort -rn > "$SUMMARY" + +HITS=$(wc -l < "$OUT_FILE" | tr -d ' ') +FILES=$(wc -l < "$SUMMARY" | tr -d ' ') +echo "class=$CLASS short=$SHORT hits=$HITS files=$FILES out=$OUT_FILE summary=$SUMMARY" diff --git a/capabilities/android-apk-research/scripts/run_corpus_inventory.py b/capabilities/android-apk-research/scripts/run_corpus_inventory.py new file mode 100644 index 0000000..fef6084 --- /dev/null +++ b/capabilities/android-apk-research/scripts/run_corpus_inventory.py @@ -0,0 +1,546 @@ +#!/usr/bin/env python3 +"""Run a parallel, resumable first-pass inventory over an APK corpus. + +This runner is intentionally an orchestrator. It reuses extract_attack_surface.py +for the cheap per-APK inventory, writes per-APK artifacts keyed by SHA256, and +emits aggregate JSONL/status files for ranking. It does not decompile APKs. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures as cf +import contextlib +import hashlib +import json +import os +import shutil +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +SOURCE_EXT = ".apk" + +# Direct import of the sibling script. `extract_attack_surface.py` lives in the +# same scripts/ directory and provides `summarize_packers` + `rank` (pure-Python, +# no APK input) for the re-rank pass after APKiD signal lands. Importing as a +# module is materially cleaner than the dynamic spec_from_file_location dance +# the earlier version used; sys.path gymnastics live here so the rest of the +# module reads as normal Python. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +try: + import extract_attack_surface as _eas # type: ignore[import-not-found] +except ImportError as exc: + _eas = None # type: ignore[assignment] + print(f"warning: extract_attack_surface import failed: {exc}", file=sys.stderr) + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def iter_apks(paths: Iterable[Path], limit: int | None = None) -> list[Path]: + out: list[Path] = [] + for p in paths: + p = p.expanduser().resolve() + if p.is_dir(): + out.extend(sorted(x for x in p.rglob(f"*{SOURCE_EXT}") if x.is_file())) + elif p.is_file() and p.suffix.lower() == SOURCE_EXT: + out.append(p) + deduped = sorted(set(out)) + return deduped[:limit] if limit else deduped + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def atomic_write_json(path: Path, obj: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + f".tmp.{os.getpid()}") + tmp.write_text(json.dumps(obj, indent=2, sort_keys=True) + "\n") + tmp.replace(path) + + +def atomic_write_text(path: Path, data: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + f".tmp.{os.getpid()}") + tmp.write_text(data) + tmp.replace(path) + + +def load_json(path: Path) -> dict[str, Any] | None: + try: + return json.loads(path.read_text(errors="ignore")) + except Exception: + return None + + +def run_androguard(apk: Path, out_path: Path, timeout: int) -> dict[str, Any] | None: + if not shutil.which("uv"): + return { + "tool": "androguard", + "status": "skipped_no_uv", + "reason": "uv is required to run scripts/androguard_inventory.py (PEP 723 inline deps)", + } + script = Path(__file__).resolve().with_name("androguard_inventory.py") + if not script.exists(): + return { + "tool": "androguard", + "status": "skipped_script_missing", + "reason": f"androguard_inventory.py not found at {script}", + } + cmd = ["uv", "run", "--script", str(script), str(apk), "--out", str(out_path)] + try: + proc = subprocess.run( + cmd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired: + return {"tool": "androguard", "status": "timeout", "timeout_sec": timeout} + except Exception as exc: + return {"tool": "androguard", "status": "error", "error": str(exc)} + if proc.returncode != 0 or not out_path.exists(): + return { + "tool": "androguard", + "status": "error", + "returncode": proc.returncode, + "stderr": (proc.stderr or "")[-4000:], + } + try: + return json.loads(out_path.read_text(errors="ignore")) + except Exception as exc: + return { + "tool": "androguard", + "status": "error", + "error": f"could not parse androguard.json: {exc}", + } + + +def run_apkid(apk: Path, apkid_out: Path, timeout: int) -> dict[str, Any] | None: + if not shutil.which("apkid"): + return None + cmd = ["apkid", "-j", str(apk)] + try: + proc = subprocess.run( + cmd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + return { + "tool": "apkid", + "status": "timeout", + "timeout_sec": timeout, + "output": str(exc), + } + except Exception as exc: + return {"tool": "apkid", "status": "error", "error": str(exc)} + raw = proc.stdout or "" + parsed: Any + try: + parsed = json.loads(raw) + except Exception: + parsed = {"raw_output": raw[-12000:]} + result = { + "tool": "apkid", + "returncode": proc.returncode, + "status": "ok" if proc.returncode == 0 else "error", + "result": parsed, + } + atomic_write_json(apkid_out, result) + return result + + +def analyze_one(args: tuple[str, str, str, bool, int, bool]) -> dict[str, Any]: + apk_s, out_root_s, script_s, resume, timeout, include_apkid = args + apk = Path(apk_s) + out_root = Path(out_root_s) + script = Path(script_s) + started = time.time() + started_at = utc_now() + sha = "" + status: dict[str, Any] = { + "apk": str(apk), + "stage": "inventory", + "status": "started", + "started_at": started_at, + "tool": "extract_attack_surface.py", + } + try: + sha = sha256_file(apk) + artifact_dir = out_root / "apks" / sha + inventory_path = artifact_dir / "inventory.json" + status_path = artifact_dir / "status.json" + apkid_path = artifact_dir / "apkid.json" + status.update( + { + "sha256": sha, + "artifact_dir": str(artifact_dir), + "size": apk.stat().st_size, + } + ) + + if resume and inventory_path.exists() and status_path.exists(): + previous = load_json(status_path) or {} + if previous.get("status") == "ok": + inventory = load_json(inventory_path) or {} + return { + **status, + "status": "skipped", + "reason": "resume_existing_ok", + "finished_at": utc_now(), + "duration_sec": round(time.time() - started, 3), + "inventory": inventory, + } + + artifact_dir.mkdir(parents=True, exist_ok=True) + atomic_write_json(status_path, status) + tmp_jsonl = artifact_dir / "inventory.jsonl.tmp" + cmd = ["python3", str(script), str(apk), "--out", str(tmp_jsonl)] + try: + proc = subprocess.run( + cmd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + output = proc.stdout or "" + except subprocess.TimeoutExpired as exc: + output = exc.stdout or "" + if isinstance(output, bytes): + output = output.decode(errors="ignore") + status.update( + { + "status": "timeout", + "returncode": 124, + "output": output[-12000:], + "finished_at": utc_now(), + "duration_sec": round(time.time() - started, 3), + } + ) + atomic_write_json(status_path, status) + return status + + if proc.returncode != 0: + status.update( + { + "status": "error", + "returncode": proc.returncode, + "output": output[-12000:], + "finished_at": utc_now(), + "duration_sec": round(time.time() - started, 3), + } + ) + atomic_write_json(status_path, status) + return status + + lines = ( + [ + line + for line in tmp_jsonl.read_text(errors="ignore").splitlines() + if line.strip() + ] + if tmp_jsonl.exists() + else [] + ) + if not lines: + status.update( + { + "status": "error", + "returncode": proc.returncode, + "error": "inventory output was empty", + "output": output[-12000:], + "finished_at": utc_now(), + "duration_sec": round(time.time() - started, 3), + } + ) + atomic_write_json(status_path, status) + return status + inventory = json.loads(lines[0]) + inventory["artifact_dir"] = str(artifact_dir) + atomic_write_json(inventory_path, inventory) + with contextlib.suppress(OSError): + tmp_jsonl.unlink() + + apkid_result = None + if include_apkid: + apkid_result = run_apkid(apk, apkid_path, timeout=min(timeout, 120)) + if apkid_result: + inventory["apkid_status"] = apkid_result.get("status") + inventory["apkid_path"] = str(apkid_path) + atomic_write_json(inventory_path, inventory) + + androguard_path = artifact_dir / "androguard.json" + androguard_result = run_androguard( + apk, androguard_path, timeout=min(timeout, 180) + ) + if androguard_result: + inventory["androguard_status"] = androguard_result.get("status") or ( + "error" if androguard_result.get("error") else "ok" + ) + inventory["androguard_path"] = str(androguard_path) + inventory.setdefault("package", androguard_result.get("package")) + inventory.setdefault("version_name", androguard_result.get("version_name")) + inventory.setdefault("permissions", androguard_result.get("permissions")) + ag_schemes = androguard_result.get("schemes") or [] + ag_hosts = androguard_result.get("hosts") or [] + ag_browsable = androguard_result.get("browsable_components") or [] + ag_components = androguard_result.get("components") or [] + if ag_schemes: + inventory["schemes"] = ag_schemes + if ag_hosts: + inventory["hosts"] = ag_hosts + if ag_browsable: + inventory["browsable_components"] = ag_browsable + if ag_components: + inventory["components"] = ag_components + atomic_write_json(inventory_path, inventory) + + # Re-rank with APKiD packer/protector signal applied. + # extract_attack_surface.summarize_packers + rank consume `apkid_summary` + # and apply heavy/medium/ambiguous penalties. Reading the apkid.json on + # disk is the canonical source (handles resume runs where apkid_result + # is None because the per-APK invocation was skipped earlier). + if _eas is not None: + apkid_disk: dict[str, Any] | None = None + if apkid_path.exists(): + try: + apkid_disk = json.loads(apkid_path.read_text(errors="ignore")) + except json.JSONDecodeError: + apkid_disk = None + try: + inventory["apkid_summary"] = _eas.summarize_packers( + apkid_disk or apkid_result + ) + inventory = _eas.rank(inventory) + except (AttributeError, TypeError, ValueError) as exc: + # Ranking is a refinement; never block on a malformed apkid + # record. Surface the cause so a real summarize_packers / + # rank bug doesn't hide silently. + inventory.setdefault("rank_warnings", []).append( + f"apkid_rank_failed: {exc}" + ) + atomic_write_json(inventory_path, inventory) + + status.update( + { + "status": "ok", + "returncode": proc.returncode, + "output": output[-4000:], + "finished_at": utc_now(), + "duration_sec": round(time.time() - started, 3), + "inventory_path": str(inventory_path), + } + ) + atomic_write_json(status_path, status) + return {**status, "inventory": inventory} + except Exception as exc: + artifact_dir = out_root / "apks" / sha if sha else out_root / "errors" + status.update( + { + "status": "error", + "error": str(exc), + "finished_at": utc_now(), + "duration_sec": round(time.time() - started, 3), + } + ) + atomic_write_json(artifact_dir / "status.json", status) + return status + + +def compact_preview(record: dict[str, Any]) -> dict[str, Any]: + inv = ( + record.get("inventory") if isinstance(record.get("inventory"), dict) else record + ) + return { + "status": record.get("status"), + "apk": inv.get("apk") or record.get("apk"), + "sha256": inv.get("sha256") or record.get("sha256"), + "package": inv.get("package"), + "size": inv.get("size") or record.get("size"), + "semantic_priority": inv.get("semantic_priority"), + "schemes_count": len( + inv.get("schemes") + or inv.get("manifest_string_hints", {}).get("scheme_hints", []) + or [] + ), + "hosts_count": len( + inv.get("hosts") + or inv.get("manifest_string_hints", {}).get("host_hints", []) + or [] + ), + "urls_count": len(inv.get("urls") or []), + "artifact_dir": inv.get("artifact_dir") or record.get("artifact_dir"), + } + + +def main() -> int: + ap = argparse.ArgumentParser( + description="Run parallel resumable first-pass APK corpus inventory" + ) + ap.add_argument( + "paths", nargs="+", type=Path, help="APK files or directories containing APKs" + ) + ap.add_argument("--out-dir", type=Path, required=True, help="Output run directory") + ap.add_argument( + "--jobs", + type=int, + default=max(1, min(4, (os.cpu_count() or 2) // 2)), + help="Parallel worker count", + ) + ap.add_argument( + "--resume", action="store_true", help="Skip APKs with existing ok status" + ) + ap.add_argument( + "--timeout", type=int, default=180, help="Per-APK inventory timeout in seconds" + ) + ap.add_argument("--limit", type=int, help="Limit number of APKs for smoke testing") + ap.add_argument( + "--include-apkid", action="store_true", help="Run APKiD when installed" + ) + ap.add_argument( + "--preview", + type=int, + default=20, + help="Number of compact preview records in summary", + ) + args = ap.parse_args() + + out_dir = args.out_dir.expanduser().resolve() + out_dir.mkdir(parents=True, exist_ok=True) + script = Path(__file__).resolve().with_name("extract_attack_surface.py") + apks = iter_apks(args.paths, limit=args.limit) + run_started = time.time() + summary: dict[str, Any] = { + "started_at": utc_now(), + "out_dir": str(out_dir), + "apk_count": len(apks), + "jobs": args.jobs, + "resume": args.resume, + "timeout_sec": args.timeout, + "include_apkid": args.include_apkid, + "tools": { + "aapt": shutil.which("aapt"), + "aapt2": shutil.which("aapt2"), + "apkid": shutil.which("apkid"), + "uv": shutil.which("uv"), + "androguard_inventory_script": str( + Path(__file__).resolve().with_name("androguard_inventory.py") + ), + }, + } + atomic_write_json(out_dir / "run_status.json", {**summary, "status": "running"}) + + task_args = [ + ( + str(apk), + str(out_dir), + str(script), + args.resume, + args.timeout, + args.include_apkid, + ) + for apk in apks + ] + + # Stream JSONL output to disk as workers finish so resident memory stays at + # O(jobs * one_inventory) instead of O(corpus_size). Keep only a bounded heap + # of compact previews; full records live on disk. + surface_path = out_dir / "attack_surface.jsonl" + status_path = out_dir / "status.jsonl" + surface_tmp = surface_path.with_suffix(surface_path.suffix + f".tmp.{os.getpid()}") + status_tmp = status_path.with_suffix(status_path.suffix + f".tmp.{os.getpid()}") + surface_path.parent.mkdir(parents=True, exist_ok=True) + + import heapq + + counts: dict[str, int] = {} + top_heap: list[tuple[int, int, int, dict[str, Any]]] = [] + preview_n = max(1, args.preview) + idx = 0 + + def _drain(fut_result: dict[str, Any], surface_fh, status_fh) -> None: + nonlocal idx + idx += 1 + st = fut_result.get("status") + counts[str(st)] = counts.get(str(st), 0) + 1 + inventory = ( + fut_result.get("inventory") + if isinstance(fut_result.get("inventory"), dict) + else None + ) + if inventory: + surface_fh.write(json.dumps(inventory, sort_keys=True)) + surface_fh.write("\n") + score = int(((inventory.get("semantic_priority") or {}).get("score")) or 0) + urls_n = len(inventory.get("urls") or []) + preview = compact_preview({"status": "ok", "inventory": inventory}) + heapq.heappush(top_heap, (score, urls_n, -idx, preview)) + if len(top_heap) > preview_n: + heapq.heappop(top_heap) + status_only = {k: v for k, v in fut_result.items() if k != "inventory"} + status_fh.write(json.dumps(status_only, sort_keys=True)) + status_fh.write("\n") + apk = fut_result.get("apk") or (inventory or {}).get("apk") + print(f"{st}\t{apk}", file=sys.stderr, flush=True) + + with ( + surface_tmp.open("w") as surface_fh, + status_tmp.open("w") as status_fh, + cf.ProcessPoolExecutor(max_workers=max(1, args.jobs)) as pool, + ): + in_flight: dict[Any, Any] = {} + max_inflight = max(args.jobs * 2, args.jobs + 2) + it = iter(task_args) + for t in it: + in_flight[pool.submit(analyze_one, t)] = t + if len(in_flight) >= max_inflight: + break + while in_flight: + done_set, _ = cf.wait(in_flight.keys(), return_when=cf.FIRST_COMPLETED) + for fut in done_set: + in_flight.pop(fut, None) + _drain(fut.result(), surface_fh, status_fh) + for t in it: + in_flight[pool.submit(analyze_one, t)] = t + if len(in_flight) >= max_inflight: + break + + surface_tmp.replace(surface_path) + status_tmp.replace(status_path) + + top_sorted = sorted(top_heap, key=lambda x: (-x[0], -x[1])) + final_summary = { + **summary, + "status": "done", + "finished_at": utc_now(), + "duration_sec": round(time.time() - run_started, 3), + "counts": counts, + "attack_surface_jsonl": str(surface_path), + "status_jsonl": str(status_path), + "preview": [item[3] for item in top_sorted], + } + atomic_write_json(out_dir / "run_status.json", final_summary) + print(json.dumps(final_summary, indent=2, sort_keys=True)) + return 0 if counts.get("error", 0) == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/capabilities/android-apk-research/skills/android-corpus-prep/SKILL.md b/capabilities/android-apk-research/skills/android-corpus-prep/SKILL.md new file mode 100644 index 0000000..65a7e1e --- /dev/null +++ b/capabilities/android-apk-research/skills/android-corpus-prep/SKILL.md @@ -0,0 +1,230 @@ +--- +name: android-corpus-prep +description: "Use when preparing Android APK corpora — AndroZoo metadata download, popular-package selection via DuckDB, APK download from Play or AndroZoo, and per-pass provenance manifests." +allowed-tools: + - run_corpus_inventory + - bash + - read + - grep + - glob + - web_search + - web_extract + - report +license: MIT +--- + +# Android corpus preparation + +Use this skill when the user needs APKs to analyze, wants a popular/common Android app corpus, or asks how to select targets from AndroZoo / F-Droid / device extraction. The flow is **scripts + DuckDB on Parquet** end-to-end; the only LLM tool in this skill is `run_corpus_inventory` (called after APKs are on disk). + +## Outcome + +Produce a reproducible corpus manifest that answers: + +- where each package/APK came from +- why it was selected +- which exact SHA256/version was downloaded +- what authorization/provenance constraints apply +- where the APK files and manifests live + +## Corpus layout (mandatory) + +Organize every corpus pass as a self-contained directory under `corpus/passes/`. The pass directory — not the downloader — owns the APKs. + +``` +corpus/ +├── shared/ +│ └── androzoo-meta/ # AndroZoo metadata (gz + parquet), shared across passes +├── selections/ # Exploratory target lists not tied to any pass +└── passes/ + └── corpus-N/ + ├── selection.jsonl # Input target list + ├── manifest-gplaydl.jsonl # Per-source download manifests + ├── manifest-androzoo.jsonl # (only sources actually used) + ├── apks/ # All APKs for this pass, regardless of source + ├── logs/ # Downloader stdout/stderr, smoke runs, .tmp files + └── README.md # What the pass is and how to feed it forward +``` + +Rules: + +- **One pass = one directory.** Never split APKs from the same pass across multiple folders based on which downloader produced them. +- `apks/` is the only place APKs live. `source` (`gplaydl`, `androzoo`, ...) is recorded as a field in the manifest row, not in the path. +- AndroZoo metadata files (`*.csv.gz`, `*.jsonl.gz`, Parquet) live under `corpus/shared/androzoo-meta/` — they are corpus-independent and 9 GB+. +- Exploratory target lists that don't correspond to a real download pass go to `corpus/selections/`. +- Manifest `path` fields must point at `corpus/passes/corpus-N/apks/...`. + +## AndroZoo metadata-first path + +This is the preferred path for popular/common APK hunting. + +### 1. Fetch the metadata sources + +```bash +# Google Play metadata aggregate (~1.3 GB gzipped JSONL). +python3 scripts/androzoo_gp_metadata.py download \ + --kind aggregate \ + --out corpus/shared/androzoo-meta/gp-metadata-aggregate.jsonl.gz \ + --progress + +# AndroZoo APK index (~3.6 GB gzipped CSV). +curl -L -o corpus/shared/androzoo-meta/latest_with-added-date.csv.gz \ + https://androzoo.uni.lu/static/lists/latest_with-added-date.csv.gz +``` + +### 2. Convert both sources to ZSTD Parquet (one-time) + +The shipped sources are large gzipped files you will join against repeatedly during corpus selection. Convert them to Parquet once. Working set drops 5–10× and DuckDB pushes predicates and column projections into the scan. + +```bash +# ~80s for the JSONL, ~30s for the CSV on a Mac M-series. +python3 scripts/androzoo_to_parquet.py csv \ + corpus/shared/androzoo-meta/latest_with-added-date.csv.gz \ + corpus/shared/androzoo-meta/parquet/androzoo_latest.parquet + +python3 scripts/androzoo_to_parquet.py json \ + corpus/shared/androzoo-meta/gp-metadata-aggregate.jsonl.gz \ + corpus/shared/androzoo-meta/parquet/androzoo_gp_metadata.parquet +``` + +What the script does that matters: + +- CSV path runs the DuckDB CLI through a named pipe so the 7 GB decompressed CSV is never written to disk. Output ~3 GB Parquet. +- JSONL path uses **streaming pyarrow** with an **explicit schema**. DuckDB's `read_json` schema sniffer OOMs at 6 GB on the 1.3 GB source — never sniff a multi-GB JSONL. Nested `related_apks_in_AZ_info` is JSON-stringified into a single TEXT column to preserve round-trip without pulling List typing in. + +### 3. Materialize a popular-clean candidate set once per refresh + +Run the popular-package selection + clean-Play-APK join straight in DuckDB. On a ~15k popular-package candidate set this is seconds with <200 MB peak RAM, and the output Parquet is ~1.3 MB — re-queryable in milliseconds: + +```bash +duckdb -c " +COPY ( + WITH popular AS ( + SELECT pkg_name, max_nb_downloads AS dl, max_star_rating AS rating, max_ratingsCount AS ratings + FROM 'corpus/shared/androzoo-meta/parquet/androzoo_gp_metadata.parquet' + WHERE max_nb_downloads >= 10000000 + AND max_star_rating >= 3.5 + AND max_ratingsCount >= 1000 + ), + candidates AS ( + SELECT + l.sha256, l.pkg_name, l.vercode, l.apk_size, l.markets, l.added, l.vt_detection, l.dex_date, + p.dl, p.rating, p.ratings, + ROW_NUMBER() OVER (PARTITION BY l.pkg_name ORDER BY l.added DESC NULLS LAST, l.vercode DESC) AS rn + FROM 'corpus/shared/androzoo-meta/parquet/androzoo_latest.parquet' l + JOIN popular p USING (pkg_name) + WHERE l.markets LIKE '%play.google.com%' + AND coalesce(try_cast(l.vt_detection AS INTEGER), 0) <= 0 + AND coalesce(try_cast(l.apk_size AS BIGINT), 0) BETWEEN 1000000 AND 200000000 + ) + SELECT * EXCLUDE rn FROM candidates WHERE rn = 1 +) TO 'corpus/shared/androzoo-meta/parquet/popular_latest_clean.parquet' (FORMAT PARQUET, COMPRESSION ZSTD); +" +``` + +Use this as the canonical input for per-corpus filter passes (banking, messaging, healthcare, etc.). Cuts the cost of "try a different filter" from 30s to 50ms. + +### 4. Per-pass filter → selection.jsonl + +Pick the subset for this pass with a focused DuckDB query against the canonical candidate set. Examples: + +```bash +# Wallets — selection.jsonl with the columns the downloaders need. +# Tune the LIKE / IN lists for the target class; the example below is purely illustrative. +duckdb -c " +COPY ( + SELECT pkg_name AS package, sha256, vercode AS version_code, apk_size, dl, rating + FROM 'corpus/shared/androzoo-meta/parquet/popular_latest_clean.parquet' + WHERE lower(pkg_name) LIKE ANY ('%wallet%','%crypto%','%coin%','%defi%') + ORDER BY dl DESC + LIMIT 30 +) TO 'corpus/passes/corpus-wallet/selection.jsonl' (FORMAT JSON); +" +``` + +### 5. Download the APKs + +```bash +mkdir -p corpus/passes/corpus-wallet/{apks,logs} +``` + +**Primary — `gplaydl_bulk.py` for any selection that targets currently-listed Play apps.** Roughly 30–60× faster than AndroZoo per APK and needs no API key (anonymous Google Play auth via [Aurora Store](https://auroraoss.com/)'s public token dispenser): + +```bash +python3 scripts/gplaydl_bulk.py \ + corpus/passes/corpus-wallet/selection.jsonl \ + --out-dir corpus/passes/corpus-wallet/apks \ + --manifest-out corpus/passes/corpus-wallet/manifest-gplaydl.jsonl \ + --jobs 8 +``` + +The selection JSONL needs at minimum `{"package": "com.foo"}` per line; the script ignores `sha256` because Google Play serves the current version directly. Failures are written to the manifest; common errors are "App not found" (delisted) and "No download URL returned" (region-locked or paid). + +**Fallback — `androzoo_download.py` for delisted, paid, or AndroZoo-only historical versions.** `--out-dir` is the SAME `apks/` directory as the gplaydl primary; the pass directory is the unit, not the downloader. + +```bash +python3 scripts/androzoo_download.py \ + corpus/passes/corpus-wallet/fallback-androzoo-selection.jsonl \ + --out-dir corpus/passes/corpus-wallet/apks \ + --manifest-out corpus/passes/corpus-wallet/manifest-androzoo.jsonl \ + --jobs 10 +``` + +AndroZoo throttles each connection to ~440 KB/s but allows ~20 concurrent. The default `--jobs 12` is a safe sweet spot. Use `--api-key-file` or `ANDROZOO_API_KEY` env var; never inline the key. + +**Empirical comparison on an 89-package, 6.8 GB corpus:** + +| Path | Wall time | Speed | +|---|---|---| +| `gplaydl_bulk` jobs=6 (78 succeed) + AndroZoo fallback (10 left) | ~10 min total | ~12 MB/s aggregate | +| Old serial AndroZoo only | ~3 h | ~440 KB/s | + +See `../android-semantic-vuln-hunting/references/corpus-acquisition.md` for the full source-by-source landscape, terms, and tradeoffs. + +## Other corpus sources + +Read `../android-semantic-vuln-hunting/references/corpus-acquisition.md` before recommending a source. Short version: + +- User-provided/internal APKs are best for actionable findings. +- Official device extraction is best for curated popular apps when authorization/terms allow. +- F-Droid is best for clean smoke testing and reproducible open-source samples. +- APK mirrors are fallback only; record URL, timestamp, version, SHA256, and signature/provenance caveats. + +## Selection bias for semantic bugs + +Prioritize packages with likely high-impact logic: + +- account/login/session flows +- OAuth, SSO, identity, wallet, payments +- retail/travel/loyalty/telecom/health/enterprise SaaS +- WebView-heavy hybrid flows +- partner/campaign/shortlink/deferred deep link SDKs +- file sharing, messaging, documents, support chat +- React Native, Flutter, Cordova, Capacitor bridges + +Avoid spending first-pass cycles on simple utilities, games, launchers, or static content unless metadata/inventory suggests rich entrypoints. + +## Output manifest + +For every corpus pass, preserve JSONL like: + +```json +{"package":"com.example","title":"Example","selection_reason":"wallet keyword + high downloads","source":"androzoo","sha256":"...","version_code":"123","path":"corpus/passes/corpus-N/apks/com.example_123_deadbeef.apk","selection":"corpus/passes/corpus-N/selection.jsonl"} +``` + +`source` is the downloader (`gplaydl`, `androzoo`, ...) — it's metadata on the row, not a directory in the path. Use the manifest as the entrypoint for later inventory, decompilation, and reporting. + +## Hand off to inventory + +Once APKs are on disk, run `run_corpus_inventory` to produce SHA256-keyed artifact directories with Androguard manifest decoding and optional APKiD packer detection. That output drives the semantic-vuln-hunting pipeline. + +```python +run_corpus_inventory( + paths=["corpus/passes/corpus-N/apks"], + out_dir="findings/corpus-N/inventory", + jobs=8, + include_apkid=True, +) +``` + +One pass directory → one inventory directory. Don't mix passes in a single inventory unless you've thought through how findings will attribute back. diff --git a/capabilities/android-apk-research/skills/android-protector-triage/SKILL.md b/capabilities/android-apk-research/skills/android-protector-triage/SKILL.md new file mode 100644 index 0000000..aa4951a --- /dev/null +++ b/capabilities/android-apk-research/skills/android-protector-triage/SKILL.md @@ -0,0 +1,137 @@ +--- +name: android-protector-triage +description: "Use when an Android APK is packed by a commercial protector — detect with apkid + protector_detect, structurally unpack DexProtector (arm64) where supported, and triage what survives in JADX. Routes between detection, optional unpack, and protector-aware adjacency analysis." +allowed-tools: + - detect_protector + - dexprotector_unpack + - bash + - read + - grep + - glob + - web_search + - web_extract + - report +license: MIT +--- + +# Android protector-aware triage + +Grounding: +- OWASP MASVS-RESILIENCE (Resilience Against Reverse Engineering and Tampering): https://mas.owasp.org/MASVS/09-MASVS-RESILIENCE/ +- MASTG-TECH-0019 Reverse Engineering — Android dynamic analysis pre-flight: https://mas.owasp.org/MASTG/techniques/android/MASTG-TECH-0019/ +- MASTG-TEST-0089 Testing Resiliency Against Reverse Engineering: https://mas.owasp.org/MASTG/tests/android/MASVS-RESILIENCE/MASTG-TEST-0089/ + +Trigger: the regular targeted-assessment / semantic-vuln-hunting flow runs into one or more of these symptoms: + +- `apkid.json` flags a known commercial protector +- `lib//libdpboot.so` + `libdexprotector.so`, or analogous protector libs, are present +- `assets/` contains opaque high-entropy `.dat` files (`se.dat`, `classes.dex.dat`, `mm.dat`, `dp.mp3`, …) +- JADX decompiles a `Protected` Application class that immediately `System.loadLibrary` a small native blob +- `classes.dex` is conspicuously small relative to the app's apparent complexity + +When any of these fire, **stop the regular workflow** and run this skill first. Continuing without protector-awareness produces false negatives (scanners see no exposed code) and false confidence (scanner baselines look clean because most code is encrypted). + +**Scope as shipped:** DexProtector is the only protector with a structural unpack path. Promon Shield static recovery exists in `scripts/research/promon/promon_*` but is research-grade and not wired into this skill — see `references/promon-shield.md` for the standalone runbook. Other protectors (AppSealing, BoxedApp, AppGuard, Jiagu/360, …) are detected only as adjacency signals; triage them as black boxes per §3. + +## 0. Identify the protector + +```python +detect_protector(target="path/to.apk", out="findings//protector.json") +``` + +Read the returned dict (also written to `out`). Key fields: + +- `protector` — `dexprotector` (structural unpack supported) or `promon_shield` (detection only; static recovery is research-grade, see `references/promon-shield.md`); other names fall through to adjacency analysis +- `confidence` (`high` / `medium` / `low`) +- `signals.native_libs`, `signals.protected_assets`, `signals.dplf` — concrete evidence +- `triage_strategy` (`protector_aware` vs `default`) +- `artifacts.dexprotector_unpack_supported` (true when arm64-v8a libdexprotector is present) +- `notes` — strategy hints + +If the detector returns `protector: unknown` but the user has strong contextual evidence of a different protector, file the gap in `references/` and proceed with **only the regular targeted-assessment workflow**, treating the encrypted assets as black boxes. + +## 1. Structural unpack — DexProtector + +Only run this if `detect_protector` reported `protector: dexprotector` AND `artifacts.dexprotector_unpack_supported: true`. + +```python +dexprotector_unpack(apk="path/to.apk", out="findings//libdp.so") +``` + +This emulates the bootstrap chain (libdexprotector.so → libdp.so) without an Android device. It does **not** execute libdp.so; everything is static + Unicorn. It does **not** trigger the master-key corruption that the Romain Thomas writeup describes (see `references/sources.md`), because libdp.so itself is never hooked. + +The produced `libdp.so` is a synthetic AArch64 ET_DYN ELF; pass it directly to Ghidra / radare2 / Binary Ninja for the next step. Strings include the BoringSSL algorithm table, NDK AssetManager imports, and RASP plumbing — see `references/dexprotector.md`. + +Limitations: arm64-v8a only today. For 32-bit-only ABI variants, port the offsets table at the top of `scripts/dexprotector_unpack.py` (the DPLF watermark is the anchor). + +## 2. Protector-aware decompile + +Decompile the APK **with the expectation that 60–95% of first-party code is missing**: + +```bash +JAVA_OPTS="-Xmx2g" jadx --show-bad-code --no-debug-info \ + -d findings//sources path/to.apk +``` + +What survives in JADX output: + +- `Protected` bootstrap class and its small set of native entry points +- public manifest-declared components (intents/activities are still readable) +- third-party libraries the app vendored *before* protection (these are usually ``-excluded; AppCloner-injected code has been observed surviving in protected samples) +- string-encryption call sites: `ProtectedFoo.s("\uXXXX")` shims + +What does NOT survive: + +- the bulk of first-party code (packed inside `assets/classes.dex.dat`) +- string literals (replaced by `s(...)` calls keyed on `assets/se.dat`) +- field/method references (replaced by an indirection table — the `ha.i(int, ...)` / `Lib.i(int, ...)` pattern documented in the Romain Thomas DexProtector writeup; see `references/sources.md`) + +ripgrep / Semgrep over JADX output is therefore **adjacency analysis only** — it can find the bootstrap, native entry points, manifest-anchored components, and the protector-injected call site patterns. It cannot find first-party logic bugs the way it would on an unprotected app. + +## 3. What you can still report (without unpacking the DEX) + +Even before recovering plaintext DEX, you can build evidence-backed findings on: + +- **Cryptographic posture of the unprotected layer.** DexProtector's `` filters routinely exclude vendored third-party crypto utility classes; weak primitives (DES/ECB, MD5, hardcoded keys, plain-HTTP base URLs) shipped in those excluded classes are visible in JADX output and worth flagging. The Romain Thomas writeup (see `references/sources.md`) documents one ~10M-installs sample (`com.flashget.parentalcontrol`) where this exact shape applied. +- **Manifest-level attack surface.** Exported components, BROWSABLE intent filters, schemes/hosts, dangerous permissions — DexProtector does not rewrite the manifest beyond injecting the bootstrap class. +- **Network config.** `network_security_config.xml`, `usesCleartextTraffic`, certificate pinning configuration — all outside the protector. +- **Protected-asset fingerprint.** Hashes of `assets/*.dat` files are stable across runs and give a per-version identifier; useful for diffing across versions or comparing apps to detect shared keystores. + +## 4. Static asset RE — what to attempt next + +The unpacked `libdp.so` is the entry point to all of these. They are not yet wired in this capability: + +| Goal | Where in libdp.so | Difficulty | +| --- | --- | --- | +| Master-key derivation | functions that read the APK's v1/v2 signature block + iterate `classes.dex` + reference embedded config | high (post documents inputs but not the KDF) | +| `assets/classes.dex.dat` unpack | the EVP_aead_* call chain reachable from `AAssetManager_open` hooks; the trailer at the end of the file describes anti-dump unmap windows | medium-high | +| `assets/se.dat` string decrypt | string-decrypt native shim reachable from `s(String)` JNI entrypoint | medium (the post specifies the inputs: index + calling class hash code) | +| Generic asset decrypt | the vtable-hook entry point for `android::_FileAsset::*` | medium | + +For each, plan the work in `references/dexprotector.md` and add a script under `scripts/`. + +## 5. Dynamic validation — only if explicitly authorized + +The post's key insight is that the DexProtector key derivation is bound to APK signature + unprotected DEX + libdp.so itself, AND that hooking libdp.so corrupts the master key. Consequences: + +- **frida hooks against libdp.so are detected at start-up** and silently corrupt the asset-decryption path. Do not attempt dynamic instrumentation as a first-pass discovery technique. +- **frida-server's `rtld_db_dlactivity` trampoline is persistent**: a device that has *ever* run frida-server keeps the trampoline overlay on `rtld_db_dlactivity`. DexProtector incorporates the first 4 bytes of that function into its payload key. Consequence: on a frida-tainted device the unpacker hardcoded into libdexprotector.so will decrypt the payload incorrectly and the app will appear broken in ways that look unrelated to instrumentation. Use a clean device or our static unpacker. +- For validation, prefer environments where you have not previously installed frida; or use a fresh emulator image; or use the static unpacker output as ground truth. + +## 6. Hand-off back to the standard workflow + +Once you've labelled what's reachable statically and recovered any classes.dex.dat / asset content you can, hand back to `android-targeted-assessment` (or `android-semantic-vuln-hunting` for scaled-out work) with: + +- `findings//protector.json` +- `findings//libdp.so` (and Ghidra / IDA project files) +- `findings//decrypted/` for every asset you've successfully decrypted +- a list of ``-protected packages you've recovered DEX for + +Then run the regular rg / Semgrep / Joern pipeline on the **union** of (JADX sources, decrypted DEX disassembly). Mark every hypothesis `scanner_gap = adjacent` or `not found` — by construction scanners had no access to the protected code, so they cannot have an `exact` finding inside it. + +## Reference material + +- `../../references/sources.md` — capability-root citation registry (Romain Thomas DexProtector writeup, APKiD rules, Promon community RE, 34C3 talk, DIMVA 2018 paper). +- `references/dexprotector.md` — IoCs, file glossary, RE notes against the LiveNet sample, recovered key, payload format. +- `references/promon-shield.md` — research-grade Promon Shield recovery runbook (not wired into this skill workflow). +- `references/appsealing-doverunner.md`, `references/jiagu-360.md` — adjacency notes for other commercial protectors; detection only, no in-tree recovery. diff --git a/capabilities/android-apk-research/skills/android-protector-triage/references/appsealing-doverunner.md b/capabilities/android-apk-research/skills/android-protector-triage/references/appsealing-doverunner.md new file mode 100644 index 0000000..1eadb77 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-protector-triage/references/appsealing-doverunner.md @@ -0,0 +1,140 @@ +# AppSealing / DoveRunner triage reference + +Status: early triage scaffold. This is the starting point for adding AppSealing detection and later building an unprotection flow when we source a representative sample. + +Primary sources to read first: + +- DoveRunner / AppSealing Android app security page: +- AppSealing Android obfuscation page, currently redirecting in some environments: +- APKiD APK rules for AppSealing: +- APKiD ELF rules for AppSealing core version strings: + +## Identification (Tier 1) + +AppSealing / DoveRunner is a commercial **app wrapping / sealing / RASP** target. Vendor material describes a post-build flow: upload application, apply security features, download sealed app, publish. Expect added native libraries, sealed DEX/assets, and runtime checks. + +| Surface | Signal | Source / confidence | +| --- | --- | --- | +| `apkid.json` | `packer: AppSealing` | High when APKiD hits. | +| `lib//` | `libcovault.so` | High; APKiD APK rule. | +| `lib//` | `libcovault-appsec.so` | High; APKiD APK rule. | +| `assets/` | `assets/appsealing.dex` | High; APKiD APK rule. | +| `assets/` | `assets/sealed1.dex` | High; APKiD APK rule. | +| `assets/` | `assets/AppSealing/*` with multiple entries | High; APKiD alternate rule. | +| ELF strings | `APPSEALING-CORE-VERSION_2.10.10` or similar | Medium-high; APKiD ELF rule for a known version. | + +### Current corpus status + +No AppSealing hit was observed in the current local APKiD corpus slice during early ranking. That means this file is detection/research prep first; we still need a representative sample before reproducing an unprotection flow. + +Useful sample acquisition criteria: + +- APKiD hit for `AppSealing`. +- At least one `arm64-v8a` protected native library. +- Presence of `assets/appsealing.dex` and `assets/sealed1.dex`, or an `assets/AppSealing/` directory. +- Prefer a benign/free app or test/demo artifact that can be redistributed internally as a fixture. + +## Expected protection shape + +Working hypothesis from APKiD rules and vendor material: + +``` +Original APK + -> AppSealing post-build wrapper adds native covault/appsec libraries + -> wrapper adds appsealing.dex / sealed*.dex / AppSealing asset bundle + -> launch path initializes AppSealing runtime + -> runtime performs code protection, integrity checks, anti-debug, root/emulator, memory access, network sniffing, and cheat-tool checks + -> sealed code/assets are loaded or mediated by the wrapper +``` + +Vendor page claims code protection via obfuscation and encryption, binary/resource integrity protection, anti-debugging, memory access detection, emulator blocking, rooting detection, and network packet sniffing detection. + +Key expectation for triage: AppSealing may sit between DexProtector-style DEX packing and Promon-style native RASP. Do not assume all first-party DEX is encrypted until the sample proves it. The `sealed1.dex` / `appsealing.dex` split should be mapped first. + +## Initial static triage workflow + +For a candidate APK: + +```bash +python3 scripts/protector_detect.py path/to.apk -o findings//protector.json +apkid path/to.apk -j > findings//apkid.json +unzip -l path/to.apk > findings//ziplisting.txt +``` + +Extract AppSealing artifacts: + +```bash +mkdir -p findings//appsealing +unzip -p path/to.apk assets/appsealing.dex > findings//appsealing/appsealing.dex 2>/dev/null || true +unzip -p path/to.apk assets/sealed1.dex > findings//appsealing/sealed1.dex 2>/dev/null || true +unzip -p path/to.apk lib/arm64-v8a/libcovault.so > findings//appsealing/libcovault-arm64.so 2>/dev/null || true +unzip -p path/to.apk lib/arm64-v8a/libcovault-appsec.so > findings//appsealing/libcovault-appsec-arm64.so 2>/dev/null || true +``` + +Minimum evidence to capture: + +- Which APKiD AppSealing rule fired (`appsealing`, `appsealing_a`, or ELF core version). +- Exact native library names and ABI coverage. +- Whether `assets/appsealing.dex`, `assets/sealed1.dex`, or `assets/AppSealing/*` are present. +- Sizes/entropy of sealed DEX and AppSealing assets. +- Whether `sealed1.dex` begins with `dex\n` or is encrypted/encoded. +- Manifest/Application changes and Java stub entrypoints. +- Native load sites and classloader usage. + +## Unprotection-flow hypotheses + +| Goal | Starting point | Difficulty | Notes | +| --- | --- | --- | --- | +| Detect AppSealing reliably | APKiD APK rules: covault libs + appsealing/sealed DEX assets | Low | Add direct ZIP-name support to `protector_detect.py`. | +| Map wrapper bootstrap | Manifest Application + `appsealing.dex` decompile | Medium | Determine how control transfers to original app. | +| Classify `sealed1.dex` | Header/entropy/checksum and JADX/dexdump behavior | Low-medium | It may be a real DEX, encrypted DEX, or wrapper metadata depending on version. | +| Recover original app code | AppSealing loader xrefs to sealed assets / classloader / native decrypt | Medium-high | Need sample-specific RE; compare with DexProtector asset-load methodology. | +| Recover policy/config | `assets/AppSealing/*`, native strings, appsealing.dex constants | Medium | Build an asset glossary once a sample is available. | +| Neutralize runtime checks | Native covault/appsec checks and Java stubs | Medium-high | RASP checks may be independent from DEX recovery. | + +## Dynamic validation cautions + +Only do this with explicit authorization for the target sample. + +- Expect anti-debugging, emulator, root, memory access, and packet-sniffing checks; these are vendor-advertised features. +- Establish a clean baseline launch before adding instrumentation. +- Test whether repackaging to add debug flags invalidates integrity checks. +- If dynamic DEX dumping is attempted, first determine whether sealed code is loaded through standard classloaders or native runtime mapping. + +## Detector implementation notes + +Add an AppSealing branch to `scripts/protector_detect.py` with: + +- ZIP-name scan for: + - `lib//libcovault.so` + - `lib//libcovault-appsec.so` + - `assets/appsealing.dex` + - `assets/sealed1.dex` + - `assets/AppSealing/*` +- ELF string scan for: + - `APPSEALING-CORE-VERSION_` +- Confidence model: + - high: covault libs plus appsealing/sealed DEX, or APKiD hit. + - medium: `assets/AppSealing/` with several entries, or ELF core version string. + - low: only marketing-ish names without covault/sealed artifacts. + +Suggested normalized output: + +```json +{ + "protector": "appsealing_doverunner", + "triage_strategy": "protector_aware_app_wrapping", + "signals": { + "native_libs": {"arm64-v8a": ["libcovault.so", "libcovault-appsec.so"]}, + "protected_assets": ["assets/appsealing.dex", "assets/sealed1.dex"] + } +} +``` + +## Open questions for the first research pass + +1. Can we source a current benign AppSealing-protected APK with `arm64-v8a` support? +2. Does `sealed1.dex` contain a valid DEX header in current builds, or is it encrypted until runtime? +3. Is the original application class named in manifest metadata, an AppSealing config asset, or a native table? +4. Which runtime checks are Java-level vs native-level? +5. Is a static asset decryptor plausible, or is the first useful flow a dynamic loader trace + DEX dump/repair pipeline? diff --git a/capabilities/android-apk-research/skills/android-protector-triage/references/dexprotector.md b/capabilities/android-apk-research/skills/android-protector-triage/references/dexprotector.md new file mode 100644 index 0000000..04740a9 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-protector-triage/references/dexprotector.md @@ -0,0 +1,151 @@ +# DexProtector triage reference + +Source: Romain Thomas, *A Glimpse Into DexProtector*, 2026-01-04. + + +Grounding: this work sits under OWASP MASVS-RESILIENCE (); see also MASTG-TEST-0089 *Testing Resiliency Against Reverse Engineering* () — DexProtector is a vendor implementation of the RASP / packing controls MASVS-RESILIENCE describes. + +This page is the IoC + RE notes the in-tree DexProtector detector + unpacker (`scripts/protector_detect.py`, `scripts/dexprotector_unpack.py`) are built from. It is **not** a copy of the blog post — it captures the facts we needed to make the workflow reproducible inside the capability, including offsets and routines we verified by re-doing the RE in Ghidra against LiveNet `com.playnet.androidtv.ads-5.0.1` (`sha256:810634a3…`) and the envchecks sample (`com.dexprotector.detector.envchecks-2.1`). + +## Identification (Tier 1) + +Any of these is enough on its own; combinations raise confidence. + +| Surface | Signal | +| --- | --- | +| `AndroidManifest.xml` | `` — DexProtector injects a bootstrap subclass of the original `Application`. | +| `lib//` | `libdpboot.so` AND `libdexprotector.so` (or `libdexprotector_h.so`) shipped per ABI. | +| `libdexprotector.so` body | `DPLF` (`44 50 4c 46`) magic. In modern builds this is the *first 4 bytes* of the encrypted payload inside the last `PT_LOAD` (file offset `0x3ac0` in the LiveNet arm64 build). Older builds may place the payload at the end of the file outside a segment. | +| `assets/` | Any of: `se.dat`, `classes.dex.dat`, `mm.dat`, `dp.mp3`, `resources.dat`, `ic.dat`, `ct.dat`, `rcdb.dat`, `ict.dat`, `dp.arm-v7.so.dat`. | + +## Asset file glossary + +| File | Role | +| --- | --- | +| `assets/classes.dex.dat` | Encrypted + compressed concatenation of all protected `classes.dex` files plus a footer header describing offsets and the **anti-dump unmap windows** (regions to `munmap` after the DEX is mapped into memory). | +| `assets/se.dat` | String-encryption lookup table. Maps a 16-bit index (passed to the native `ProtectedFoo.s(String)` shim as a `\uXXXX` literal) into a file offset + length pair that the native decryptor consumes. The crypto is keyed by the index AND the calling class's hash code. | +| `assets/resources.dat` | Encrypted resources covered by `` filters. Decoded on the fly via vtable hooks on `android::_FileAsset::*` inside `libandroidfw.so` and on `StringBlock.nativeGetString` / `AssetManager.nativeGetResourceIdentifier`. | +| `assets/mm.dat`, `assets/dp.mp3`, `assets/ic.dat`, `assets/ct.dat`, `assets/rcdb.dat` | Other DexProtector-managed blobs (config, runtime checks, integrity tables). The post does not enumerate the exact role of each; their presence is itself a high-confidence IoC. | +| `assets/dp.arm-*.so.dat` | Per-ABI stub used by `libdpboot.so` to locate the unpacker binary. | +| `assets/.dat` (e.g. `zpoasosdi.dat`, `regtbeonuev.dat`, `btylusqrepu.dat` in LiveNet) | Per-app encrypted assets covered by ``. In LiveNet these are serialized BouncyCastle keystores used to authenticate to the IPTV backend. | + +## Bootstrap chain (verified) + +``` +Protected.attachBaseContext() [Java] + -> integrity check (JNI native) + -> System.loadLibrary("dpboot") [-> libdpboot.so JNI_OnLoad] + -> JNI native -> System.loadLibrary("dexprotector") + [-> libdexprotector.so _INIT_0 + JNI_OnLoad] + -> custom ELF loader: decrypt + map payload (= libdp.so) into a fresh mmap region + -> jump into libdp.so's DT_FINI_ARRAY entry +``` + +## libdexprotector.so internals — arm64 (LiveNet 5.0.1) + +All offsets are file offsets into the arm64 `libdexprotector.so` (Ghidra base 0x100000 → subtract for file). + +| Routine | Address | Notes | +| --- | --- | --- | +| `_INIT_0` | `0x29d4` | walks program headers, locates the PT_LOAD that starts with `DPLF`, calls the driver | +| `_resolve_dlactivity` (was `FUN_001021ac`) | `0x21ac` | opens `/proc/self/exe`, reads its ELF, walks section/program headers for the dynamic table, parses `DT_DEBUG` (tag 0x15) → caches `r_debug*` at `.data+0x698` | +| `_dlactivity_getter` (was `FUN_001021a0`) | `0x21a0` | thin getter returning that cached `r_debug*` | +| `_KDF` (was `FUN_00100790`) | `0x00790` | obfuscated stack-machine VM over a 16 KB bytecode at file offset `0x8122`. Writes 32 bytes into the buffer passed in `x0`. At one specific state, calls the getter and **XORs bytes 0/4/8/12 of the key with 4 bytes read from `rtld_db_dlactivity`** (with a `+4` skip if a BTI landing pad is detected — pattern `5F 24 03 D5`). This is the frida-server-persistence detector documented in the Romain Thomas writeup (see `../../../references/sources.md`). | +| `_cipher` (was `FUN_00100c2c`) | `0x00c2c` | 32-round Feistel keystream cipher. State: u32 counter at +0xc, u64 key-mixing accumulator at +8, 8 round constants (the 32-byte key) at +0x18, 16-byte keystream block at +0x10. Output is XORed byte-by-byte against the ciphertext. | +| `_cipher_static` (was `FUN_001028d0`) | `0x028d0` | same round function as `_cipher`, but operates on `(key,ct)` blobs embedded in `.rodata` for short static strings (`/proc/self/exe`, the `r_debug` lookup, etc.). Useful for decoding the small in-binary constants without emulation. | +| `_lz4` (was `FUN_00101b58`) | `0x01b58` | LZ4 block decompression. Standard token>>4 literal length, `0xff` extension byte counting, 2-byte LE match offset, 4-byte minmatch — drop-in compatible with `lz4.block.decompress`. | +| `_unpack` (was `FUN_0010114c`) | `0x0114c` | top-level: KDF -> cipher init -> decrypt 0x24-byte super-header -> per segment (≤4) decrypt 0x18-byte descriptor table -> decrypt + LZ4-decompress segment bytes into a mmap'd image, validating each segment's 16-bit rolling hash | +| `_relocate` (was `FUN_001018e8` + `FUN_0010171c`) | `0x018e8`, `0x0171c` | apply ELF-style relocations on the custom dynamic table, then zero the dynamic table to defeat post-load memory dumps. Recognised tags map roughly to `DT_*`: 2=`STRSZ`, 5/6/7/8/10=`STRTAB`/`SYMTAB`/`RELA`/`RELASZ`/`RELAENT`-ish, 0x17/0x23/0x24/0x25=Android packed-RELA, INIT_ARRAY, INIT_ARRAYSZ, etc. | + +## Payload format + +``` +[+0x00] 'D' 'P' 'L' 'F' u32 magic +[+0x04] watermark32 (rendered as 5 ASCII hex bytes into .data+0x690) +[+0x08] 0x24 bytes of ciphertext = encrypted super-header + struct { + u32 page_size; // local_a0 + u32 vaddr_min_neg; // local_9c (used as -delta) + u32 vaddr_high; // local_90 + u32 second_range_off; // uStack_8c + u32 second_range_sz; // local_88 + u32 nseg; // local_80 (1..4) + // 3 more u32 of housekeeping + }; +[+0x2c] for i in 0..nseg-1: + 0x18 bytes of ciphertext = encrypted segment descriptor + struct { + u32 vaddr; // local_78 + u32 vsize_pad; // uStack_74 + u32 csize; // local_64 + u32 plain_len; // local_70 (LZ4 decompressed length) + u32 flags; // local_6c (PT_LOAD flags, low 3 bits) + u32 hash16; // local_68 (rolling 16-bit hash, validated) + }; + csize bytes of ciphertext, LZ4-decompressed into mmap+vaddr +``` + +After all segments are mapped, `libdexprotector.so` applies relocations on the in-memory image then zeroes the dynamic/relocation regions described by `DT_ANDROID_RELA` etc. before transferring control to the unpacked `libdp.so`'s `DT_FINI_ARRAY[0]`. + +## Recovered key (LiveNet arm64) + +``` +b0 b1 4a 07 c4 dc 4a dd ed 85 8d 03 0a 6b b1 61 +e1 54 ae f2 4c a8 2d 24 b6 a6 d5 91 61 2e 4f 31 +``` + +Bytes 0/4/8/12 are XORed with `c0 03 5f d6` (= `ret`). Any frida-server trampoline overlaying `rtld_db_dlactivity` at start-up corrupts these 4 positions and the cipher diverges immediately, which is the persistence quirk called out in the Romain Thomas writeup (see `../../../references/sources.md`). + +## What libdp.so contains + +Confirmed from the unpacked image (Tier-2 output of `scripts/dexprotector_unpack.py`): + +- **BoringSSL** statically linked: full algorithm string table — AES-{128,192,256}-{ECB,CBC,CTR,GCM,KW,KWP}, ChaCha20, ChaCha20-Poly1305, RSASSA-PSS, RSA-OAEP, SHA-{1,224,256,384,512}, SHA-3 family, all standard EC curves including `brainpoolP{256,512}r1`, `secp256k1`. +- **NDK Asset Manager** function imports — `AAssetManager_fromJava`, `AAssetManager_open`, `AAssetManager_openDir`, `AAssetDir_getNextFileName`, `AAsset_close`, `AAsset_getBuffer`, `AAsset_getLength` — used by the vtable-hook path in `libandroidfw.so`. +- **Linker / property / RASP plumbing** — `__system_property_get`, `pthread_atfork`, `getauxval`, `sigaction`, `fts_open`, `arc4random_buf`. + +This is consistent with the post's "RASP detections, the engine to load encrypted classes, the logic to load protected assets" characterization. + +## Tier-2 unpacker contract + +`scripts/dexprotector_unpack.py` accepts an APK or a bare arm64 `libdexprotector.so` and writes a synthetic ET_DYN AArch64 ELF wrapping the recovered segments. The output is byte-identical across runs on the same input (validated against LiveNet 5.0.1 and the envchecks 2.1 sample). + +What it currently solves: + +- ✅ arm64-v8a libdexprotector → libdp.so end-to-end without an Android device +- ✅ no instrumentation of libdp.so itself; the master-key corruption mechanism never triggers + +Not yet: + +- ⌛ armeabi-v7a / x86 / x86_64 — same payload layout, different function addresses (a small port) +- ⌛ libdp.so master-key derivation (needs follow-on RE on the unpacked libdp.so) +- ⌛ per-asset subkey derivation + decryption (`se.dat`, `classes.dex.dat`, custom `*.dat`) +- ⌛ classes.dex.dat anti-dump unmap-window reconstruction + +These are the next pieces of the post that the capability should reproduce. The unpacked `libdp.so` produced today is the input to all of them. + +## Workflow integration + +| Task | Use | +| --- | --- | +| Is this APK protected? | `scripts/protector_detect.py ` → `protector.json` | +| I want libdp.so | `scripts/dexprotector_unpack.py -o libdp.so` | +| I want to RE the asset crypto | Import the recovered `libdp.so` into Ghidra; start from `AAssetManager_open` xrefs (the vtable hooks) and the BoringSSL `EVP_aead_*` symbols. | + +## Known applications shipping DexProtector + +Verified via APKiD + manifest signals; sample list compiled from the Romain Thomas writeup (see `../../../references/sources.md`). + +| App | Version | +| --- | --- | +| `com.revolut.revolut` | 10.109.1 | +| `istark.vpn.starkreloaded` | 7.1-rc | +| `com.dexprotector.detector.envchecks` | 2.1 | +| `ar.tvplayer.tv` | 5.2.0 | +| `org.unhcr.zakat` | 2.1.54 | +| `com.Hyatt.hyt` | 6.16.0 | +| `com.kms.free` (Kaspersky) | 11.129.4.14969 | +| `com.flashget.parentalcontrol` | 1.3.6.0 | +| `com.belongtail.ai` | 2.8.4 | +| `com.kidoprotect.app` | 11.1 | +| `com.playnet.androidtv.ads` (LiveNet) | 5.0.1 | diff --git a/capabilities/android-apk-research/skills/android-protector-triage/references/jiagu-360.md b/capabilities/android-apk-research/skills/android-protector-triage/references/jiagu-360.md new file mode 100644 index 0000000..47bc884 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-protector-triage/references/jiagu-360.md @@ -0,0 +1,148 @@ +# 360 Jiagu / Qihoo 360 triage reference + +Status: early triage scaffold. This is the starting point for reproducing prior Jiagu unpacking research and turning it into capability-native detection / unprotection flows. + +Primary sources to read first: + +- 360 Jiagu official site: +- APKiD APK rules for Jiagu / Qihoo 360: +- APKiD ELF rules for Jiagu native: +- 2025 packer prevalence paper, *To Unpack or Not to Unpack: Living with Packers to Enable Dynamic Analysis of Android Apps*: +- Public unpacker baseline: +- Older public RE artifacts: + +## Identification (Tier 1) + +Jiagu is a classic **APK packer / reinforcement** target. Expect an exposed stub DEX and a native loader that recovers the real application code at runtime. + +| Surface | Signal | Source / confidence | +| --- | --- | --- | +| `apkid.json` | `packer: Jiagu` | High when APKiD hits. | +| `lib//` or `assets/` | `libjiagu.so`, `libjiagu_art.so` | High; APKiD APK rule. | +| `assets/` | `libjiagu_a64.so`, `libjiagu_x64.so`, `libjiagu_x86.so` | High in our corpus sample; native libs are shipped as assets instead of normal `lib//`. | +| `lib//` | `libprotectClass.so` | Qihoo 360 rule in APKiD; treat as Jiagu-family / Qihoo reinforcement. | +| Native strings | `JIAGU_APP_NAME`, `JIAGU_SO_BASE_NAME`, `JIAGU_ENCRYPTED_DEX_NAME`, `JIAGU_HASH_FILE_NAME` | High if found in ELF; APKiD `jiagu_native` rule. | +| Native behavior | direct syscalls / anti-hook patterns | Medium by itself; raises confidence when co-located with Jiagu libs. | + +### Current corpus anchor + +Observed in this repo's AndroZoo-84 inventory: + +| Package | APK | Artifact | Evidence | +| --- | --- | --- | --- | +| `com.kbzbank.kpaycustomer` | `corpus/androzoo/apks-84/com.kbzbank.kpaycustomer_novc_A9209A1A7D8B.apk` | `findings/androzoo-84/inventory/apks/a9209a1a7d8bfdb3e60d00fbcb01b3d32bf1c0cac787ed8b04dd38c933d5691c/` | APKiD flags APK and `assets/libjiagu*.so` as `Jiagu`; `assets/libjiagu_a64.so` also has `anti_hook: syscalls`. | + +APKiD also flags `classes.dex` as `dexlib 2.x` and `BlackObfuscator`. Treat this as a stub / app-adjacent obfuscation signal until decompiled. + +## Expected protection shape + +Working hypothesis from APKiD rules, public unpackers, and prior writeups: + +``` +classes.dex stub + -> loads / extracts Jiagu native library from assets or lib// + -> native loader validates environment and APK integrity + -> native loader locates encrypted DEX / app payload + -> decrypts and maps the real DEX at runtime + -> transfers control to original Application / entrypoint +``` + +The 2025 prevalence paper maps APKiD's `Jiagu` label to 360 and notes regional packer naming / alias issues. It also found packing is far more common in Chinese app-market datasets than non-Chinese APKPure-style datasets. For our capability, normalize `Jiagu`, `Qihoo 360`, and `360 reinforcement` under one triage family unless evidence indicates a distinct implementation. + +## Initial static triage workflow + +For a candidate APK: + +```bash +python3 scripts/protector_detect.py path/to.apk -o findings//protector.json +apkid path/to.apk -j > findings//apkid.json +unzip -l path/to.apk > findings//ziplisting.txt +``` + +Extract all Jiagu-like native blobs, including assets: + +```bash +mkdir -p findings//jiagu/libs +python3 - path/to.apk findings//jiagu/libs <<'PY' +import zipfile, pathlib, re, sys +apk = sys.argv[1] +out = pathlib.Path(sys.argv[2]) +pat = re.compile(r'(^|/)(libjiagu.*\.so|libprotectClass\.so|libjgbibc.*\.so)$', re.I) +with zipfile.ZipFile(apk) as z: + for n in z.namelist(): + if pat.search(n): + dest = out / n.replace('/', '__') + dest.write_bytes(z.read(n)) + print(n, '->', dest) +PY +``` + +Minimum evidence to capture: + +- Whether Jiagu libraries live under `assets/` or `lib//`. +- Exact library names, ABI mapping, SHA-256, section table, and entropy. +- Exposed `classes.dex` size and decompiled stub classes. +- All `System.loadLibrary`, `Runtime.load`, `DexClassLoader`, `PathClassLoader`, `InMemoryDexClassLoader`, and reflection call sites in the stub. +- Any encrypted payload candidates in `assets/`, `res/raw/`, or nonstandard ZIP entries. + +## Unprotection-flow hypotheses + +| Goal | Starting point | Difficulty | Notes | +| --- | --- | --- | --- | +| Detect Jiagu reliably | APK / asset names plus native strings | Low | Add `assets/libjiagu*.so` support; APKiD already caught this layout. | +| Locate stub entrypoint | JADX decompile of exposed `classes.dex` | Low-medium | Find Application subclass / attachBaseContext / loadLibrary path. | +| Locate encrypted DEX payload | Native strings `JIAGU_ENCRYPTED_DEX_NAME`, asset xrefs, ZIP entry names, file reads | Medium | Do not assume normal `classes*.dex` are real app code. | +| Recover runtime DEX | Dynamic memory dump or native decrypt routine emulation | Medium-high | Public unpackers may help identify expected memory layout / DEX repair steps. | +| Repair dumped DEX | Header/map/string/type/method table repair after memory extraction | Medium | Compare with public `jiagu_unpacker` and `360reverse` notes. | +| Static native decrypt port | Native function tracing from libjiagu init / decrypt routines | High | Target if dynamic dump is unreliable or blocked by anti-debug. | + +## Dynamic validation cautions + +Only do this with explicit authorization for the target sample. + +- Expect anti-debug and anti-hook logic; our corpus sample has APKiD `anti_hook: syscalls` on `assets/libjiagu_a64.so`. +- Start with baseline launch on a clean emulator/device and collect logcat / process maps. +- If using Frida, test attach and spawn separately. Record whether the app exits, crashes, or silently corrupts output. +- Prefer a minimal DEX-dump observation before modifying control flow. Hooking the wrong native routine may change unpacker behavior. + +## Detector implementation notes + +Add a Jiagu branch to `scripts/protector_detect.py` with: + +- ZIP-name scan for: + - `lib//libjiagu.so` + - `lib//libjiagu_art.so` + - `lib//libprotectClass.so` + - `assets/libjiagu*.so` + - `assets/libjgbibc*.so` +- ELF string scan for: + - `JIAGU_APP_NAME` + - `JIAGU_SO_BASE_NAME` + - `JIAGU_ENCRYPTED_DEX_NAME` + - `JIAGU_HASH_FILE_NAME` + - `libjiagu` +- Confidence model: + - high: APKiD hit OR `libjiagu*` native blob found. + - medium: Qihoo `libprotectClass.so` or native string cluster. + - low: exposed stub-only indicators without native proof. + +Suggested normalized output: + +```json +{ + "protector": "jiagu_360", + "triage_strategy": "protector_aware_dex_unpack", + "signals": { + "native_libs": {"assets": ["libjiagu.so", "libjiagu_a64.so"]}, + "anti_hook": ["syscalls"] + } +} +``` + +## Open questions for the first research pass + +1. Where does `com.kbzbank.kpaycustomer` store the encrypted real DEX: inside a Jiagu asset library, another asset, appended data, or a ZIP entry? +2. Does `assets/libjiagu_a64.so` contain the native string constants from APKiD's `jiagu_native` rule? +3. Does a public Jiagu unpacker work on this sample as-is? If not, what assumption breaks? +4. What DEX repair steps are needed after memory extraction on current Android runtime versions? +5. Is a static emulator/unicorn-style path practical for the decrypt routine, or is the first useful capability a controlled dynamic dump + repair pipeline? diff --git a/capabilities/android-apk-research/skills/android-protector-triage/references/promon-shield.md b/capabilities/android-apk-research/skills/android-protector-triage/references/promon-shield.md new file mode 100644 index 0000000..fddebf4 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-protector-triage/references/promon-shield.md @@ -0,0 +1,490 @@ +# Promon Shield triage reference (research preview) + +Grounding: like DexProtector, Promon Shield sits under OWASP MASVS-RESILIENCE (); MASTG-TEST-0089 *Testing Resiliency Against Reverse Engineering* covers the RASP / shield-binding controls Promon implements (). + +**Status: research-grade — not wired into the protector-triage skill workflow.** `protector_detect.py` flags Promon Shield from APKiD + native-library + ELF-section signals, but the static recovery scripts under `scripts/research/promon/promon_*` are first-pass evaluators that have not been validated across enough samples to recommend in the routine flow. Treat output as exploratory adjacency analysis; do not promote Promon-shielded findings to `strong_static_chain` based on these artifacts alone. + +This file captures public Promon Shield reversing references and the experimental recovery pipeline. The agent should not invoke it unless the user explicitly asks for the Promon research path. + +Primary sources: + +- Promon product page: +- APKiD ELF rules for Promon Shield: +- APKiD issue referencing Promon use in German banking apps / 34C3 talk: +- 34C3 talk, *Die fabelhafte Welt des Mobilebankings*: +- DIMVA 2018 paper, *Honey, I Shrunk Your App Security: The State of Android App Hardening*: / +- Public reversing repo: +- Public string-deobfuscator repo: + +## Research findings + +## Implemented static workflow + +Current capability-native scripts: + +| Stage | Script | Status | Output | +| --- | --- | --- | --- | +| Detect/routing | `scripts/protector_detect.py` | Implemented for APKiD inventory hits, Promon native library names, `.ncu/.ncc/.ncd` ELF section pairs, and historical assets. | `protector.json` | +| Smali string recovery | `scripts/research/promon/promon_string_recover.py` | Implemented first-pass evaluator for inline char-array/`String.intern()` and helper methods returning `[C`. Does not rebuild APKs by default. | `strings.jsonl`, optional patched smali, summary markdown | +| Java/smali integration triage | `scripts/research/promon/promon_java_triage.py` | Implemented mapper for native method declarations, native callsites, load-library callsites, framework hints, and string-recovery coverage hints. | `java-triage.json`, `native-methods.jsonl`, `native-call-sites.jsonl`, `load-library-sites.jsonl`, summary markdown | +| Binding callsite map | `scripts/research/promon/promon_binding_triage.py` | Implemented focused extraction of native string/class binding IDs and immediate sink context. | `promon-bindings.jsonl`, `promon-bindings.json`, summary markdown | +| Native ELF triage | `scripts/research/promon/promon_elf_triage.py` | Implemented static APK/ELF profiler: candidate extraction, section/segment metadata, entropy, `.init_array`, visible imports, RASP strings, AArch64 `SVC #0` sites. | `elf-triage.json`, `elf-sections.jsonl`, `syscall-sites.jsonl`, `native-imports.txt`, summary markdown | +| Orchestration | `scripts/research/promon/promon_recover.py` | Implemented wrapper for detection, optional apktool decode, string recovery, ELF triage, and a roll-up summary. | `RECOVERY_SUMMARY.md` plus all above artifacts | + +Preferred command for a full static pass: + +```bash +python3 scripts/research/promon/promon_recover.py path/to.apk -o findings//promon +``` + +If apktool is unavailable or you only want native triage: + +```bash +python3 scripts/research/promon/promon_recover.py path/to.apk -o findings//promon --skip-apktool +``` + +Manual pipeline equivalent: + +```bash +# 1. Detect/routing +python3 scripts/protector_detect.py path/to.apk \ + -o findings//promon/protector.json + +# 2. Decode smali +apktool d -f path/to.apk \ + -o findings//promon/smali-raw + +# 3. Recover Promon-style string constants +python3 scripts/research/promon/promon_string_recover.py \ + findings//promon/smali-raw \ + -o findings//promon/strings.jsonl \ + --patched-out findings//promon/smali-strings-recovered \ + --summary findings//promon/string-recovery-summary.md + +# 4. Map Java/native integration and Promon binding IDs +python3 scripts/research/promon/promon_java_triage.py \ + findings//promon/smali-raw \ + -o findings//promon/java-triage.json \ + --native-methods-out findings//promon/native-methods.jsonl \ + --native-calls-out findings//promon/native-call-sites.jsonl \ + --load-sites-out findings//promon/load-library-sites.jsonl \ + --summary findings//promon/promon-java-summary.md + +python3 scripts/research/promon/promon_binding_triage.py \ + findings//promon/smali-raw \ + -o findings//promon/promon-bindings.jsonl \ + --json findings//promon/promon-bindings.json \ + --summary findings//promon/promon-binding-summary.md + +# 5. Profile native shield library/libraries +python3 scripts/research/promon/promon_elf_triage.py path/to.apk \ + -o findings//promon +``` + +Expected artifact layout: + +```text +findings//promon/ + protector.json + smali-raw/ # when apktool decode is run + smali-strings-recovered/ # optional patched smali for analysis + strings.jsonl + string-recovery-summary.md + java-triage.json + native-methods.jsonl + native-call-sites.jsonl + load-library-sites.jsonl + promon-java-summary.md + promon-bindings.jsonl + promon-bindings.json + promon-binding-summary.md + elf-triage.json + elf-sections.jsonl + syscall-sites.jsonl + native-imports.txt + promon-elf-summary.md + RECOVERY_SUMMARY.md + libs//.so # extracted Promon candidates from APK +``` + +Current implemented limitations: + +- `promon_string_recover.py` expects apktool-decoded smali. It does not invoke apktool itself. +- String recovery supports a conservative subset of integer/char operations and does not execute arbitrary generated code. +- Patched smali is for analysis/decompilation assistance; it is not guaranteed rebuild-valid in every control-flow context. +- `promon_binding_triage.py` maps IDs and sink context but does not recover plaintext values. +- `promon_elf_triage.py` does not decrypt/unpack protected native sections. +- AArch64 syscall labels are heuristic and only resolve simple nearby `movz/movn w8, #imm` patterns. +- Direct real-sample APK validation is pending restoration of the local `in.org.npci.upiapp` APK; current local anchor has inventory/APKiD metadata. + +### What Promon is, operationally + +Promon Shield is primarily a **post-compile native app-shielding / RASP** target, not a DexProtector-style “most DEX is hidden in `assets/classes.dex.dat`” target. + +Promon's own product page describes: + +- post-compile integration; +- protection against tampering, reverse engineering, malware, rooted / jailbroken devices, debuggers, hooking, repackaging, screenshots, overlays, and keyloggers; +- app / Shield binding, so protection cannot be disabled by simply removing the shield component; +- response modes including reporting, blocking, and exiting. + +Practical consequence: JADX may still preserve significant app code. The first-class missing artifacts are usually **runtime strings, protected class / field bindings, native shield internals, policy/config, and native RASP decisions**, not necessarily all first-party DEX. + +### Public prior art + +| Source | What it contributes | Capability implications | +| --- | --- | --- | +| APKiD ELF rules | High-confidence IoCs: `libshield.so` or random `lib[a-z]{10,12}.so`; at least two of `.ncu`, `.ncc`, `.ncd`. APKiD comments describe `.ncc` as code, `.ncd` as data, `.ncu` as another protected segment. | Implement Promon branch in `scripts/protector_detect.py` by scanning native libraries and parsing ELF section names. | +| DIMVA 2018 paper / Springer abstract | Describes an in-depth Promon Shield case study. Springer page states the authors demonstrate two attacks: one removes the protection scheme statically, the other disables security measures dynamically at runtime. Search/PDF snippets describe encryption of `.rodata`, `.text`, and `.ncd` with three version-specific keys, plus encrypted assets `config-encrypt.txt`, `mappings.bin`, and `pbi.bin`. | The full recovery goal is not just bypassing checks. We need: section unpack, config recovery, mapping reversal, and Java/native binding removal or modeling. | +| 34C3 talk | Nomorp was used to automatically disable central security/hardening measures in 31 mobile banking apps; German financial apps were heavily represented. APKiD issue #72 was opened from this talk. | Treat banking samples as likely Promon candidates and compare against Nomorp/DIMVA flow. | +| KiFilterFiberContext/promon-reversal | Modern public reversal against Supercell/Brawl Stars. Confirms unobfuscated Dalvik with Promon string/class indirection; native shield packed via `.init_array`; direct `SVC #0` syscalls; dynamic imports; anti-debug ptrace/fork/prctl flow; JDWP interference; APK signing-block parsing via `openat`; checksum of shield library; root/emulator property and filesystem checks; anti-hook checks for Frida, Xposed, Substrate; memory-page integrity. Includes PoC code with sample-specific offsets for `openat` syscall stub replacement and many hook/log points. | Good map of runtime checks and dynamic observability points, but not a generic unpacker. Offset-dependent bypass code should be mined for signatures/check classes, not copied directly as a generic solution. | +| RevealedSoulEven/promon-string-deobfuscator | APKTool/smali pipeline that recognizes Promon-style char-array / `String.intern()` string obfuscation and replaces encrypted constants. Handles methods returning `[C` and dynamic integer-driven string patterns. | First static recovery script should be a clean, non-rebuilding variant that emits JSONL patches and optionally writes deobfuscated smali/JADX-adjacent sources. | + +## Identification (Tier 1) + +Promon is primarily a **native app-shielding / RASP** target rather than a DexProtector-style DEX asset packer. JADX may still show meaningful Java/Kotlin code. The native shield library is the first-class target. + +| Surface | Signal | Source / confidence | +| --- | --- | --- | +| `apkid.json` | `packer: Promon Shield` on a native library | High when APKiD hits. | +| `lib//` | `libshield.so` | High if section signals also match. | +| `lib//` | Random-looking `lib[a-z]{10,12}.so` with Promon sections | Medium by name alone; high if section signals match. Our corpus sample is `libnamfidcmogmm.so`. | +| ELF sections | At least two of `.ncu`, `.ncc`, `.ncd` | High; this is APKiD's main Promon rule. | +| `assets/` | `config-encrypt.txt`, `mappings.bin`, `pbi.bin` | Medium; described in older Promon integration flows. Newer builds may move config into ELF sections or rename assets. | +| Java/smali | native string method like `static native String a(int)` plus native class-init method like `static native void a(Class,int)` | Medium; from public reversal. Needs app-specific naming normalization. | +| Java/smali | XOR-built library name passed to `System.loadLibrary(new String(char[]).intern())` | Medium-high when it resolves to the Promon native library. | + +### Current corpus anchor + +Observed in this repo's AndroZoo-84 inventory: + +| Package | APK | Artifact | Evidence | +| --- | --- | --- | --- | +| `in.org.npci.upiapp` | `corpus/androzoo/apks-84/in.org.npci.upiapp_203000102_1A643C45E6BA.apk` | `findings/androzoo-84/inventory/apks/1a643c45e6ba07d60be1bfcdb899950dbab6f9b7fb2dfb4492f022527774650c/` | APKiD flags `lib/arm64-v8a/libnamfidcmogmm.so` as `Promon Shield`. | + +APKiD also flagged the exposed `classes.dex` with `Debug.isDebuggerConnected()` and multiple `Build.*` VM checks. Treat these as app/stub-adjacent anti-analysis hints, not proof that all checks belong to Promon. + +## Expected protection shape + +Working model synthesized from APKiD, DIMVA, and public reversal: + +```text +App launch / class initialization + -> Java glue builds or references random shield library name + -> System.loadLibrary() + -> shield library constructor in .init_array runs before JNI_OnLoad + -> constructor dynamically resolves imports and unpacks/decrypts protected native sections + -> Java string/class/field indirection binds app code to native shield + -> shield parses installed base.apk and APK signing block, verifies shield checksum / app binding + -> policy/config is decrypted/evaluated + -> runtime integrity, anti-debug, anti-hook, root/emulator, repackaging checks run + -> configured response: report, block, exit, or degrade behavior +``` + +Promon's public material emphasizes app binding, runtime threat response, and post-compile integration. The public reversal confirms this is implemented with a large native runtime using dynamic imports, direct syscalls, and runtime integrity checks. + +Key expectation for triage: **do not assume JADX is empty or mostly missing**. This is closer to “native shield and string/class binding around an otherwise analyzable app” than “encrypted first-party DEX hidden in assets”. The unprotection flow should therefore recover Java strings and shield policy/config, then map/neutralize/model native checks before escalating to dynamic validation. + +## Initial static triage workflow + +For a candidate APK: + +```bash +python3 scripts/protector_detect.py path/to.apk -o findings//protector.json +apkid path/to.apk -j > findings//apkid.json +unzip -l path/to.apk > findings//ziplisting.txt +``` + +Then extract the Promon candidate library: + +```bash +mkdir -p findings//promon +unzip -p path/to.apk lib/arm64-v8a/.so > findings//promon/.so +readelf -S findings//promon/.so > findings//promon/readelf-sections.txt +readelf -l findings//promon/.so > findings//promon/readelf-program-headers.txt +readelf -d findings//promon/.so > findings//promon/readelf-dynamic.txt +readelf -r findings//promon/.so > findings//promon/readelf-relocs.txt +readelf -W -x .init_array findings//promon/.so > findings//promon/init-array.hex 2>/dev/null || true +strings -a findings//promon/.so > findings//promon/strings.txt +sha256sum findings//promon/.so > findings//promon/sha256.txt +``` + +Minimum evidence to capture: + +- Which ABI(s) contain a Promon-like library. +- Exact library names and SHA-256 hashes. +- Presence / absence of `.ncu`, `.ncc`, `.ncd` sections. +- Section sizes, entropy, file offsets, virtual addresses, and flags for Promon sections. +- Init-array entries and whether `JNI_OnLoad` / ELF entry are packed or unhelpful. +- Assets that look like shield config (`config-encrypt.txt`, `mappings.bin`, `pbi.bin`, or renamed high-entropy equivalents). +- Java/Kotlin/smali load site: xrefs to `System.loadLibrary`, XOR-char-array library-name builders, native string/class indirection methods. +- Whether app code remains analyzable in JADX before string recovery. + +## Full recovery flow design + +### Stage 0 — Corpus and ground truth + +1. Start with an APKiD-positive APK and inventory artifacts. +2. Prefer arm64-v8a samples because the public reversal and current corpus anchor are arm64. +3. Save a stable working layout: + +```text +findings//promon/ + protector.json + apkid.json + ziplisting.txt + libs//.so + elf/.sections.json + smali-raw/ + smali-strings-recovered/ + native-unpacked/ + config/ + dynamic/ # only if authorized +``` + +### Stage 1 — Native detector and ELF section parser + +Implement Promon support in `scripts/protector_detect.py`: + +- scan `lib//*.so` names for `libshield.so` and `/lib[a-z]{10,12}\.so/`; +- parse ELF section table directly from zipped bytes; +- mark high confidence when any library has at least two of `.ncu`, `.ncc`, `.ncd`; +- optionally ingest APKiD JSON if present; +- record ABI, library path, section names, section offsets/sizes/flags, and section entropy; +- record historical assets: `config-encrypt.txt`, `mappings.bin`, `pbi.bin`. + +Suggested normalized output: + +```json +{ + "protector": "promon_shield", + "triage_strategy": "protector_aware_native_rasp", + "signals": { + "native_libs": {"arm64-v8a": ["libnamfidcmogmm.so"]}, + "elf_sections": { + "lib/arm64-v8a/libnamfidcmogmm.so": [".ncc", ".ncd", ".ncu"] + }, + "promon_assets": [] + }, + "artifacts": { + "promon_recovery_supported": "research", + "java_string_recovery_supported": true, + "static_native_unpack_supported": false + } +} +``` + +### Stage 2 — Java/smali recovery before native unpack + +This is the highest-value first recovery step because Promon often leaves Dalvik structure intact but externalizes strings and class metadata. + +1. Decode with apktool, not just JADX: + +```bash +apktool d -f path/to.apk -o findings//promon/smali-raw +``` + +2. Port the public `promon-string-deobfuscator` into a capability script that: + - does **not** require rebuilding the APK by default; + - scans all `smali*` directories; + - identifies char-array + XOR + `String.intern()` patterns; + - identifies helper methods returning `[C` and call sites passing integer IDs; + - writes `strings.jsonl` with file, method, line range, ciphertext pattern, plaintext, and confidence; + - optionally writes patched smali into `smali-strings-recovered/`. + +3. Feed recovered strings back into JADX/ripgrep triage: + - native library names; + - URLs/hosts/API paths; + - class names / reflection targets; + - anti-analysis strings; + - deep-link and WebView-relevant constants. + +Expected output: + +```text +findings//promon/strings.jsonl +findings//promon/smali-strings-recovered/ +findings//promon/string-recovery-summary.md +``` + +### Stage 3 — Shield bootstrap map + +From smali/JADX and ELF metadata, build a launch map: + +- Java load site and deobfuscated library name. +- Native methods registered by the shield (`RegisterNatives` table if recoverable after unpack; otherwise Java declarations). +- `.init_array` function addresses and their file offsets. +- Imported and dynamically resolved functions; known Promon runtime list from public reversal includes `dlsym`, `dlopen`, `dl_iterate_phdr`, `prctl`, `fork`, `ptrace`, `inotify_*`, `__system_property_get`, direct `openat/read/write/close/mmap/kill/getpid/exit_group/sigaction` syscalls. +- Direct syscall stubs (`SVC #0` on ARM64) with nearby syscall-number materialization. + +Expected output: + +```text +findings//promon/bootstrap-map.json +findings//promon/syscall-sites.jsonl +findings//promon/native-imports.txt +``` + +### Stage 4 — Static native section unpack research + +Goal: produce a synthetic normalized shared object or memory map with decrypted `.text` / `.rodata` / `.ncd` / `.ncc` / `.ncu` content, analogous in spirit to `scripts/dexprotector_unpack.py` but with a very different target. + +Research path: + +1. Parse ELF and locate `.init_array` entry. Public reversal says constructor code runs before normal JNI entry and unpacks the binary. +2. Compare protected sections: + - file bytes / entropy; + - segment permissions; + - relocation coverage; + - whether `.ncc/.ncd/.ncu` are mapped into executable/loadable segments. +3. Identify decrypt/unpack primitive: + - xrefs from init-array function into section ranges; + - `mprotect`/cache-flush/syscall sites changing section permissions; + - loops writing into `.text`, `.rodata`, `.ncd`, `.ncc`, or anonymous mappings; + - constants/keys near init routine. DIMVA snippets indicate three version-specific keys for `.rodata`, `.text`, and `.ncd` in older builds. +4. Build an emulator harness only for the constructor/decrypt routine when static lifting is understood enough to provide imports/memory. Do not execute the app or arbitrary shield response code as the first step. +5. Emit either: + - `native-unpacked/.decrypted.so` if sections can be written back; or + - `native-unpacked/.memory-map/` with section dumps and address metadata if the runtime layout is not ELF-compatible. + +Expected output contract: + +```text +findings//promon/native-unpacked/.sections.json +findings//promon/native-unpacked/.decrypted.so # if possible +findings//promon/native-unpacked/.memory.json # otherwise +findings//promon/native-unpacked/strings.txt +``` + +Current confidence: **medium** that this is feasible per-version; **low** that it will be one-shot generic across modern Promon versions without version classifiers. + +### Stage 5 — Config and policy recovery + +Older public material names encrypted assets: + +- `assets/config-encrypt.txt` — shield policy/config; +- `assets/mappings.bin` — Java/native binding or mapping metadata; +- `assets/pbi.bin` — protected binding/integrity metadata. + +Modern samples may move or rename these. Recovery strategy: + +1. Inventory assets and entropy; identify small high-entropy blobs loaded by shield. +2. Search recovered native strings and syscall/open hooks for asset names. +3. If static keys/routines are recovered, write decryptor for the config assets. +4. If dynamic validation is authorized, hook **after config decryption and before config evaluation** to dump plaintext config. DIMVA describes disabling Promon dynamically by rewriting config values after decryption but before evaluation; for our workflow, first dump and model the config rather than modifying it. +5. Normalize config into feature toggles: + - anti-debug / ptrace; + - anti-hook / Frida / Xposed / Substrate; + - root/emulator checks; + - APK signature / repackaging; + - shield self-checksum; + - response action. + +Expected output: + +```text +findings//promon/config/plain-config.* +findings//promon/config/policy.json +findings//promon/config/mappings.json +``` + +### Stage 6 — Binding removal or modeling + +The DIMVA paper reports a static removal attack; public reversal notes that app code is bound to shield through native string and class/field initialization methods. A safe research workflow should first **model** bindings before trying to rewrite APKs. + +Static modeling tasks: + +- Resolve native string IDs to plaintext strings from Stage 2 or native string tables. +- Resolve native class/field initialization calls like `initializeClassByID(Class,int)` where possible. +- Produce a binding map from Java call sites to recovered constants/fields. +- Identify code that will fail if shield native methods are removed. + +Optional rewrite tasks for owned/authorized samples: + +- Replace string native calls with `const-string` / Java constants. +- Replace class/field initialization native calls with direct field assignments if mappings are recovered. +- Remove or stub `System.loadLibrary` only after all dependent native calls are eliminated or stubbed. +- Rebuild and sign a research APK, then compare behavior to original in a clean environment. + +### Stage 7 — Dynamic validation, only when explicitly authorized + +Promon is designed to detect instrumentation and environmental tampering. Do not lead with Frida hooks against the shield library. + +Baseline-first plan: + +1. Clean emulator/device image; no Frida server/gadget; no root artifacts if avoidable. +2. Install original APK and record: + - logcat; + - process lifetime; + - `/proc//maps` snapshots where permitted; + - loaded libraries; + - baseline network/process behavior. +3. Introduce the least-invasive observability: + - loader/linker maps; + - syscall tracing if allowed by environment; + - JVMTI/JDWP only after baseline confirms it does not trigger immediate response. +4. If bypassing or dumping is authorized, prefer narrowly scoped hooks based on the static map: + - config-decrypt output dump; + - asset open/read for config files; + - `openat` path redirection only for controlled repackaging experiments; + - detection check return values only after policy is understood. + +The public PoC hooks many libc/dynamic-linker functions and uses sample-specific offsets for an `openat` syscall stub. Treat it as a research map and validation aid, not a generic capability script. + +## What remains valid before full unshielding + +Even before native section/config recovery: + +- Manifest attack surface: exported components, BROWSABLE filters, schemes/hosts, permissions. +- JADX semantic triage over preserved first-party code, with a caveat that strings may be missing until Stage 2. +- Java-level anti-analysis and environment checks. +- Native library supply-chain/fingerprint analysis: library name, hash, section layout, APKiD rule, Promon version clusters. +- Recovered strings from smali char-array patterns. + +Scanner-gap label: findings in preserved Java code are not necessarily scanner gaps, but any logic hidden behind Promon native config/string/class binding should be marked `scanner_gap = adjacent` or `not found` depending on whether scanners saw the source/sink after string recovery. + +## Unprotection-flow hypotheses + +| Goal | Starting point | Difficulty | Notes | +| --- | --- | --- | --- | +| Detect Promon reliably | APKiD ELF rule: library name + `.ncu/.ncc/.ncd` section pairs | Low | Add this to `protector_detect.py`; scan native libs directly, not just ZIP names. | +| Recover Java strings | Smali char-array / XOR / `String.intern()` patterns and helper methods returning `[C` | Low-medium | Public tool exists; port into a non-rebuilding JSONL-emitting script. | +| Map shield bootstrap | Java xrefs to `System.loadLibrary` + native `.init_array` | Medium | Identify whether app loads random Promon library directly or via wrapper. | +| Recover section decrypt / unpack | `.init_array`, `.ncc/.ncd/.ncu`, mprotect/syscall/write loops | Medium-high | Likely version-dependent. Compare on-disk sections to memory only with authorization. | +| Recover policy/config | `config-encrypt.txt`, `mappings.bin`, `pbi.bin`, or modern ELF-embedded config | Medium-high | DIMVA dynamic config rewrite suggests a plaintext config exists after decrypt; first dump/model it. | +| Neutralize anti-analysis | ptrace/prctl/fork, JDWP patching, `/proc` scans, direct syscalls, Frida/Xposed/Substrate checks | High | Static mapping first; dynamic hooks may trigger response paths. | +| Static shield removal | recovered strings + mappings + native-call stubs | High | Reported by DIMVA, but requires mappings and careful bytecode rewrite. | + +## Detector implementation notes + +Implemented in `scripts/protector_detect.py`: + +- ZIP-name scan for `lib//libshield.so`. +- Random `lib[a-z]{10,12}.so` handling only when APKiD or Promon section evidence exists, to avoid false positives like React Native `libjscexecutor.so`. +- ELF section parser for `.ncu`, `.ncc`, `.ncd` pairs. +- APKiD JSON ingest if an inventory directory has `apkid.json`. +- Asset-name scan for historical config assets. +- Confidence model: + - high: APKiD hit OR ELF section pair match. + - medium: `libshield.so` OR random lib name plus suspicious section/entropy. + - low: historical asset names only. + +Suggested normalized output is in Stage 1 above. + +## Implementation backlog + +1. Run `scripts/research/promon/promon_recover.py` against the corpus anchor once the APK is available locally or restored from AndroZoo. +2. Compare at least two current Promon samples to classify section layouts and avoid hardcoding one vendor/customer version. +3. Improve `scripts/research/promon/promon_string_recover.py` with additional smali opcode patterns observed in real samples. +4. Add a Java/native binding-map script for native string/class initialization methods that are not recoverable by smali-only char-array evaluation. +5. Defer any `promon_unpack_sections` work until native layout comparison indicates a stable static path; this script is not implemented in the released capability. +6. If explicitly authorized for a target sample, create `scripts/research/promon/promon_dynamic_dump.md` / harness notes for clean-device config dumps and memory-map comparison. + +## Open questions for the next research pass + +1. Are `.ncc/.ncd/.ncu` encrypted on disk in our `in.org.npci.upiapp` sample, or are they nonstandard protected/runtime sections? +2. Does the sample ship older Promon assets (`config-encrypt.txt`, `mappings.bin`, `pbi.bin`) or a newer config layout? +3. Does JADX preserve the app's first-party code, and how much improves after smali string recovery? +4. Can we recover native string/class binding tables statically, or do they require config/native section unpack first? +5. What exact anti-Frida / anti-debug checks trigger under attach vs spawn on a clean test device? +6. Can a static ELF transform produce a normalized library with decrypted sections, or is the first practical native recovery output a memory-map dump? diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/SKILL.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/SKILL.md new file mode 100644 index 0000000..6ff6b47 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/SKILL.md @@ -0,0 +1,599 @@ +--- +name: android-semantic-vuln-hunting +description: Use when hunting Android APK logic bugs at scale — deep links, intent redirection, WebView trust, exported components, auth/session/client-state bypasses that scanners miss or flatten to generic warnings. +allowed-tools: + - inventory_status + - run_corpus_inventory + - extract_components + - rank_components + - detect_runtime_kind + - extract_api_map + - rank_backend_richness + - normalize_semantic_findings + - bash + - read + - grep + - glob + - web_search + - web_extract + - report +license: MIT +--- + +# Android semantic vulnerability hunting + +Work only on APKs the user is authorized to analyze. Default to static analysis and authorized read-only validation; do not patch APKs, automate exploitation, attack production backends, or test live accounts without explicit scope. + +This skill is methodology + CLI. The `android-research` MCP wraps the orchestration layer (inventory, component ranking, runtime detection, protector triage, API map extraction, finding normalization); JADX, ripgrep, Semgrep, Joern, CodeQL, FlowDroid, hbctool, blutter, and adb run as bash because the heap-tier table, rule-pack ensemble, and query recipe *are* the methodology. See `references/workflow.md` § "Why bash, not MCP" for the per-tool rationale. + +This skill rides the same pipeline as `android-targeted-assessment` (one-APK depth mode). When the pipeline shape changes here (new MCP tool, heap-tier update, new bug class), update both skills in the same commit. + +Ground findings in OWASP MASVS ([MASVS](https://mas.owasp.org/MASVS/)) and MASTG procedures ([MASTG](https://mas.owasp.org/MASTG/)), with weakness mapping to CWE ([cwe.mitre.org](https://cwe.mitre.org/)) and MASWE ([MAS Weakness Enumeration](https://mas.owasp.org/MASWE/)). The `normalize_semantic_findings` MCP tool auto-fills these from the `class` slug per `references/output-schema.md`. OWASP notes automated static analysis is useful but noisy and requires careful review ([MASTG-TECH-0025](https://mas.owasp.org/MASTG/techniques/android/MASTG-TECH-0025/)); here scanners are baseline evidence, not the discovery driver. + +## First: discover the available flows + +Before choosing commands from memory, read the local utility index when you need orientation: + +```bash +cat references/agent-utility-index.md +``` + +Use it to route the session: + +- **manifest/component surface** → `run_corpus_inventory`, `extract_components`, `rank_components` (all MCP tools) +- **runtime/decompile routing** → `detect_runtime_kind` (MCP tool), then JADX with the right heap +- **impact-class source triage** → `run_class_rg.sh` plus the universal source/sink set +- **backend-rich APK/API map** → `extract_api_map.py`, then `rank_backend_richness.py` for corpus inboxes +- **feature flags / request signing / workflow state machines / hybrid bridge-to-backend** → the matching references under `references/` +- **scanner gap / normalization** → Semgrep baseline, then `normalize_semantic_findings` + +If the user asks “what can we try next?” or “what utilities do we have?”, summarize this index instead of improvising. +## Pick the mode before picking the tools + +Three intent shapes; pick one explicitly. Do not default to "per-class top-3". + +### Mode A — single APK (the user named one) + +The user said "look at `com.example`" or handed you an APK. **Skip the broad scan entirely.** Jump to Step 3 (decompile) with the right heap, then Step 4 (rg) on the first-party tree. The other ~85 APKs in the corpus are not your problem this session. Use the sibling skill `android-targeted-assessment` if you've got it. + +### Mode B — broad corpus pass (the user wants coverage) + +"What's interesting across these 86 APKs?" "Find me low-hanging deep-link bugs." "We've got 8 hours and 7 unstarted classes." + +Use a **funnel**, not a depth-first per-class crawl: + +1. **Tier A — component-level ranking, no decompile.** Run `extract_corpus_components.py` over the inventory directory and `rank_components.py` over the output. ~30 minutes wall to cover the whole corpus. Emits a ranked-component inbox of maybe 100–200 entries (out of ~15k components in an 86-APK corpus) at score >= 4. +2. **Tier B — decompile only APKs that own a top component.** Usually 15–30 APKs out of the corpus. JADX in a single parallel batch, `-Xmx` chosen by class count. +3. **Tier C — 5-minute reading pass per top component.** Outcome is one of `escalate` / `hardening_only` / `not_a_chain`. Strict per-component budget; queue overflow for later. Half the components dismiss in under a minute. +4. **Tier D — depth + Semgrep only on `escalate` candidates.** This is where per-target hypothesis + tier1 validation happens. Realistically 5–15 candidates from the whole corpus. + +Tier A scripts live at `scripts/extract_corpus_components.py` and `scripts/rank_components.py`; see "Step 2.5" below. + +### Mode C — single impact class (the user picked one) + +"Do `D_secret` next." Roughly halfway between A and B. Run the per-class rg profile (Step 4) and target the top 3 by `semantic_priority`, *but* cross-reference with the Tier A `components_ranked.md` view if it exists — sometimes a class's #4 or #5 APK has a higher-scoring component than the #1. Don't decompile #1 if its top component scored 3 and #4's top component scored 9. + +## The pipeline (Mode B/C view) + +``` +corpus → run_corpus_inventory (Androguard + APKiD, parallel, resumable) + → extract_corpus_components + rank_components → components_ranked.md + → jadx -d for APKs owning top-N ranked components + → rg the focused source/sink set on first-party code + → semgrep p/android for scanner baseline + scanner_gap labelling + → joern for call/data context on shortlisted methods + → optional codeql for precise Java/Kotlin path evidence (MASVS-grade) + → optional FlowDroid for lifecycle-aware taint when stakes warrant it + → finding hypotheses → normalize_semantic_findings + → authorized validation with adb +``` + +Each step is a deliberate escalation. Most APKs stop after the component rank; only chains with real impact reach Joern/CodeQL/FlowDroid. + +## Step 1 — inventory the corpus (parallel, resumable) + +```bash +# Tool — call from the agent. +run_corpus_inventory(paths=["corpus/apks"], out_dir="findings/run", jobs=8, include_apkid=true) +``` + +Each APK becomes `findings/run/apks//` with: + +- `inventory.json` — package, schemes/hosts, browsable/exported components, embedded URLs/domains, library hints +- `androguard.json` — authoritative manifest facts via Androguard ([Androguard](https://androguard.readthedocs.io/en/latest/intro/gettingstarted.html)) +- `apkid.json` — packer/protector/obfuscator signal ([APKiD](https://github.com/rednaga/APKiD)) +- `status.json` — per-stage status + +Aggregate JSONLs live at the run root: `attack_surface.jsonl` and `status.jsonl`. + +## Step 2 — rank with jq, not a custom tool + +```bash +RUN=findings/run + +# Highest semantic-priority APKs. +jq -c 'select(.semantic_priority.score>=10)' $RUN/attack_surface.jsonl \ + | jq -s 'sort_by(-.semantic_priority.score) | .[:25]' > $RUN/top25.json + +# Just the file paths for the next step. +jq -r '.[] | .apk' $RUN/top25.json > $RUN/top25.paths + +# Externally reachable BROWSABLE deep-link surfaces, by package. +jq -r '. as $r | $r.browsable_components[]? | "\($r.package)\t\(. )"' \ + $RUN/attack_surface.jsonl | sort -u +``` + +Selection bias: finance, identity, wallet, retail, travel, payments, OAuth, WebView-heavy hybrid apps, big component/scheme/host counts, packed/protected apps de-prioritised unless they are the target. + +## Step 2.5 — component-level ranking (Mode B/C only) + +`semantic_priority` ranks APKs; the actually-interesting things are **components**. An APK with 7 boring exported activities and 1 wildcard-host SSO callback ranks the same as one with 8 boring activities. The Tier A tools fix that by emitting one row per component and scoring it. + +```python +# MCP tools — call from the agent. +extract_components( + inventory_dir=f"{RUN}/apks", + triage_manifest=f"{RUN}/triage/manifest.jsonl", # optional join + runtime_kind=f"{RUN}/inventory/runtime_kind.jsonl", # optional join + out=f"{RUN}/triage/components.jsonl", +) +rank_components( + components=f"{RUN}/triage/components.jsonl", + out_jsonl=f"{RUN}/triage/components_ranked.jsonl", + out_md=f"{RUN}/triage/components_ranked.md", + top_md=150, min_score=4, +) +``` + +`extract_components` reads every `androguard.json` under `$RUN/apks//`, falls back to `aapt2 dump xmltree` for APKs where Androguard errored (empirically: multi-dex APKs with ≥22 `classes*.dex` files break Androguard 4.1.3), and emits one JSONL row per `(apk, type, name)` carrying exported/permission/scheme/host/path/action facts plus APK-level joins (`apkid_tier`, `impact_class`, `runtime_kind`). + +`rank_components` applies risk priors. The full table is in `scripts/rank_components.py`; in short: + +- **+5** exported BROWSABLE with no permission +- **+3** host wildcard; high-risk path/action/scheme (`join`, `accept`, `invite`, `redirect`, `callback`, `sso`, `oauth`, `transfer`, `recover`, `magic`, `intent`, `router`, `proxy`, `import`, `share`, `IMPORT_`, `EXPORT_`, `otpauth`, `smsto`, etc.); exported/no-perm share/import targets (`SEND`, `SEND_MULTIPLE`, `GET_CONTENT`, `OPEN_DOCUMENT`, `PICK`) +- **+2** exported AutofillService / AccessibilityService / CredentialProviderService; exported ContentProvider with `grantUriPermissions`; exported receiver with custom action and no perm; broad MIME share/import target (`*/*`, `image/*`, `application/octet-stream`) +- **+1** App-Link (http/https) surface with non-wildcard host +- **−5** permission with `protectionLevel=signature`; **−3** `medium` packer; **−12** `heavy` packer + +`read_budget` field: `5m` if score ≥ 7, `1m` if 3–6, `skip` if below 3. Use that as the per-component time cap in Tier C. + +Empirics from a representative 86-APK corpus (~15.8k components): + +- 14,810 components scored 0 → not worth a glance +- ~190 components scored ≥ 4 → the actual inbox +- Top hits at score 11–12 in internal corpus runs surfaced shapes like: a 7-auth-scheme LaunchActivity in an enterprise authenticator app (Azure Authenticator), a `DeepLinkRouterActivity` with 12 paths in a 2FA app (Duo Mobile), an SMS-MMS multi-scheme entry point in a lightweight email client (Microsoft Outlook Lite), and a wildcard-host SSO in a password-manager class (Proton Pass). Provenance framing in `references/sources.md`. + +The markdown view groups the top entries by `impact_class` first, then a full ranked list by package. Read it as your inbox: top of the page = next decompile candidate; bottom = "skim only if time". + +## Step 3 — decompile only what you'll actually read + +Before JADX, identify the **app runtime kind** with a 1-second probe. This determines (a) whether you need to budget for a JS-bundle or Dart-AOT trace after the Java pass and (b) how high to raise the JADX heap. + +```python +# Single APK — MCP tool returns dict with runtime_kind in +# {native, react_native_js, react_native_hermes, flutter_aot, +# capacitor, cordova, unity, xamarin, maui}. +detect_runtime_kind(apk="path/to.apk") +``` + +```bash +# Corpus sweep (one second per APK; output is JSONL keyed by apk path) +for a in corpus/apks/*.apk; do + bash scripts/detect_runtime_kind.sh --jsonl "$a" +done > findings/run/runtime_kind.jsonl +``` + +The probe uses `unzip -l` only — it does not extract the APK — and reads the Hermes magic bytes (`c6 1f bc 03`, per facebook/hermes `BCVersion.h`) to distinguish Hermes from plain JS without unpacking the bundle. + +Corpus-2 sweep (86 APKs, mostly C_wallet/G_messenger/D_secret): 76 native, 6 react_native_hermes, 4 flutter_aot. **C_wallet alone:** 5 of 7 targets are non-native. The 7.5/7.6 bundle trace is the dominant workload for that class, not an optional escalation. + +Decompile, capping the JADX heap by **DEX class count** (from `androguard.json` or `apkanalyzer dex packages`). JADX is unbounded by default; a banking APK can grow a single JVM to 6-8 GB. + +```bash +# Heap tier table — pick by class count, fall back to APK size +# <15k classes (<50 MB APK) : -Xmx2g +# 15-30k classes (50-100 MB) : -Xmx6g <-- many wallet/messenger apps +# >30k classes (>100 MB) : -Xmx8g +JAVA_OPTS="-Xmx6g" jadx --show-bad-code --no-debug-info -d findings/decompiled/ path/to.apk +``` + +**Empirical (corpus observation):** a wallet-class APK at ~83 MB / 27k classes (Trust Wallet) OOMs at `-Xmx2g` partway through decompile and succeeds clean at `-Xmx6g`; a smaller wallet at ~13k classes (MetaMask) and most fintech apps finish at `-Xmx2g`. Always check the class count before choosing parallelism. + +**>150 MB / >60k-class APK hang.** JADX 1.5.x has been observed to write the full source tree to disk and then hang at 99% on the shutdown phase rather than exit cleanly. Corpus observation: a banking-class APK at ~200 MB / 74k classes (Chase Mobile) at `-Xmx8g` finished output but the JVM never returned. The decompiled sources are fully usable while the JVM is hung; if the log shows the progress line stuck at the penultimate count for several minutes, kill the JVM (`ps -ef | grep jadx`; `kill -9 `) and continue. Do not wait for clean shutdown on banking-class APKs. + +Use the `--source-dir` output (`findings/decompiled//sources`) as the search root for the rest of the pipeline. Do not decompile the whole corpus. + +**Memory budgeting.** JADX is the heaviest tool in the pipeline. Multiply expected resident set by parallelism: + +| Stage | Per-worker RSS | Safe `jobs=` on 32 GB host | +|---|---|---| +| `run_corpus_inventory` | ~150 MB (streamed) | 8 | +| `jadx` <15k classes (`-Xmx2g`) | ~2.5 GB | 2–3 | +| `jadx` 15-30k classes (`-Xmx6g`) | ~6.5 GB | 1–2 | +| `jadx` >30k classes (`-Xmx8g`) | ~8.5 GB | 1 | +| `semgrep` (large tree) | ~1–2 GB | 2 | +| `joern` import | ~6–10 GB | 1 | +| `codeql database create` | ~4–8 GB | 1 | + +Run decompile + scanner stages serially when in doubt; Joern and CodeQL must be single-process on most laptops. **At scale (>20 APKs):** sort the queue by class count ascending and run the cheap APKs first — failure of one heavy APK doesn't burn the small ones queued behind it. + +## Step 4 — ripgrep the focused source/sink set on first-party code + +If the corpus has an `impact_class` per APK (the `android-corpus-prep` skill assigns these — `A_ingress`, `B_remote`, `C_wallet`, `D_secret`, `E_file_cloud`, `F_family`, `G_messenger`, `H_email`, `I_browser`, `J_iot`), run the per-class profile **in addition to** the universal source/sink set: + +```bash +bash scripts/run_class_rg.sh "$SRC" findings// +``` + +The per-class profiles capture the bug patterns unique to each app type — WalletConnect for wallets, AutofillService for password managers, MIME parsing for email, mDNS for IoT — and are documented in `references/impact-class-rg-profiles.md`. Reading the class-specific hits first is usually 3× faster than reading the universal set. + +### Universal source/sink set + +Limit search to the app package and adjacent first-party classes (`com..…`, `.…`). Skip `androidx.`, `kotlin.`, `kotlinx.`, `com.google.`, `com.facebook.react.`, etc. + +```bash +SRC=findings/decompiled//sources/ # narrow to first-party +rg -n -e 'getIntent\(' -e 'getData\(' -e 'getStringExtra\(' \ + -e 'getParcelableExtra\(' -e 'getSerializableExtra\(' \ + -e 'Intent\.parseUri\(' -e 'Uri\.parse\(' -e 'getQueryParameter\(' \ + -e 'startActivity\(' -e 'startService\(' -e 'sendBroadcast\(' \ + -e 'WebView' -e 'loadUrl\(' -e 'postUrl\(' \ + -e 'addJavascriptInterface\(' -e 'setJavaScriptEnabled\s*\(\s*true' \ + -e 'shouldOverrideUrlLoading' \ + -e 'CookieManager' -e 'Authorization' -e 'Bearer ' \ + -e 'SharedPreferences' -e 'isLoggedIn' \ + -e 'FLAG_GRANT_READ_URI_PERMISSION' -e 'FileProvider' \ + -e 'ACTION_SEND|ACTION_SEND_MULTIPLE|EXTRA_STREAM|EXTRA_TEXT|EXTRA_TITLE|ClipData' \ + -e 'OpenableColumns\.DISPLAY_NAME|MediaStore\.MediaColumns\.DISPLAY_NAME' \ + -e 'ContentResolver\.query|openInputStream|openOutputStream|openFileDescriptor' \ + -e 'FileOutputStream|Files\.copy|copyTo|openFileOutput|new File\(' \ + -e 'getCacheDir|getFilesDir|canonicalPath|getCanonicalPath|normalize|createTempFile' \ + -e 'rawQuery|execSQL|SQLiteQueryBuilder|selection|projection|sortOrder|setStrict|setProjectionMap' \ + -e 'https?://|/api/|/v[0-9]+/|/graphql|/grpc|/rpc|/gateway|/mobile|/internal' \ + -e 'Retrofit|OkHttpClient|Request\.Builder|HttpUrl|ApolloClient|GraphQL|operationName|persistedQuery' \ + -e 'io\.grpc|ManagedChannel|MethodDescriptor|GeneratedMessageLite|protobuf|WebSocket|Socket\.IO|SignalR|EventSource|MQTT|FirebaseFirestore' \ + -e 'X-Api-Key|apiKey|x-device|deviceId|installationId|sessionId|refreshToken' \ + -e 'Hmac|Mac\.getInstance|signature|signRequest|nonce|timestamp|canonical|attestation|SafetyNet|PlayIntegrity' \ + -e 'tenantId|orgId|accountId|userId|ownerId|familyId|childId|vaultId|orderId|paymentId|roomId|messageId' \ + -e 'role|isAdmin|verified|entitlement|premium|subscription|scope|permissions|status|state|price|amount|discount' \ + -e 'accept|approve|complete|activate|claim|redeem|recover|reset|verify|bind|pair|link|migrate|transfer' \ + -e 'callback|redirect_uri|returnUrl|webhook|avatarUrl|imageUrl|preview|unfurl|importUrl|sourceUrl' \ + "$SRC" > findings/triage/.rg.txt +``` + +Sort by file, then read the **handful** of files that show source+sink categories together. Prefer files implementing components that came back from `androguard.json` with `exported=true` and a BROWSABLE intent filter. For rich-backend apps, also group hits by API client / DTO / operation family; APK-discovered backend issues usually need endpoint + auth/header mechanics + object/workflow model before they are useful hypotheses. + +**R8 + Kotlin Metadata caveat.** Public Kotlin top-level objects (`object Foo { val BAR = "https://..." }`) survive R8 minification because the `@Metadata` annotation pins them, even when nothing in DEX reads `Foo.BAR`. Empirically on a leaked-host triage pass over ~14 popular Play APKs, **~71% of leaked-host literal hits in production APKs had zero DEX consumers** — they were R8/Kotlin-Metadata leftovers from build flavors that were stripped of their callers but not their data. Before walking back a host literal, confirm the containing class is *consumed* somewhere by another DEX file: + +```bash +# Once you have the file that holds the literal, name the obfuscated enclosing class: +rg -nl 'verified-it\.capitalone\.com' "$SRC" +# -> com/example/SomeObfuscatedClass.java + +# Then check whether any other DEX file references that class name (excluding kotlin.Metadata strings): +rg -nFw 'SomeObfuscatedClass' "$SRC" | grep -v 'kotlin/Metadata\|@Metadata' | head +``` + +If the only references are inside the file itself (self-reference) or inside `kotlin.Metadata` annotation literals on other classes, the string is dead. Record as `unreachable`. For the full leaked-host runbook including the four gate categories (A1 BuildConfig / A2 feature flag / A3 intent extra / A4 build-pin), see `references/leaked-host-triage.md`. + +**Reflection-trampoline caveat (pairip-style anti-tamper).** Some Play-protected apps replace the body of a manifest-named Activity/Service with a single reflective call whose `Method` handle is populated at runtime by native code: + +```java +// com.dashlane.ui.activities.DeepLinkRoutingActivity.onCreate() +nRvCZi.QDdhPXfzXCrHxb.invoke(null, this, bundle); +``` + +Following the call graph from the manifest-named entry class will dead-end. Switch to **content search on the known deep-link path strings** (paths and schemes pulled from the manifest): + +```bash +# We know dashlane://*/vault and dashlane://*/mplesslogin exist from the manifest. +rg -nl '"/vault"|"/mplesslogin"|"dashlane://"' "$SRC" +``` + +The real router will appear in the matching files — usually a `NavigatorImpl.handleDeepLink` or similar, often in a `navigation` / `routing` / `deeplink` package. Add this content-search step to the rg pass whenever the manifest entry class body is one or two lines of reflection. + +### Backend-rich API map pass + +If the APK looks like a rich backend client (many API paths, generated clients, request signing, GraphQL/gRPC/WebSocket, feature flags, tenant/account/device/payment/order concepts), run the lightweight API map pass before writing hypotheses: + +```python +extract_api_map( + src="findings/decompiled//sources", + out="findings//api_map.jsonl", + summary="findings//backend_richness.json", + dedupe=True, +) +# Returned dict includes `summary` with backend_richness / total_score / +# unique_value_counts / synergy_flags — no jq needed for routine inspection. +``` + +For JS/Dart shells, run the same script over `findings//js-analysis` or `findings//dart-analysis` after Step 7.5/7.6. Use `references/backend-rich-apk-workflows.md` to decide whether to reconstruct API maps, feature flags, request signing, workflow state machines, or bridge-to-backend traces. + +Backend-rich outputs are target maps. Default their hypotheses to `confidence_tier=needs_backend_validation` unless authorized backend testing proves BOLA, workflow bypass, mass assignment, SSRF/open redirect, replay, or bridge-to-backend action impact. + +## Step 5 — Semgrep scanner baseline (label scanner gaps) + +Semgrep is a precise, explainable scanner baseline. Use the public Android security rule pack against the JADX sources, then label each hypothesis against it. + +```bash +semgrep --config p/security-audit \ + --config p/mobsfscan \ + --config r/java \ + --metrics=off \ + --json --output findings/baselines//semgrep.json \ + findings/decompiled//sources +``` + +For each finding hypothesis, attach `scanner_gap`: + +- `exact baseline finding` — Semgrep flagged the same entrypoint/source/sink +- `adjacent baseline finding` — Semgrep saw the risky API but not the chain +- `generic warning only` — Semgrep produced a category, not a chain +- `not found` — no relevant Semgrep rule + +Most impactful logic bugs end up at `adjacent` or `not found`; that is the value of the capability. + +For Android-specific rule packs, see `references/scanners.md`. + +## Step 6 — Joern for call/data context on shortlisted methods + +Joern is the right tool when a chain needs caller/callee or data-flow context. Use it on selected JADX trees, not the whole corpus. CPG concept and query language: [docs.joern.io](https://docs.joern.io/code-property-graph/), [docs.joern.io/quickstart](https://docs.joern.io/quickstart/). + +```bash +joern --script <(cat <<'EOF' +importCode(inputPath = "findings/decompiled//sources", projectName = "") +EOF +) +``` + +Then run focused queries (recipes in `references/joern-recipes.md`): + +- exported activities whose `onCreate` reads `getIntent()` and reaches `startActivity` +- methods reading `getStringExtra` whose values flow into `loadUrl` +- callers of `addJavascriptInterface` and what the bridge exposes + +Stop using Joern as soon as you have enough context to write the hypothesis. It is not a corpus scanner. + +## Step 7 — optional CodeQL for path-precise evidence + +CodeQL has Android-specific Java/Kotlin queries: intent redirection ([java-android-intent-redirection](https://codeql.github.com/codeql-query-help/java/java-android-intent-redirection/)), unsafe WebView fetch ([java-android-unsafe-android-webview-fetch](https://codeql.github.com/codeql-query-help/java/java-android-unsafe-android-webview-fetch/)), arbitrary APK installation, WebView JavaScript settings, and more ([java/](https://codeql.github.com/codeql-query-help/java/)). Use CodeQL when a finding needs path-precise (SARIF) evidence for an external report. + +Recipes: `references/codeql-recipes.md`. + +CodeQL database creation from JADX-decompiled source is best-effort: it works for source-rich apps but not every decompile. Treat it as optional escalation. + +## Step 7.5 — React Native and hybrid apps: trace into the JS bundle + +For React Native, Capacitor, Cordova, and other JS-driven shells, Java-side findings are almost always incomplete. The Java side typically: + +- declares a `MainActivity` that wraps a JS runtime +- forwards intent data / cookies / file paths into JS via native bridges with **no validation in Java** +- delegates trust to the JS layer + +A bare Java-side finding ("activity hands intent.getData() to JS, no scheme validation") is **unactionable until you read the JS handler**. Most of the time the JS layer enforces its own allowlist; sometimes it does not. Without the JS-side trace, the hypothesis is `needs_route_map_validation` at best. + +### 7.5a — Identify the bundle format first + +```bash +find findings/decompiled//resources/assets -maxdepth 1 \ + \( -name 'index*.bundle*' -o -name '*.jsbundle' \) -exec file {} \; +``` + +Three outcomes drive different recipes: + +- **Plain JS** (`ASCII text`, `data`, or `UTF-8`) → recipe 7.5b (prettier). Most older RN apps and Capacitor/Cordova ship plain JS. +- **Hermes bytecode** (magic `c6 1f bc 03`, `file` reports `data`) → recipe 7.5c (hbctool disasm). Default for newer RN; **observed on MetaMask, Trust Wallet, Bitpay, Coinbase, Discord**. +- **Encrypted / packed bundle** → expect `setBundleAssetName` to point at an alternate file and a custom loader; treat as dynamic-only and note in `missing_evidence`. + +### 7.5b — Plain JS bundle + +```bash +BUNDLE=findings/decompiled//resources/assets/index.bundle +mkdir -p findings//js-analysis +npx --yes prettier@3.3.3 --parser babel --print-width 120 "$BUNDLE" \ + > findings//js-analysis/index.pretty.js +wc -l findings//js-analysis/index.pretty.js +JSDIR=findings//js-analysis +``` + +### 7.5c — Hermes bytecode bundle + +```bash +pipx install hbctool 2>/dev/null # one-time +BUNDLE=findings/decompiled//resources/assets/index.android.bundle +JSDIR=findings//js-analysis +mkdir -p "$JSDIR" +hbctool disasm "$BUNDLE" "$JSDIR/hbc" +# disasm produces a string table + per-function HBC instructions; the string table +# is the single most useful artifact for tracing bridge names and route keys. +rg -nN -e 'YOUR_BRIDGE' -e 'addEventListener' -e 'Linking' "$JSDIR/hbc/strings.json" | head +``` + +String-table grep is fast: route keys (`open_url`, `_PATHS_TO_SCREENS_MAP`, `DEEP_LINK_*`, scheme literals like `"metamask"`, `"trust"`, `"wc"`) live there verbatim. Cross-reference hits against the HBC function bodies that load them. + +### 7.5d — Grep the bridge surface, then follow the data backwards + +For each Java-side bridge you flagged (`emitNewIntentReceived`, `RNCookieManagerAndroid`, `RNFileViewer`, `addJavascriptInterface` name, etc.), find in `$JSDIR`: + +1. **Where the bridge symbol is referenced** (`rg -n RNCookieManagerAndroid $JSDIR`). +2. **Whether the surrounding module wraps it and whether anyone imports the wrapper.** A bridge that is wrapped but never called is `unused_bridge_attack_surface`, not a live finding. +3. **What controls the arguments at every call site.** Trace backwards: are URL/path/cookie values derived from intent data, remote config, or static constants? If they trace back to `Linking.addEventListener('url', ...)`, you are reading the deep-link handler — capture the allowlist there. +4. **The allowlist / router map.** Look for `*_PATHS_TO_SCREENS_MAP`, `ROUTES`, `DEEP_LINK_*`, `ALLOWED_HOSTS`. A strict allowlist downgrades the Java-side finding to `hardening_only`. A `contains`/`startsWith`/regex check is a real bug. + +### What a clean trace looks like + +``` +intent.getData() (Java) + -> emitNewIntentReceived(data) (Java bridge) + -> Linking 'url' event in JS (module ABC) + -> if (scheme === 'offlinepay') validateHost('merchant.app') -> validatePath0('navigate') -> screenMap[name] || fallback + -> if (scheme === 'https') validateHost(APAY_DOMAIN) -> ... + -> else metric('InvalidProtocol') +``` + +The chain is only as strong as its weakest validator. Record the JS-side validators in the hypothesis evidence so the same finding can be re-evaluated against the next bundle version. + +### Pre-bundle hypotheses are wrong half the time + +A Java-side static finding for a React Native or Capacitor/Cordova app should default to `confidence_tier=needs_route_map_validation` until the JS trace lands. Promoting it to `strong_static_chain` without the JS layer review produces over-grade reports. + +## Step 7.6 — Flutter/Dart AOT apps: trace into `libapp.so` + +Mirror of 7.5 for Flutter/Dart-AOT shells. The Java/Kotlin side declares a `FlutterActivity` (or subclass) plus a wall of MethodChannel registrations in `configureFlutterEngine` / `o(flutterEngine)`; every routing decision is in the AOT-compiled Dart in `lib//libapp.so` (plus `flutter_assets/kernel_blob.bin` for the kernel snapshot). + +A Java-side finding ("MainActivity forwards `intent.getDataString()` into the Flutter engine without validation") is **unactionable until you read the Dart handler**. Default grade: `needs_route_map_validation`. + +### 7.6a — Detect and locate + +```bash +find findings/decompiled/ -path '*lib/*/libapp.so' -o -name 'kernel_blob.bin' 2>/dev/null +file findings/decompiled//resources/lib/arm64-v8a/libapp.so 2>/dev/null +``` + +Presence of `libapp.so` plus `libflutter.so` confirms Flutter AOT. + +### 7.6b — Recover Dart structure with blutter + +[`blutter`](https://github.com/worawit/blutter) reverse-engineers `libapp.so` into class names, methods, and string constants. One-time setup is non-trivial (clones Dart SDK headers per Flutter version); cache the output. + +```bash +# https://github.com/worawit/blutter — Python 3 + Dart SDK headers +blutter findings/decompiled//resources/lib/arm64-v8a/ findings//dart-analysis/ +``` + +Produces: + +- `objs.txt` — class names + field layout +- `pp.txt` — every Dart object literal (string constants, including route keys, allowlists, scheme literals) +- `radare2//blutter_frida.js` — Frida hook scaffold for dynamic validation + +Grep `pp.txt` for the same things you'd grep a JS bundle for: + +```bash +rg -nN -e 'safepalwallet' -e 'wc:' -e 'allowedHost' -e 'PATHS_TO' \ + findings//dart-analysis/pp.txt | head +``` + +### 7.6c — Match MethodChannel names + +List every Java-side `new MethodChannel(messenger, "")` from the Flutter engine setup; each name is an entry point exposed to Dart. Find the Dart-side handler for each (blutter naming follows the original class path): + +```bash +# Java side — names registered in MainActivity.configureFlutterEngine / o(flutterEngine) +rg -n 'new C6.o\|new MethodChannel\|new EventChannel' findings/decompiled//sources \ + | rg -o '"[^"]+"' | sort -u +# Dart side — find the corresponding handler +rg -nN '' findings//dart-analysis/ +``` + +The Java-to-Dart routing topology is the equivalent of the RN router map. Strict allowlist in Dart → `hardening_only`. Loose check → real bug. + +### When blutter is unavailable + +Fall back to: + +- `strings -a libapp.so | rg -e 'scheme://' -e 'PATHS_TO' -e 'allowedHost'` — quick smoke; misses structure but surfaces route keys. +- `radare2 -A libapp.so` + manual class-walk on the addresses blutter would have named. +- Dynamic-only: Frida hook on the MethodChannel call-site and capture every (method, args) pair as you fuzz the deep-link surface. + +## Step 8 — FlowDroid for lifecycle-aware taint when stakes warrant it + +For complex source→sink claims spanning the Android lifecycle, FlowDroid ([blog](https://blogs.uni-paderborn.de/sse/tools/flowdroid/), [github](https://github.com/secure-software-engineering/FlowDroid)) is the canonical engine. Cost is real (Java, `android.jar` for the API level, sources/sinks config), so only escalate when impact and evidence demand it. + +Reserve FlowDroid for hypotheses where: + +- impact is account takeover / payment / identity +- the chain crosses lifecycle/callback boundaries +- Semgrep + Joern + CodeQL still can't prove or disprove the flow + +## Step 9 — write hypotheses, normalize, validate + +Use the schema in `references/output-schema.md`. One JSONL record per candidate; supply a `class` slug to auto-fill MASVS/CWE/MASWE tags. Then: + +```python +# MCP tool — emit deterministic Markdown report + JSONL appendix. +normalize_semantic_findings( + inputs=["findings/hypotheses.jsonl"], + output_format="markdown", + out="findings/report.md", +) +``` + +For authorized validation, derive ADB commands from `androguard.json` (decoded schemes/hosts/components): + +```bash +adb shell am start -a android.intent.action.VIEW \ + -d 'myapp://route?dn_probe=1' \ + com.target.package +adb logcat -d | rg -i 'target|webview|deep' +``` + +Validation tiers (from `references/workflow.md`): + +- `tier0_static_only` +- `tier1_local_device_no_live_backend` +- `tier2_test_account_or_qa_backend` +- `tier3_explicit_production_authorization` + +## Bug classes to prioritise + +Detailed patterns in `references/bug-classes.md`. Summary: + +- **Deep link / router bugs** — host validation by `contains`/`startsWith`, stale allowlisted partner domains, `next`/`redirect`/`returnUrl` reaching `WebView.loadUrl` with auth context, OAuth callback abuse. Background: [Android deep-link risks](https://developer.android.com/privacy-and-security/risks/unsafe-use-of-deeplinks), [Oversecured deep-link research](https://oversecured.com/blog/android-deep-link-vulnerabilities), [MASTG-TEST-0028](https://mas.owasp.org/MASTG/tests/android/MASVS-PLATFORM/MASTG-TEST-0028/). +- **Intent redirection / private component reachability** — exported component reads `getParcelableExtra`/nested intent and `startActivity`, preserving data URI/flags/grants ([Android guidance](https://developer.android.com/privacy-and-security/risks/intent-redirection), [CodeQL](https://codeql.github.com/codeql-query-help/java/java-android-intent-redirection/)). +- **WebView trust-boundary bugs** — attacker-influenced URL plus `setJavaScriptEnabled(true)` + `addJavascriptInterface` / `postWebMessage` + auth cookies/headers; `shouldOverrideUrlLoading` of `intent://` or app routes ([Android unsafe URI loading](https://developer.android.com/privacy-and-security/risks/unsafe-uri-loading), [native bridges](https://developer.android.com/privacy-and-security/risks/insecure-webview-native-bridges), [CodeQL](https://codeql.github.com/codeql-query-help/java/java-android-unsafe-android-webview-fetch/)). +- **Dirty Stream / share-target file overwrite** — exported share/import target trusts `content://` provider display names, `EXTRA_TITLE`, or caller paths and writes into app-private storage; impact depends on later trusted use of overwritten config/token/code/cache ([Microsoft Dirty Stream](https://www.microsoft.com/en-us/security/blog/2024/05/01/dirty-stream-attack-discovering-and-mitigating-a-common-vulnerability-pattern-in-android-apps/), [Android filename guidance](https://developer.android.com/privacy-and-security/risks/untrustworthy-contentprovider-provided-filename)). +- **Provider SQLi / provider file exposure** — exported provider with no signature permission accepts caller-controlled selection/projection/path/table routing or broad FileProvider paths; distinguish exported boilerplate from sensitive rows/files reachable. +- **APK-discovered backend API bugs** — APK reveals mobile-only REST/GraphQL/gRPC/WebSocket endpoints, DTOs, feature flags, object IDs, request-signing, and workflow verbs that map to BOLA/IDOR, mass assignment, SSRF/open redirect, state-machine bypass, or webview-bridge-to-native-API actions. Static APK evidence is usually `needs_backend_validation` until tested with authorized accounts/QA backend. +- **Auth/session/client-state bugs** — login decisions from mutable local state, client-validated reset/invite/magic tokens, hardcoded keys participating in account-state operations, premium/admin gates from local booleans. +- **Non-prod host / endpoint reachable from production (leaked-host chain)** — production APK ships a QA/stage/dev/sandbox hostname *plus* the selector logic to pick it. Triage with the four-category gate matrix (A1 BuildConfig / A2 feature flag / A3 intent extra / A4 build-pin via Dagger or bootstrap initializer). Empirically ~7% of leaked hosts in popular apps are real `feature_flag_gated` chains, ~21% are A4 build-pinned, ~71% are R8-leftover dead strings. Full runbook in `references/leaked-host-triage.md`. + +Historical pattern reference: `references/historical-patterns-2023-2026.md`. It is intentionally search-biased toward public CVEs/writeups; use it for pattern inspiration, not prevalence claims. APK-to-backend references: `references/apk-to-backend-api.md`, `backend-rich-apk-workflows.md`, `feature-flag-mining.md`, `request-signing-and-attestation.md`, `workflow-state-machines.md`, and `hybrid-bridge-to-backend.md`. + +Every hypothesis must connect entrypoint → source → trust boundary → sink → impact. If one is missing it stays a hypothesis, not a finding. Record `missing_evidence` honestly. + +## Scale strategy + +- Inventory first, decompile second. Most APKs in a corpus never need JADX. +- **Detect runtime kind before JADX** (Step 3 probe). If the app is React Native or Flutter AOT, the Java pass is a thin layer; budget time for 7.5 / 7.6 up front, not after. +- **Sort the JADX queue by class count ascending.** Cheap APKs finish first; heavy APKs (banking/wallet, 25k+ classes, `-Xmx6g`/`-Xmx8g`) run last and at lower parallelism. Failure of one heavy APK then doesn't starve the rest. +- Read **handfuls of files** per APK, not whole trees. Use `rg` + `androguard.json` exported/BROWSABLE components to pick which files. +- Skip `namespace_kind=known_library` unless first-party code supplies attacker-controlled data or sensitive context. +- Compare against Semgrep only after forming hypotheses, never before — scanners anchor discovery if you read them first. +- Deduplicate findings by `(package, entrypoint, source, sink, impact)`. `normalize_semantic_findings` does this for you. +- **JS/Dart-shell apps cluster** on similar Java-side patterns. After two or three per impact class, the recurring shape (e.g. all C_wallet wallets forward intent data to JS unchecked) becomes the class-level finding; per-APK reports should focus on the JS/Dart-side delta from the class baseline. + +## Finding hypothesis contract + +```json +{ + "title": "short impact-oriented title", + "apk": "file.apk", + "package": "com.example", + "masvs": ["MASVS-PLATFORM"], + "class": "deep_link_to_authenticated_webview", + "entrypoint": "exported BROWSABLE Activity / scheme / host / receiver / provider", + "source": "attacker-controlled value", + "trust_boundary": "why the app should not trust it", + "sink": "security-sensitive operation reached", + "impact": "account takeover / token theft / private component access / auth bypass / data exposure", + "evidence": ["file:line snippets, manifest facts, joern/codeql output paths"], + "validation_plan": ["adb command, helper-app plan, test-account steps"], + "scanner_gap": "not found | generic warning only | adjacent baseline finding | exact baseline finding", + "confidence_tier": "confirmed_dynamic|strong_static_chain|needs_backend_validation|needs_route_map_validation|hardening_only|generic_library_noise", + "validation_tier": "tier0_static_only|tier1_local_device_no_live_backend|tier2_test_account_or_qa_backend|tier3_explicit_production_authorization", + "missing_evidence": ["what must be checked before exploitability can be claimed"] +} +``` + +## References (read on demand) + +- `../../references/sources.md` — capability-root citation registry (industry standards, public CVE/research links, tool docs, internal-research provenance). Every external claim in the per-skill references resolves here. +- `references/bug-classes.md` — full pattern catalogue with grounding URLs. +- `references/leaked-host-triage.md` — runbook for non-prod hostnames shipped in production DEX (A1/A2/A3/A4 gate categories, outcome schema, known false-positive shapes). +- `references/workflow.md` — validation tier definitions and corpus-vs-targeted timing. +- `references/output-schema.md` — finding JSONL + corpus manifest schema. +- `references/joern-recipes.md` — Joern import + focused Android CPG queries. +- `references/codeql-recipes.md` — CodeQL Java/Kotlin DB build + Android query pack invocation. +- `references/scanners.md` — Semgrep/MobSF/APKHunt/ripgrep recipes for scanner-gap labelling. +- `references/corpus-acquisition.md` — AndroZoo / F-Droid / device extraction; pair with `../android-corpus-prep` for the DuckDB-on-Parquet selection path. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/agent-utility-index.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/agent-utility-index.md new file mode 100644 index 0000000..d152f2e --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/agent-utility-index.md @@ -0,0 +1,74 @@ +# Agent utility index + +When you load an Android APK research skill, use this as the quick discovery map for what the capability can do. Prefer these utilities before inventing ad hoc commands. + +The orchestration layer is the `android-research` MCP (10 tools listed below). Methodology — decompile, search, scan, query, taint — runs as bash + canonical CLIs because the heap-tier table, rule-pack ensemble, query recipe, and pattern selection *are* the value. See `workflow.md` § "Why bash, not MCP" for the rationale per upstream tool we considered. + +## Environment probe + +| Intent | Utility | Output | When to use | +|---|---|---|---| +| Probe which underlying CLIs are reachable | `inventory_status` MCP tool | dict: uv / apkid / aapt2 / jadx / semgrep / joern / codeql / adb / hbctool / blutter status | Once per session, at start. Tells you which skill steps will work end-to-end. | + +## Corpus and manifest triage + +| Intent | Utility | Output | When to use | +|---|---|---|---| +| Inventory APKs with manifest/components/APKiD | `run_corpus_inventory` MCP tool | `apks//{inventory,androguard,apkid,status}.json`, `attack_surface.jsonl` | First pass over one or many APKs. | +| Extract one row per component | `extract_components` MCP tool | `components.jsonl` | After inventory, before decompile, for component-level ranking. | +| Rank exported/BROWSABLE/share/provider components | `rank_components` MCP tool | `components_ranked.jsonl`, `components_ranked.md` | Build the component inbox for broad corpus or impact-class pass. | +| Detect runtime kind | `detect_runtime_kind` MCP tool | dict with `runtime_kind` in {`native`, `react_native_hermes`, `flutter_aot`, ...} | Always before JADX; decides JS/Dart follow-up and heap budget. | + +## Protector triage (when APKiD flags packing) + +| Intent | Utility | Output | When to use | +|---|---|---|---| +| Detect DexProtector / Promon Shield signals | `detect_protector` MCP tool | dict with `protector`, `confidence`, `triage_strategy`, `artifacts.dexprotector_unpack_supported` | Whenever APKiD reports a packer hit or `.dat` blobs appear in `assets/`. | +| Static unpack of libdp.so (arm64-v8a) | `dexprotector_unpack` MCP tool | recovered `libdp.so` ELF | Only when `detect_protector` reports `dexprotector_unpack_supported=true`. | + +## Decompile and source triage + +| Intent | Utility | Output | When to use | +|---|---|---|---| +| Decompile APK | `jadx --show-bad-code --no-debug-info -d ...` (bash) | source/resource tree | Only for APKs you will read. Heap based on class count. | +| Run class-specific grep | `scripts/run_class_rg.sh "$SRC" findings//` (bash) | `rg-class-*.txt` | First reading pass when impact class is known. | +| Universal source/sink grep | recipe in `android-semantic-vuln-hunting` Step 4 (bash) | `rg.txt` / `triage/.rg.txt` | Always after decompile; read files with source+sink clusters. | +| Scanner baseline | `semgrep --config p/security-audit --config p/mobsfscan --config r/java` (bash) | `semgrep.json` | After hypotheses form; label scanner gaps. The semgrep MCP can't express our multi-config ensemble — see workflow.md. | + +## Backend-rich APK discovery + +| Intent | Utility | Output | When to use | +|---|---|---|---| +| Extract APK-derived API/backend map | `extract_api_map` MCP tool | endpoint/client/auth/signing/object/workflow/flag/bridge rows + richness summary | Any APK with Retrofit/Apollo/gRPC/WebSocket/Firebase, request signing, feature flags, tenant/account/device/payment/order concepts. | +| Rank backend-rich targets | `rank_backend_richness` MCP tool | backend-rich APK inbox | After extracting API maps across multiple APKs. | +| Feature flag mining | `references/.../feature-flag-mining.md` grep recipe | `rg-feature-flags.txt` | When remote config/experiments/flag names appear. | +| Request signing / attestation review | `references/.../request-signing-and-attestation.md` grep recipe | `rg-signing-attestation.txt` | When HMAC/signature/nonce/Play Integrity/cert pinning appears. | +| Workflow state-machine reconstruction | `references/.../workflow-state-machines.md` | workflow template | When verbs like accept/approve/complete/recover/pair/transfer appear. | +| Hybrid bridge to backend trace | `references/.../hybrid-bridge-to-backend.md` | bridge-to-API trace | RN/Flutter/WebView/Capacitor/Cordova apps where JS/Dart/web input reaches native API clients. | + +## Hybrid/runtime follow-up + +| Intent | Utility | Output | When to use | +|---|---|---|---| +| Plain RN/JS bundle | `prettier` recipe in skill Step 7.5 (bash) | `index.pretty.js` | `react_native_js` or plain bundle. | +| Hermes bundle | `hbctool disasm` recipe in skill Step 7.5 (bash) | HBC strings/functions | `react_native_hermes`; grep strings first. | +| Flutter/Dart AOT | `blutter` recipe in skill Step 7.6 (bash) | Dart objects/strings | `flutter_aot`; fallback to `strings libapp.so`. | + +## Deep evidence escalation + +| Intent | Utility | Output | When to use | +|---|---|---|---| +| Focused call/data context | Joern recipes (bash) | CPG slices/query output | Only on shortlisted methods/files. The Joern MCP exists but its volume budget doesn't justify adoption — see workflow.md. | +| Path-precise Java/Kotlin evidence | CodeQL recipes (bash) | SARIF/query results | Public/MASVS-grade claims needing precise path evidence. | +| Lifecycle-aware taint | FlowDroid (bash) | taint paths | High-impact lifecycle-spanning claims. | +| Normalize hypotheses | `normalize_semantic_findings` MCP tool | Markdown/JSONL normalized report with MASVS/CWE/MASWE tags | Final report generation. | + +## Quick decision tree + +1. **Have APK(s), no decompile yet?** `inventory_status` → `run_corpus_inventory` → `extract_components` → `rank_components` → `detect_runtime_kind`. +2. **Single APK?** Use `android-targeted-assessment`; decompile only that APK. +3. **APKiD flagged a packer / `.dat` blobs in assets?** `detect_protector` first; if `dexprotector_unpack_supported`, run `dexprotector_unpack` before any other reading. +4. **Rich backend signs?** Run `extract_api_map` immediately after decompile and read `backend_richness.json`. +5. **Hybrid shell?** Do JS/Dart analysis before grading Java-side findings. +6. **Finding formed?** Add scanner baseline label, confidence tier, validation tier, missing evidence. Run `normalize_semantic_findings` for the report. +7. **Backend/API claim?** Default to `needs_backend_validation` and write a tier2/tier3 validation plan unless authorized testing already happened. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/apk-to-backend-api.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/apk-to-backend-api.md new file mode 100644 index 0000000..5729c74 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/apk-to-backend-api.md @@ -0,0 +1,187 @@ +# APK-to-backend API vulnerability paths + +Use this when the likely impact is not contained inside the APK. Many high-value Android apps are thin clients for rich API backends whose real authorization, workflow, object lifecycle, payment, identity, and moderation logic is only discoverable by starting from the APK. The APK gives you routes, auth mechanics, request signing, feature flags, protobuf schemas, GraphQL documents, WebSocket events, and hidden endpoint families that a normal web crawl will not reach. + +Default posture: static discovery and local reconstruction first. Only send requests to live backends with explicit authorization and the right validation tier. Treat production API probing, account-to-account tests, payment actions, invite abuse, and tenant boundary checks as `tier2_test_account_or_qa_backend` or `tier3_explicit_production_authorization`. + +## What makes an APK valuable as a backend map + +Prioritize apps where the APK suggests a complex backend that is hard to enumerate from the public web UI: + +- many API base URLs, versioned paths, or endpoint constants (`/v1/`, `/api/`, `/graphql`, `/rpc`, `/gateway`, `/mobile/`) +- generated clients: Retrofit, Apollo GraphQL, gRPC/protobuf, OpenAPI, Swagger, Moshi/Gson DTO forests, Kotlin serialization, Room-to-network sync layers +- request signing or anti-abuse layers: HMAC, nonce, timestamp, canonical request, device attestation, certificate pinning, encrypted bodies, custom headers +- rich object models: account, user, tenant, org, family, child, vault, device, payment, order, booking, ride, ticket, chat, room, group, invite, membership, subscription +- workflow verbs: create, accept, approve, transfer, redeem, claim, activate, enroll, recover, reset, verify, bind, pair, link, migrate, escalate, impersonate, delegate +- realtime/event transports: WebSocket, MQTT, SSE, SignalR, Socket.IO, Firebase RTDB/Firestore, push-driven command handlers +- web content bridged to API actions: in-app browser, WebView JS bridge, deep link to WebView, web checkout, OAuth, support/chat widgets + +## Bug classes to hunt from APK-derived API maps + +### 1. BOLA / IDOR in mobile-only object APIs + +Shape: + +```text +APK DTO / endpoint reveals object ID parameter + -> object belongs to account/tenant/family/vault/chat/order/payment/device + -> client passes ID in path/query/body/header + -> app gates UI locally or derives allowed IDs locally + -> backend authorization for substituted ID is unknown or missing +``` + +High-value sinks: + +- `/users/{id}`, `/accounts/{accountId}`, `/tenants/{tenantId}`, `/orgs/{orgId}` +- `/families/{familyId}/children/{childId}` +- `/vaults/{vaultId}/items/{itemId}` +- `/orders/{orderId}`, `/trips/{tripId}`, `/tickets/{ticketId}` +- `/rooms/{roomId}`, `/messages/{messageId}`, `/attachments/{attachmentId}` +- `/devices/{deviceId}`, `/sessions/{sessionId}`, `/subscriptions/{subscriptionId}` + +Evidence standard: static endpoint + object model + auth context is a hypothesis. Claiming BOLA needs authorized dynamic validation with two test accounts/tenants or explicit backend evidence. + +### 2. Server-side workflow confusion / state machine bypass + +Shape: + +```text +APK exposes multi-step flow + -> client tracks step/state/role locally or in request fields + -> API has verbs like approve/accept/complete/activate/recover/transfer + -> hidden or out-of-order request may skip confirmation, KYC, payment, 2FA, invite ownership, or admin approval +``` + +Hunt examples: + +- call `complete` without `start` / `verify` +- accept invite with a different account than the intended recipient +- approve device pairing after only local unlock +- redeem/claim/activate with reused or cross-account token +- change email/phone/payment instrument before verification completes +- replay an old signed request if nonce/timestamp is weak + +### 3. Mobile API mass assignment / client-trusted fields + +Shape: + +```text +DTO contains fields the UI never exposes or marks read-only + -> request serializer sends whole object or mutable map + -> fields look privilege-bearing: role, isAdmin, premium, verified, ownerId, tenantId, price, discount, status, scope + -> backend may accept client-supplied field +``` + +Static signals: + +- request classes with both user-controlled and server-controlled fields +- `copy(...)`, `toMap()`, `HashMap`, `JSONObject.put`, `@SerializedName`, `@JsonClass`, Kotlin data classes sent wholesale +- fields named `role`, `admin`, `owner`, `verified`, `status`, `state`, `plan`, `price`, `amount`, `discount`, `scope`, `permissions`, `entitlements` + +### 4. SSRF / open redirect / URL fetch through backend APIs + +Shape: + +```text +APK passes URL/domain/callback/image/avatar/webhook/preview/import parameter + -> backend fetches, previews, redirects, imports, or stores the remote resource + -> URL validation is unknown or weak in client +``` + +High-value endpoints: + +- link preview / unfurl / OpenGraph fetch +- avatar/profile import by URL +- file import from URL / cloud sync +- webhook callback / redirect URL / return URL +- OAuth `redirect_uri`, SSO metadata, support/chat upload +- QR/barcode scanned URL submitted to backend + +Static evidence is a lead. Backend SSRF/open redirect claims require authorized dynamic validation and safe canary endpoints. + +### 5. GraphQL / gRPC / protobuf hidden operations + +Shape: + +```text +APK ships query documents, operation names, persisted query hashes, proto descriptors, or generated stubs + -> operations reveal mobile-only admin/support/payment/device flows + -> object IDs and role/state fields map to BOLA, mass assignment, or workflow bypass probes +``` + +Search for: + +- GraphQL: `.graphql`, `ApolloClient`, `operationName`, `query`, `mutation`, `persistedQuery`, `sha256Hash`, `__typename` +- gRPC/protobuf: `.proto`, `MethodDescriptor`, `io.grpc`, `GeneratedMessageLite`, `parseFrom`, `toByteArray`, `service`, `rpc` +- custom RPC: `/rpc/`, `/gateway/`, `method`, `params`, `jsonrpc`, `procedure` + +Do not assume introspection is enabled. The APK itself is the schema fragment. + +### 6. WebView-to-backend bridge paths + +Shape: + +```text +WebView/deep link loads web content + -> JS bridge or URL handler calls native API client with mobile auth/session/device context + -> web-origin controls method/args or route params + -> backend action executes as mobile user/device +``` + +This is the APK-to-web-vuln crossover: a web bug becomes higher impact because the web content can reach mobile-only native API methods, auth headers, device identifiers, or signed request helpers. + +## Grep profile: backend API map + +Run this after first-party source narrowing. For React Native / Flutter / hybrid apps, run equivalent searches over JS bundle strings / Hermes strings / Dart `pp.txt` too. + +```bash +rg -n \ + -e 'https?://|/api/|/v[0-9]+/|/graphql|/grpc|/rpc|/gateway|/mobile|/internal' \ + -e 'Retrofit|OkHttpClient|Request\.Builder|HttpUrl|Volley|Ktor|ktor|ApolloClient|GraphQL|operationName|persistedQuery' \ + -e 'io\.grpc|ManagedChannel|MethodDescriptor|GeneratedMessageLite|parseFrom|toByteArray|protobuf|proto3' \ + -e 'WebSocket|Socket\.IO|SignalR|EventSource|SSE|MQTT|FirebaseFirestore|FirebaseDatabase' \ + -e 'Authorization|Bearer|X-Api-Key|apiKey|x-device|deviceId|installationId|sessionId|refreshToken' \ + -e 'Hmac|Mac\.getInstance|SHA256|signature|signRequest|nonce|timestamp|canonical|attestation|SafetyNet|PlayIntegrity' \ + -e 'tenantId|orgId|accountId|userId|ownerId|familyId|childId|vaultId|deviceId|orderId|paymentId|roomId|messageId' \ + -e 'role|isAdmin|verified|entitlement|premium|subscription|scope|permissions|status|state|price|amount|discount' \ + -e 'accept|approve|complete|activate|claim|redeem|recover|reset|verify|bind|pair|link|migrate|transfer' \ + -e 'callback|redirect_uri|returnUrl|webhook|avatarUrl|imageUrl|preview|unfurl|importUrl|sourceUrl' \ + "$SRC" > findings//rg-api-backend.txt +``` + +## Extraction checklist + +1. **Base URLs and environments** — prod/stage/dev hosts, API gateways, regional domains, CDN hosts, WebView origins. +2. **Auth material and headers** — bearer token source, refresh flow, device/session IDs, tenant/account headers, request-signature inputs. Do not extract real user secrets from production accounts. +3. **Endpoint inventory** — method, path, operation name, request DTO, response DTO, required auth state, feature flag. +4. **Object graph** — IDs and ownership boundaries: user/account/org/family/device/vault/order/chat/payment. +5. **Workflow graph** — allowed step order from app code; side-effecting verbs; confirmation/2FA/KYC gates. +6. **Client-only controls** — local booleans, feature flags, read-only DTO fields, disabled UI buttons, client-side amount/price/status calculations. +7. **Backend probe plan** — what must be tested with test accounts, what can be checked offline, and which actions are destructive. + +## Hypothesis wording + +Use `needs_backend_validation` unless the backend was tested under scope. A strong static hypothesis should say: + +```text +APK reveals endpoint /v2/families/{familyId}/children/{childId}/location and DTO ChildLocationRequest(childId, familyId, deviceId). The UI obtains childId from the locally cached family list, but the request builder accepts arbitrary IDs and no server-side authorization evidence is available in static code. Hypothesis: possible BOLA across family/child IDs. Validation requires two authorized test family accounts and read-only location endpoint checks. +``` + +Do **not** write: + +```text +The API is vulnerable to IDOR. +``` + +unless dynamic validation proves cross-account access. + +## Suggested class names + +- `apk_discovered_backend_bola` +- `apk_discovered_backend_workflow_bypass` +- `apk_discovered_backend_mass_assignment` +- `apk_discovered_backend_ssrf_or_open_redirect` +- `apk_discovered_graphql_operation_abuse` +- `apk_discovered_grpc_operation_abuse` +- `webview_bridge_to_mobile_api_action` +- `mobile_request_signing_replay_or_confusion` diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/backend-rich-apk-workflows.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/backend-rich-apk-workflows.md new file mode 100644 index 0000000..5762fef --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/backend-rich-apk-workflows.md @@ -0,0 +1,157 @@ +# Backend-rich APK workflows + +Use this after the normal component/deep-link triage identifies an app whose real value is the backend it exposes: finance, health, travel, IoT, enterprise, retail/loyalty, messaging, family/education, workforce, cloud files, password managers, or any app with rich generated API clients. + +The APK is treated as a protocol/workflow oracle. The output should be an API/workflow map and a validation queue, not unscoped backend probing. + +## Mode selection + +### Mode 1 — lightweight backend richness ranking + +Use before deep decompile or when ranking a corpus. Goal: identify APKs likely to contain a rich backend map. + +Inputs: + +- JADX source tree when available, or extracted strings/resources when not. +- Optional JS/Hermes/Dart analysis directories for hybrid apps. + +Outputs: + +- `api_map.jsonl` from `scripts/extract_api_map.py` +- `backend_richness.json` summary + +### Mode 2 — API map extraction + +Use after decompile for a target APK. Goal: inventory endpoints, clients, object IDs, workflow verbs, auth/signing, and URL-fetch surfaces. + +```bash +python3 scripts/extract_api_map.py \ + --src findings/decompiled//sources \ + --out findings//api_map.jsonl \ + --summary findings//backend_richness.json +``` + +For React Native / Flutter / hybrid apps, run additional maps over JS/Dart artifacts: + +```bash +python3 scripts/extract_api_map.py \ + --src findings//js-analysis \ + --out findings//api_map_js.jsonl \ + --summary findings//backend_richness_js.json +``` + +Then read top rows by category: + +```bash +jq -r 'select(.category=="endpoint" or .category=="graphql" or .category=="grpc") | [.file,.line,.kind,.value] | @tsv' findings//api_map.jsonl | head -100 +jq '.scores, .top_terms' findings//backend_richness.json +``` + +### Mode 3 — workflow/state-machine reconstruction + +Use when an endpoint family includes side-effecting verbs: + +- `accept`, `approve`, `complete`, `activate`, `claim`, `redeem`, `recover`, `reset`, `verify`, `bind`, `pair`, `link`, `migrate`, `transfer`, `cancel`, `refund`, `share`, `invite` + +Read the API client, request DTOs, ViewModel/Presenter, and confirmation UI around those verbs. Build a sequence: + +```text +workflow: device_recovery +steps: + 1. POST /recovery/start + 2. POST /recovery/verify + 3. POST /recovery/complete +client_controlled_fields: + userId, deviceId, recoveryToken, trustedDevice +server_authorization_unknown: + token bound to initiating account/device? final device already approved? +validation_tier: + tier2_test_account_or_qa_backend +``` + +### Mode 4 — bridge-to-backend trace + +Use for WebView/RN/Flutter/Capacitor/Cordova apps. + +Trace: + +```text +external route / WebView origin / JS message + -> native bridge or MethodChannel + -> API client / request signer + -> backend side-effect +``` + +Risk increases when the bridge exposes: + +- auth/session tokens or cookies +- device IDs / installation IDs +- request signing helpers +- payment/order/booking APIs +- file upload/import APIs +- recovery/device pairing APIs +- tenant/account switching APIs + +### Mode 5 — version-diff API archaeology + +Use when multiple APK versions are available. + +```bash +python3 scripts/extract_api_map.py --src findings/v1/sources --out findings/v1/api_map.jsonl --summary findings/v1/backend_richness.json +python3 scripts/extract_api_map.py --src findings/v2/sources --out findings/v2/api_map.jsonl --summary findings/v2/backend_richness.json +comm -13 \ + <(jq -r '.category+"\t"+.value' findings/v1/api_map.jsonl | sort -u) \ + <(jq -r '.category+"\t"+.value' findings/v2/api_map.jsonl | sort -u) +``` + +Prioritize newly added: + +- feature flags +- endpoint families +- workflow verbs +- request-signing code +- GraphQL operations / persisted hashes +- protobuf services / methods +- WebView bridge methods + +## Backend-richness scoring intuition + +High-priority APKs tend to have several of these: + +- many API hosts and endpoint constants +- Retrofit/Apollo/gRPC/WebSocket/Firebase clients +- request signing or device attestation +- many DTO classes with object IDs +- tenant/account/org/family/vault/device/payment/order concepts +- workflow verbs around recovery, invite, approval, transfer, redemption, checkout, pairing +- URL-fetch or callback parameters submitted to backend +- feature flag / remote config systems +- WebView/native bridge connected to API clients +- offline sync and conflict-resolution code + +## Validation boundaries + +Static APK-to-backend work produces excellent hypotheses, but most backend vulnerability claims need authorized dynamic evidence. + +Default labels: + +- `confidence_tier=needs_backend_validation` +- `validation_tier=tier2_test_account_or_qa_backend` for two-account or QA validation +- `validation_tier=tier3_explicit_production_authorization` for production account/tenant/payment/device state changes + +Safe first probes, when authorized: + +- non-destructive metadata reads +- invalid-token / wrong-device negative checks +- endpoint existence checks against QA +- URL preview checks with owned canary domains +- GraphQL query shape checks on non-sensitive objects + +Do not perform without explicit scope: + +- payment, refund, transfer, coupon/gift-card redemption +- account recovery / device pair completion +- cross-account object access in production +- invite/ownership changes +- destructive mutations +- probing real users, tenants, devices, health records, children/family data, or private messages diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/bug-classes.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/bug-classes.md new file mode 100644 index 0000000..31366fd --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/bug-classes.md @@ -0,0 +1,224 @@ +# Semantic Android bug classes + +Use this reference when reviewing `slices.jsonl` or turning a slice into a vulnerability hypothesis. The capability's bias is toward **impactful chains** that require app-specific reasoning and are likely to be missed or reduced to generic warnings by traditional scanners. + +## Review frame + +For every candidate chain, answer five questions: + +1. **Entrypoint** — Which external actor can reach the code path? Browser, another app, share sheet, notification, push, exported component, content provider, app link, custom scheme, WebView navigation, or backend-controlled config? +2. **Source** — Which values are attacker-controlled? URI scheme/host/path/query/fragment, Intent action/data/categories/extras, nested Intent, file/content URI, JavaScript message, local mutable state, server redirect, or app config? +3. **Trust boundary** — What assumption is wrong? The caller is trusted, the URL is first-party, the component is internal, local state is authoritative, the user/account context is unchanged, the nested Intent is safe, or the domain remains owned. +4. **Sink** — What sensitive operation happens? WebView navigation with auth context, component launch, API request, account/session mutation, token generation/validation, file/provider access, JavaScript bridge call, or privileged SDK action. +5. **Impact** — What can the attacker actually accomplish? Account takeover, auth bypass, token theft, private component access, unauthorized action, file/PII disclosure, tenant/account confusion, or payment/entitlement abuse. + +If one of the five is missing, keep the item as a hypothesis, not a finding. + +## Deep link / router chains + +Grounding: +- Android unsafe deep links: https://developer.android.com/privacy-and-security/risks/unsafe-use-of-deeplinks +- OWASP MASTG deep link testing: https://mas.owasp.org/MASTG/tests/android/MASVS-PLATFORM/MASTG-TEST-0028/ +- Oversecured deep link account takeover patterns: https://oversecured.com/blog/android-deep-link-vulnerabilities + +High-value patterns: + +- `BROWSABLE` Activity routes `Intent.getData()` into an internal router. +- Router accepts `url`, `redirect`, `next`, `returnUrl`, `callback`, `continue`, `target`, `path`, or `deeplink` parameters. +- Host validation uses `contains`, `startsWith`, `endsWith`, broad regex, string replacement, or URI parsing inconsistently. +- App Link/domain allowlist includes partner, staging, campaign, shortlink, or expired-looking domains. +- Deep link reaches OAuth, password reset, magic link, referral, wallet/payment, account settings, or support-chat flows. + +Validation ideas: + +- Try host-confusion payloads only in authorized test context: `https://trusted.com.attacker.tld/`, `https://trusted.com@attacker.tld/`, punycode/mixed-case/encoded-dot variants. +- Check whether `assetlinks.json` exists for App Links and whether custom schemes are used for OAuth callbacks. +- For stale domains, use passive DNS/WHOIS/HTTP status as evidence; do not take over domains unless explicitly authorized. + +## Intent redirection / private component reachability + +Grounding: +- Android intent redirection risk guidance: https://developer.android.com/privacy-and-security/risks/intent-redirection +- CodeQL intent redirection: https://codeql.github.com/codeql-query-help/java/java-android-intent-redirection/ +- Element Android intent-redirection case study: https://www.shielder.com/blog/2024/04/element-android-cve-2024-26131-cve-2024-26132-never-take-intents-from-strangers/ + +High-value patterns: + +- Exported component reads nested Intent via `getParcelableExtra`, `getSerializableExtra`, `Bundle.get`, or `Intent.parseUri` and launches it. +- Forwarded Intent preserves data URI, component name, flags, clip data, or grant flags. +- The target is private or performs privileged actions under the victim app identity. +- The exported component has no caller permission/signature check. +- Router actions can open internal screens that normally require login, KYC, 2FA, entitlement, or user confirmation. + +Validation ideas: + +- Build an explicit proof plan with `adb shell am start` for simple extras or a small helper app for nested Intent/Parcelable cases. +- Watch for `FLAG_GRANT_READ_URI_PERMISSION`, `ClipData`, and content provider access. +- Distinguish “can launch screen” from “can complete privileged action”; impact requires the latter or strong evidence it is reachable. + +## WebView trust-boundary chains + +Grounding: +- Android unsafe WebView URI loading: https://developer.android.com/privacy-and-security/risks/unsafe-uri-loading +- Android WebView native bridges: https://developer.android.com/privacy-and-security/risks/insecure-webview-native-bridges +- CodeQL unsafe Android WebView fetch: https://codeql.github.com/codeql-query-help/java/java-android-unsafe-android-webview-fetch/ +- Home Assistant Android WebView case: https://securitylab.github.com/advisories/GHSL-2023-142_Home_Assistant_Companion_for_Android/ + +High-value patterns: + +- External URL reaches `WebView.loadUrl`, `postUrl`, `loadDataWithBaseURL`, or a network request used to populate WebView content. +- `setJavaScriptEnabled(true)` and `addJavascriptInterface` coexist with externally controlled navigation. +- App attaches auth headers/cookies, user identifiers, bearer tokens, device IDs, or payment/session context to the request. +- `shouldOverrideUrlLoading` handles `intent://`, custom schemes, or app-internal routes. +- Bridge methods expose native actions, token/account data, file access, or privileged SDK calls. + +Validation ideas: + +- Confirm whether cookies/auth headers are present for attacker-influenced URL loads. +- Enumerate bridge method names and argument types from decompiled code. +- Mark as lower confidence when URL control exists but no authenticated context or sensitive bridge is found. + +## Auth/session/client-state chains + +Grounding: +- OWASP MASVS-AUTH (Authentication and Session Management): https://mas.owasp.org/MASVS/05-MASVS-AUTH/ +- MASTG-TEST-0017 Testing Stateful Session Management (Android): https://mas.owasp.org/MASTG/tests/android/MASVS-AUTH/MASTG-TEST-0017/ +- CWE-602 Client-Side Enforcement of Server-Side Security: https://cwe.mitre.org/data/definitions/602.html +- CWE-287 Improper Authentication: https://cwe.mitre.org/data/definitions/287.html +- CWE-862 Missing Authorization: https://cwe.mitre.org/data/definitions/862.html + +High-value patterns: + +- Login/session/entitlement decisions derived from `SharedPreferences`, SQLite, files, cache, extras, or feature flags. +- User ID, tenant ID, role, account ID, phone/email, or organization ID accepted from an Intent/deep link without server revalidation. +- Password reset, invite, promo, or magic-login tokens generated/validated entirely client-side. +- Hardcoded key participates in token signing/encryption, request signing, password reset, or account recovery. +- Local state gates privileged screens but backend calls lack matching authorization checks. + +Validation ideas: + +- State whether validation is static-only, local-device dynamic, or backend-dependent. +- For backend-dependent claims, require a test account and explicit authorization. +- Avoid overclaiming from local bypass alone; prove or plan how to prove server-side acceptance. + +## Deep link to account-state-change (no WebView) + +Grounding: +- OWASP MASVS-PLATFORM + MASVS-AUTH: https://mas.owasp.org/MASVS/06-MASVS-PLATFORM/ , https://mas.owasp.org/MASVS/05-MASVS-AUTH/ +- MASTG-TEST-0028 Testing Deep Links: https://mas.owasp.org/MASTG/tests/android/MASVS-PLATFORM/MASTG-TEST-0028/ +- MASWE-0058 Insecure Deep Links: https://mas.owasp.org/MASWE/MASVS-PLATFORM/MASWE-0058/ +- CWE-352 Cross-Site Request Forgery (equivalent for mobile deep-link CSRF): https://cwe.mitre.org/data/definitions/352.html +- CWE-862 Missing Authorization: https://cwe.mitre.org/data/definitions/862.html + +Distinct from the deep-link-to-WebView class: there is no WebView at all. The sink is a server-side state-changing API call (device registration, family/team invite accept, contact share-add, subscription enroll, account-key rotation, magic-link consume) that the app fires from a `LaunchedEffect` / `onCreate` / coroutine `launch` as soon as the BROWSABLE handler resolves the path. + +High-value patterns: + +- BROWSABLE Activity routes `Intent.getData()` query parameters straight into `NavDirections` arguments (`Bundle.putString("id", ...)`). +- The destination Composable / Fragment has a `LaunchedEffect` whose only precondition is `id != null && key != null` and which calls a ViewModel action that hits an "accept" / "complete" / "register" endpoint. +- The endpoint accepts attacker-supplied identifiers / keys / tokens with no server-side requirement that the *sender* device generated them. +- The pre-screen unlock prompt (if any) shows only a generic "unlock the app" string — not "you are about to register a new device" or "you are accepting an invite from X". + +**Illustrative example (internal corpus observation, password-manager class):** a `dashlane://mplesslogin?id=&key=` deep-link auto-completes a device-to-device registration in Dashlane v6.262 (2026-05) — the Java side runs the full transfer-completion crypto flow without an explicit confirmation modal between the BROWSABLE handler and the `MplessCompleteTransferService.execute(...)` call. This is a static-analysis finding from internal corpus research, not a vendor-confirmed advisory; the *shape* (BROWSABLE → `LaunchedEffect` → state-changing API with deep-link args) is what to look for in any password-manager / 2FA / identity-class APK. See `../../../references/sources.md` for the provenance framing. + +Validation ideas: + +- Distinguish "deep link can open the screen" from "deep link can complete the action" by reading every `LaunchedEffect` / `init` / coroutine `launch` on the destination screen for **side-effect calls that take the deep-link args directly**. +- For Compose Navigation, the `NavDirections.getArguments()` Bundle is the trust boundary — anything passed in there becomes ViewModel input on next composition. Trace every consumer. +- A confirmation gate that is a *button* in the same Composable is still a gate — the bug is only present when the action fires before any user interaction on the destination screen. + +## APK-discovered backend API chains + +Use this class when the APK is the map to a rich backend, and the likely vulnerability is a web/API issue rather than an on-device issue. Full workflow: `apk-to-backend-api.md`. + +High-value patterns: + +- APK reveals mobile-only endpoint families, GraphQL operations, gRPC/protobuf stubs, WebSocket events, feature-flagged routes, or request DTOs that are not visible from the public web app. +- Object IDs cross authorization boundaries: `userId`, `accountId`, `tenantId`, `orgId`, `familyId`, `childId`, `vaultId`, `deviceId`, `orderId`, `paymentId`, `roomId`, `messageId`, `attachmentId`. +- Workflow verbs can be called out of order or under a different account context: `accept`, `approve`, `complete`, `activate`, `claim`, `redeem`, `recover`, `reset`, `verify`, `bind`, `pair`, `link`, `migrate`, `transfer`. +- Request DTOs contain privilege-bearing or server-owned fields: `role`, `isAdmin`, `verified`, `ownerId`, `tenantId`, `status`, `state`, `price`, `discount`, `scope`, `permissions`, `entitlements`. +- URL/callback parameters are submitted to backend fetchers or redirectors: `redirect_uri`, `returnUrl`, `callback`, `webhook`, `avatarUrl`, `imageUrl`, `preview`, `unfurl`, `importUrl`, `sourceUrl`. +- WebView/JS bridge can invoke native API clients with mobile auth/session/device context, turning a web-origin bug into a backend action. +- Request-signing layers expose replay/confusion surfaces: HMAC canonicalization, nonce/timestamp handling, device ID binding, attestation bypass/fallback. + +Validation ideas: + +- Static endpoint + DTO + object model evidence is a **backend hypothesis**, not a confirmed web vuln. +- Label most candidates `needs_backend_validation`; proving BOLA/workflow/mass-assignment requires authorized test accounts, QA backend, or explicit production authorization. +- Prefer read-only probes first: object metadata, status endpoints, preview endpoints, non-destructive GraphQL queries. +- For destructive flows (payment, transfer, invite, device pair, account recovery), write a validation plan instead of probing unless scope is explicit. +- Record whether the APK provides enough information to reconstruct required headers/signatures without extracting real user secrets. + +## Content/file/provider exposure chains + +Grounding: +- Microsoft Dirty Stream: https://www.microsoft.com/en-us/security/blog/2024/05/01/dirty-stream-attack-discovering-and-mitigating-a-common-vulnerability-pattern-in-android-apps/ +- Android untrusted ContentProvider filename guidance: https://developer.android.com/privacy-and-security/risks/untrustworthy-contentprovider-provided-filename +- ownCloud Android provider SQLi / path validation case: https://securitylab.github.com/advisories/GHSL-2022-059_GHSL-2022-060_Owncloud_Android_app/ + +High-value patterns: + +- Exported `ContentProvider`, broad `FileProvider` paths, path traversal in provider/openFile logic, or URI grants forwarded to nested Intents. +- Share/import flows copy attacker-controlled URIs into private storage or expose private files back through share targets. +- **Dirty Stream shape:** exported `ACTION_SEND` / `ACTION_SEND_MULTIPLE` / import target reads a malicious `content://` URI, trusts `OpenableColumns.DISPLAY_NAME`, `Intent.EXTRA_TITLE`, or caller-supplied path, then writes under `cacheDir`, `filesDir`, databases, shared prefs, WebView state, plugin/config, or code-loading directories. +- **Provider SQLi shape:** exported provider without signature permission feeds caller-controlled `selection`, `projection`, `sortOrder`, URI segment, or table key into `rawQuery`, `execSQL`, `SQLiteQueryBuilder`, or custom query construction without `setStrict` / `setProjectionMap` / hardcoded table routing. +- Provider access depends on path/account parameters supplied by the caller. +- Logs, databases, cached auth material, attachments, health/financial documents, message media, shared preferences, or app configuration are reachable. + +Validation ideas: + +- Use read-only provider queries in a local emulator/test device where authorized. +- For Dirty Stream, use a local helper app/provider with a malicious display name; do not require a live backend. +- Separate “provider exported” from “sensitive rows/files reachable.” +- Separate “can write a file” from impact; impact requires showing that overwritten file is later trusted, executed, uploaded, or used as auth/config state. +- Record required permissions and whether signature permissions block external callers. + +## Non-prod host / endpoint reachable from production (leaked-host chain) + +Grounding: +- OWASP MASVS-CODE (Code Quality and Build Settings) + MASVS-NETWORK: https://mas.owasp.org/MASVS/08-MASVS-CODE/ , https://mas.owasp.org/MASVS/04-MASVS-NETWORK/ +- MASTG-TECH-0025 Automated Static Analysis (scanner caveats apply): https://mas.owasp.org/MASTG/techniques/android/MASTG-TECH-0025/ +- CWE-1188 Insecure Default Initialization of Resource: https://cwe.mitre.org/data/definitions/1188.html +- CWE-489 Active Debug Code (the build-config sub-pattern): https://cwe.mitre.org/data/definitions/489.html + +Distinct from "secret in DEX" or "hardcoded URL": the production APK *intentionally* ships a non-prod hostname (QA, staging, dev, sandbox, dogfood, canary, internal) and *also* ships the selector logic to pick it. The question is whether the selector is reachable by a third party in a production-signed install. + +For the full triage runbook, outcome schema, and known false-positive patterns (R8-retained dead objects, regex consumers, OAuth scope literals), see `leaked-host-triage.md`. + +The gate-classification matrix. Categories A1/A2/A3 are runtime reads; A4 is structurally distinct (build-time / startup-time pin) and surfaced during a leaked-host triage pass on popular Google Play apps — internal corpus observation; see `../../../references/sources.md` for provenance. + +| Gate category | Resolves at | Third-party reachable? | Grade | +|---|---|---|---| +| **A1** `BuildConfig.DEBUG`, `isDevDebug()`, `isDebug()` | runtime, but build-pinned constant after R8 | no | `build_config_gated` / production-safe | +| **A2** server-flippable feature flag (ECS, LaunchDarkly, Split.io, FirebaseRemoteConfig, Statsig, Optimizely, custom config) | runtime, vendor-controlled | partially — vendor's flag console can target arbitrary users | `feature_flag_gated` / **promote** to `strong_static_chain` + `tier2_test_account_or_qa_backend` | +| **A3** intent extra, deep-link query param, exported settings-pref writer | runtime, third-party-controlled | yes | `intent_extra_or_query_param_gated` / **promote** to `strong_static_chain` + `tier1_local_device_no_live_backend` | +| **A4** Dagger compile-time `Factory.bind*`, app-startup `Initializer` that overrides persisted state, or post-R8 build-flavor string compare (`"release" == "bet"`) | build time / cold start | no | `build_config_gated` / production-safe | + +A4 sub-patterns: + +- **A4-dagger**: a `dagger.internal.Factory` whose `bindXxx` / `provideXxx` returns a hard-wired enum (`return new FixedEnvironmentRepository(QaMode.PRODUCTION)`). No runtime read; the dagger graph for the release flavor cements the value. +- **A4-bootstrap**: an `androidx.startup.Initializer` / `EagerInitializer` / `Application.onCreate` calls `setConfig(compileTimeValue)` with a flag that explicitly *ignores* any persisted user state on every cold start. Even if an exported settings activity exists and writes the env SharedPreferences value, the bootstrap initializer reverts it before the first network call. +- **A4-flavor**: a `"release" == "bet"` literal compare (R8 substitutes the flavor string per build). The production flavor evaluates false; the gate writers inside are dead. + +A4 is a stronger guarantee than A1 because A1 is at least observable in the bytecode as a runtime read (`BuildConfig.DEBUG.get()` style), whereas A4 either disappears in dagger's generated factory (no runtime read at all) or reverts at startup. Recording A4 vs A1 in `gate_evidence` helps the next pass spot the same shape faster. + +**Illustrative outcome distribution** from an internal corpus pass over 14 popular Google Play apps that shipped a non-prod hostname literal in DEX (static analysis only; provenance per `../../../references/sources.md`): + +- 1 / 14 was a real `feature_flag_gated` shape — production banking app using a Split.io string flag (`mobl_AEMService_QAEnvironment` in Chase Mobile, swapping AEM content-CDN hosts among 4 QA tiers). +- 3 / 14 were A4 build-pin: eBay (A4-dagger, `EnvironmentRepositoryModule_BindEnvironmentRepositoryFactory` → `QaMode.PRODUCTION`), FitBit (A4-bootstrap, `HttpConfigInitializer` calls `initializeEnvironmentConfig(_, /*z=*/false)`), ESPN (A4-flavor, `if ("release" == "bet")`). +- 10 / 14 were `unreachable` (R8-leftover Kotlin top-level objects, regex consumers, OAuth scope literals, JSON fixtures). + +High-value patterns: + +- Production binary contains a `URLProvider` / `EnvironmentManager` / `BackendEnvironment` / `Endpoints` enum with 3+ entries (typically `PROD`, `STAGE`, `DEV`/`QA`). +- The current environment is selected by reading a SharedPreferences string, a feature-flag client, or an enum-by-name lookup keyed off a SharedPreferences value. +- A consumer of the per-environment host loads attacker-controlled content into a WebView, video player, or content fragment renderer inside an authenticated session — the leak can chain into a CDN-content-control finding. +- The setter for the env field is reachable from a non-exported "ServerSettingsActivity" but a bootstrap initializer always overrides it (A4-bootstrap) — flag the gap but record as production-safe. +- The feature-flag service uses targeting by account ID, so flipping for one user does not flip for everyone — impact is bounded to the targeted user set but is not a per-device toggle. + +Validation ideas: + +- Static-only: record `gate_kind`, `gate_evidence`, and the consumer code path. Distinguish A1 from A4 sub-patterns in `gate_evidence`. +- Local device (tier1): for A3 only — use `adb shell am start` with the deep-link / intent extra, or build a small helper app to write the exported SharedPreferences value, then `adb shell am force-stop` + relaunch and inspect network traffic. +- Test account (tier2): for A2 only — request a targeted flag value from the vendor's console for an authorized test account and observe. +- Do not probe live QA hostnames without explicit per-target authorization; the QA tier is usually *internal* (banking, retail, fitness) infrastructure not in public DNS. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/codeql-recipes.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/codeql-recipes.md new file mode 100644 index 0000000..c2b2894 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/codeql-recipes.md @@ -0,0 +1,100 @@ +# CodeQL recipes for Android semantic review + +CodeQL is the path-precision tier. Use it when a finding needs SARIF-grade evidence for a public/MASVS-level report or you're filing a vendor disclosure. + +Source docs: + +- Query help index for Java/Kotlin: https://codeql.github.com/codeql-query-help/java/ +- Built-in Java/Kotlin queries: https://docs.github.com/en/code-security/reference/code-scanning/codeql/codeql-queries/java-kotlin-built-in-queries +- Android intent redirection query: https://codeql.github.com/codeql-query-help/java/java-android-intent-redirection/ +- Unsafe Android WebView fetch query: https://codeql.github.com/codeql-query-help/java/java-android-unsafe-android-webview-fetch/ +- Kotlin analysis support (extends Java): https://github.blog/changelog/2022-11-28-codeql-code-scanning-launches-kotlin-analysis-support-beta/ + +## Caveat — JADX source has limits + +CodeQL was designed to run against a *buildable* project, not arbitrary decompiled output. JADX sources sometimes: + +- omit Kotlin metadata +- include syntactically rough code (`--show-bad-code`) +- lack a build graph + +Database creation succeeds for many real APKs but not all. Treat CodeQL as **optional escalation**, not a corpus tool. If DB creation fails, fall back to Joern (which handles JADX trees more permissively) or commit to the source project if the user has it. + +## Step 1 — create a CodeQL database from JADX source + +The `build-mode=none` indexer skips the build and ingests Java/Kotlin sources directly. + +```bash +SRC=findings/decompiled//sources +DB=findings/codeql/-db + +codeql database create "$DB" \ + --language=java \ + --source-root "$SRC" \ + --build-mode=none \ + --overwrite +``` + +For Kotlin-heavy apps the same Java indexer handles Kotlin under the same language flag. + +## Step 2 — install the query packs + +```bash +codeql pack download codeql/java-queries +``` + +## Step 3 — run the Android-specific queries + +```bash +RESULTS=findings/codeql/.sarif + +codeql database analyze "$DB" \ + codeql/java-queries:Security/CWE/CWE-926/AndroidIntentRedirection.ql \ + codeql/java-queries:Security/CWE/CWE-079/UnsafeAndroidAccess.ql \ + codeql/java-queries:Security/CWE/CWE-749/AndroidWebViewJavaScriptSettings.ql \ + codeql/java-queries:Security/CWE/CWE-201/SensitiveAndroidFileLeak.ql \ + --format=sarif-latest \ + --output "$RESULTS" +``` + +The full Android query set is browsable under https://codeql.github.com/codeql-query-help/java/ — search the page for "Android". + +A quick way to run *every* Android query is the security-extended suite: + +```bash +codeql database analyze "$DB" \ + codeql/java-queries:codeql-suites/java-security-extended.qls \ + --format=sarif-latest --output "$RESULTS" +``` + +Security-extended is louder but useful when you don't yet know which class of bug applies. + +## Step 4 — turn SARIF into evidence + +Each SARIF `result` has `ruleId`, `locations` (file/line), `codeFlows` for path-graph queries, and `partialFingerprints` for dedup. Attach these to your hypothesis as `evidence`: + +```bash +jq -r ' + .runs[].results[] | + [.ruleId, + (.locations[0].physicalLocation.artifactLocation.uri), + (.locations[0].physicalLocation.region.startLine|tostring), + (.message.text // "")] + | @tsv +' "$RESULTS" +``` + +For path-graph results (intent redirection, WebView fetch), iterate `result.codeFlows[].threadFlows[].locations` to extract the full source → sink chain. The chain belongs in the finding's `evidence` array verbatim — that is what CodeQL is for. + +## Custom queries + +When you find a recurring bug pattern that isn't covered by the default pack, write a small custom `.ql` file. The pattern usually looks like a `TaintTracking::Configuration` between an Android source predicate (intent extras, deep-link query params) and a project-specific sink. Keep custom queries beside the capability's outputs (e.g. `findings/codeql/queries//.ql`) and cite them in the hypothesis. + +## When CodeQL is the wrong tool + +Skip CodeQL and rely on Joern when: + +- the APK is heavily obfuscated and class names mean little +- you only need triage context for the agent's reasoning, not external evidence +- DB creation fails or takes longer than the finding is worth +- the bug class is purely behavioural (e.g. backend trust assumptions) and no Android query pack covers it diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/corpus-acquisition.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/corpus-acquisition.md new file mode 100644 index 0000000..cde76e9 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/corpus-acquisition.md @@ -0,0 +1,153 @@ +# APK corpus acquisition + +Use this reference when the user wants popular or common APKs for research. The goal is a lawful, reproducible corpus with provenance, not a mystery folder of binaries. + +## Best options + +### 1. User-provided owned or authorized APKs + +Best for actionable findings. Ask the user to provide: + +- APK directory path +- authorization/scope note +- app category or package list +- whether dynamic validation is allowed +- whether test accounts are available + +### 2. `gplaydl` (PyPI) — anonymous Google Play, recommended primary + +For *currently-listed* Play apps this is now the fastest path. `gplaydl` (v2.x, March 2026, actively maintained) authenticates anonymously via [Aurora Store](https://auroraoss.com/)'s public token dispenser, rotating ~20 device profiles. No Google account, no API key. + +- PyPI: https://pypi.org/project/gplaydl/ +- Source: https://github.com/rehmatworks/gplaydl +- Speed: **~20-25 MB/s per stream** against Google's CDN, scales with parallelism. Roughly 30-60× faster than AndroZoo per APK in our measurements. +- Use `scripts/gplaydl_bulk.py` for bounded parallel JSONL-driven bulk pulls. + +**Cannot fetch:** +- delisted apps (Zenly, Facebook Lite, old Outlook Lite, Kiwi Browser, etc.) +- paid apps (Threema, OnePassword 7, etc.) +- region-locked apps under the current token's locale +- beta-channel-only apps + +For all of those, fall back to AndroZoo (below). + +**Terms note.** This operates in a gray zone of Google's ToS. Aurora's dispenser uses accounts provisioned for anonymous read access; Google has tolerated this for years but never blessed it. Appropriate for research, not for commercial redistribution. Don't abuse Aurora's free public dispenser — keep parallelism ≤ 8 for normal corpora. + +### 3. AndroZoo — historical, paid, and delisted apps + +AndroZoo is the standard research corpus for Android APKs and metadata. It requires access/API key approval and supports reproducible selection by package, market, date, size, VT metadata, and hash. + +- Project: https://androzoo.uni.lu/ +- Catalog: 27 M APK rows, 4.4 M packages, multi-market (Play + 14 others). Historical versions retained. +- **Throughput: ~440 KB/s per connection, up to ~20 concurrent. Plan accordingly.** +- Quota: 500,000 APKs per 6-month key cycle. +- Use when you need historical versions, delisted apps, or paid apps that researchers have archived. +- Selection: convert `latest_with-added-date.csv.gz` + `gp-metadata-aggregate.jsonl.gz` to Parquet (`scripts/androzoo_to_parquet.py`) once, then run DuckDB queries against the columnar files. +- Keep the query that produced the corpus so findings can be reproduced. + +### 4. Official store/device extraction + +Good for testing “popular APKs” without relying on mirror sites: + +- Install apps from Google Play on a test device/account where terms and authorization permit. +- Extract installed package paths with `adb shell pm path `. +- Pull APK splits with `adb pull`. +- Preserve package name, version, split list, and install source. + +Example: + +```bash +adb shell pm list packages | sed 's/package://' > packages.txt +adb shell pm path com.example.app +adb pull /data/app/.../base.apk corpus/com.example.app/base.apk +``` + +Split APKs/APK bundles may need all split files, not only `base.apk`. + +### 4. F-Droid + +Best fully open-source corpus for pipeline debugging, but less representative of high-impact consumer-account logic. + +- Repo/index: https://f-droid.org/repo/index-v1.json +- Advantages: clear licensing/provenance, easy downloads, source code available for validation. +- Limitation: fewer finance/retail/identity targets and less closed-source routing/SDK complexity. + +### 5. APKMirror / APKPure style mirrors + +Use cautiously. They are convenient for popular APKs but weaker on authorization, provenance, terms, and reproducibility. If used, record exact URL, SHA256, package, version, and download time. Prefer official/device extraction or AndroZoo for serious research. + +## Strong target profile for semantic logic bugs + +Prioritize apps with: + +- login/account/session flows +- OAuth/Social login/SSO +- payments, wallet, bank, travel, retail, telecom, health, identity, enterprise SaaS +- many deep links/App Links/custom schemes +- WebView-heavy hybrid flows +- partner/campaign/shortlink domains +- push notification/deferred deep link SDKs +- file sharing, messaging, attachments, documents, or support chat +- React Native/Flutter/Capacitor/Cordova bridges + +Avoid spending early cycles on games, static content apps, or tiny utilities unless inventory shows rich entrypoints. + +## Practical first corpus + +For the first capability test, use 20-50 APKs: + +1. 5-10 known vulnerable/training APKs to sanity-check detection behavior. +2. 10-20 F-Droid or internal apps to exercise scale and references legally. +3. 10-20 popular apps obtained through AndroZoo or official device extraction if available. + +Keep a manifest: + +```json +{"package":"com.example","version":"1.2.3","source":"androzoo|device|fdroid|user","sha256":"...","downloaded_at":"...","notes":"..."} +``` + +## Selection query ideas + +- Size below 55 MB at first; decompilation and LLM slicing are cheaper. +- Recent releases from the last 12-24 months. +- Exclude obvious malware-only feeds for this workflow unless the user wants malware analysis. +- Include categories where auth/session/payment/data logic matters. +- Deduplicate by package and keep the newest version plus one older version if regression hunting is useful. + + + + +## Google Play metadata aggregate for popularity-based selection + +AndroZoo's Google Play metadata page is the best way to find *popular* targets before selecting APK hashes. The aggregate file has one JSONL record per app and includes aggregated fields such as star ratings, rating/comment counts, and download counts. It is Google Play-only. If used in research output, cite the AndroZoo 2024 metadata paper in addition to the 2016 AndroZoo paper when APKs are used. + +Capability boundary: + +- Run large downloads as operator-supervised scripts: `scripts/androzoo_gp_metadata.py download`, `scripts/androzoo_to_parquet.py`, and `scripts/androzoo_download.py` (or `scripts/gplaydl_bulk.py` for currently-listed Play apps). +- Selection itself is `duckdb -c "SELECT ..."` against the Parquet files produced by `androzoo_to_parquet.py` — see the `android-corpus-prep` skill for the canonical popular-package / impact-class selection recipe. + +Download aggregate metadata: + +```bash +python3 scripts/androzoo_gp_metadata.py download \ + --kind aggregate \ + --out corpus/androzoo/meta/gp-metadata-aggregate.jsonl.gz \ + --progress +``` + +Convert both shipped sources to ZSTD Parquet once, then query with DuckDB. See the `android-corpus-prep` skill for the canonical recipe — popular-package selection plus clean-Play-APK join in seconds, sub-200 MB peak RAM. + +Download selected APKs. The key is read from `ANDROZOO_API_KEY`; do not pass it inline unless necessary: + +```bash +python3 scripts/androzoo_download.py \ + corpus/androzoo/apk-selection.jsonl \ + --out-dir corpus/androzoo/apks \ + --manifest-out corpus/androzoo/download_manifest.jsonl \ + --sleep 0.5 +``` + +Important limitations: + +- App-level metadata fields reflect the date metadata was acquired, not necessarily the version release date. Use them for target selection, not historical claims about a specific APK version. +- AndroZoo asks users to keep concurrency modest; the downloader is sequential by default. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/feature-flag-mining.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/feature-flag-mining.md new file mode 100644 index 0000000..cdeabfd --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/feature-flag-mining.md @@ -0,0 +1,56 @@ +# Feature flag and experiment mining + +Feature flags are target generators. They expose backend workflows before they are visible in the UI, explain dormant routes, and identify rollout states where authorization assumptions are changing. + +## Look for + +- LaunchDarkly: `LDClient`, `variation`, `allFlags`, `ld_key` +- Firebase Remote Config: `FirebaseRemoteConfig`, `getBoolean`, `getString`, `fetchAndActivate` +- Split.io: `SplitClient`, `getTreatment` +- Optimizely: `Optimizely`, `isFeatureEnabled` +- Statsig: `Statsig`, `checkGate`, `getExperiment` +- Unleash: `Unleash`, `isEnabled` +- custom config endpoints: `/features`, `/experiments`, `/config`, `/remote-config`, `/bootstrap`, `/flags` +- local flag stores: SharedPreferences / DataStore / Room tables named flags, experiments, config, treatments + +## Grep + +```bash +rg -n \ + -e 'LaunchDarkly|LDClient|FirebaseRemoteConfig|RemoteConfig|SplitClient|Optimizely|Statsig|Unleash' \ + -e 'featureFlag|feature_flag|experiment|variant|treatment|isEnabled|getBoolean|getString|variation|checkGate' \ + -e 'enable_|disable_|rollout|gate|killSwitch|remote_config|fetchAndActivate|allFlags' \ + -e '/features|/experiments|/config|/remote-config|/bootstrap|/flags' \ + "$SRC" > findings//rg-feature-flags.txt +``` + +## Prioritize flags with security meaning + +High signal substrings: + +- `recovery`, `reset`, `migrate`, `transfer`, `pair`, `trusted_device`, `device_verification` +- `kyc`, `identity`, `document`, `verification`, `risk`, `fraud` +- `payment`, `checkout`, `refund`, `coupon`, `promo`, `gift`, `wallet`, `limit` +- `family`, `child`, `guardian`, `team`, `org`, `tenant`, `admin`, `role` +- `webview`, `bridge`, `native_bridge`, `external_url`, `deeplink`, `dynamic_link` +- `graphql`, `grpc`, `api_v2`, `new_backend`, `gateway`, `mobile_api` +- `bypass`, `skip`, `fallback`, `debug`, `staging`, `internal`, `dogfood`, `beta` + +## How to turn flags into hypotheses + +1. Find flag definition and default. +2. Find every call site. +3. Read the enabled and disabled branch. +4. Identify newly exposed endpoint families or workflow verbs. +5. Check whether the flag only hides UI or also changes backend authorization. +6. Record whether remote config can be influenced by account/tenant/device cohort. + +Hypothesis examples: + +- `enable_device_recovery_v2` gates a new `/recovery/complete` endpoint whose DTO accepts `deviceId` and `trustedDevice`. +- `skip_kyc_for_low_value_transfer` gates a client-only transfer threshold check; backend validation is unknown. +- `new_graphql_checkout` adds persisted mutation hashes for checkout and coupon redemption not present in web UI. + +## Validation boundary + +Feature flags rarely prove a vulnerability by themselves. They are routing evidence. Most findings remain `needs_backend_validation` until tested under authorized account/tenant conditions. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/historical-patterns-2023-2026.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/historical-patterns-2023-2026.md new file mode 100644 index 0000000..aa235c0 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/historical-patterns-2023-2026.md @@ -0,0 +1,125 @@ +# Historical Android APK vulnerability patterns (2023-2026) + +Use this as a pattern catalogue, not as a prevalence study. The public record is biased toward vulnerabilities that received CVEs, vendor advisories, conference/blog writeups, or bug-bounty disclosure. Search terms used to build this reference also bias toward semantic APK issues that are easy to describe publicly: deep links, WebViews, intent redirection, providers, and share/import flows. Absence here does **not** mean a class is rare or low impact; it means it was less visible in the public sources sampled. + +The point for corpus hunting is to copy the **shape** of repeatedly impactful findings into our ranking, grep profiles, and hypothesis taxonomy. + +## Pattern index + +| Pattern | Public signal | Entry/source/sink shape | Hunt bias | +|---|---|---|---| +| Deep link to authenticated WebView | Home Assistant Android `CVE-2023-41898`; Rakuten Ichiba Android `CVE-2024-41918`; bug-bounty reports against retail/travel/social apps | BROWSABLE/custom scheme receives URL-like param -> weak allowlist -> `WebView.loadUrl` with auth cookies/headers or phishing context | Search-heavy bias toward disclosed WebView bugs; still high ROI because scanners often flag only generic WebView APIs. | +| Intent redirection / private component reachability | Element Android `CVE-2024-26131` / `CVE-2024-26132`; Android Developers intent-redirection risk guidance; SDK-level wallet cases reported publicly after 2025 disclosure | Exported component accepts nested `Intent` / string `Intent.parseUri` -> launches it under victim app identity -> private component, grant, or privileged flow | Public examples overrepresent messaging/wallet apps, but the proxy pattern appears in many app classes. | +| Dirty Stream / untrusted ContentProvider filename | Microsoft Dirty Stream research in 2024; Xiaomi File Manager and WPS Office fixed; Android Developers ContentProvider filename guidance | Share/import target accepts attacker `content://` -> trusts `_display_name` / title / path -> writes under app-private storage -> overwrite config/token/code/cache | Very actionable statically; public signal is high because Microsoft/Google amplified it. Treat as first-class in file/cloud/email/messenger/office apps. | +| Provider SQL injection / overbroad exported provider | ownCloud Android `CVE-2023-24804`, `CVE-2023-23948`; older Nextcloud class remains relevant | Exported provider with no signature permission -> caller controls `selection`, `projection`, URI path, or table routing -> internal DB rows leak/modify | CVEs skew to open-source apps where provider code is reviewable; closed-source APKs still expose the same API shapes. | +| Share-target path traversal | ownCloud ReceiveExternalFilesActivity issues; Basecamp bug-bounty reports; recurring Nextcloud/Talk-style reports | `ACTION_SEND` / `EXTRA_STREAM` / `EXTRA_TEXT` / `EXTRA_TITLE` -> filename/path from caller -> `File(cacheDir, name)` / upload path / temp path -> read/write outside intended directory | Often bounty-disclosed rather than CVE-assigned; grep for it explicitly because manifest-only ranking undercounts it. | +| WebView native bridge origin confusion | TikTok-style JS interface takeover reports; Android Developers native bridge guidance updated 2024 | Attacker-controlled page/frame -> `addJavascriptInterface` / `postWebMessage` / `WebMessagePort` -> native methods expose account/token/file/SDK actions | Public examples skew toward social/super-apps; in corpora, RN/Flutter/hybrid shells need JS/Dart route-map validation before grading. | +| Deep link to account-state change without WebView | Publicly less CVE-rich but recurring in bounty writeups and internal corpus work | BROWSABLE route args -> navigation/ViewModel init/`LaunchedEffect` -> `accept` / `register` / `complete` / `recover` / `pair` API call before explicit confirmation | Easy to miss if the review stops at “opens a screen.” Requires reading destination side effects. | + +## Pattern recipes + +### 1. Deep link to authenticated WebView + +Grounding: + +- Android unsafe deep links: https://developer.android.com/privacy-and-security/risks/unsafe-use-of-deeplinks +- Android unsafe WebView URI loading: https://developer.android.com/privacy-and-security/risks/unsafe-uri-loading +- Home Assistant Android arbitrary WebView URL: https://securitylab.github.com/advisories/GHSL-2023-142_Home_Assistant_Companion_for_Android/ +- Rakuten Ichiba custom URL scheme issue: https://jvn.jp/en/jp/JVN56648919/ + +Grep: + +```bash +rg -n \ + -e 'getIntent\(\)\.getData|getDataString|getQueryParameter' \ + -e 'url|redirect|next|returnUrl|callback|continue|target|deeplink' \ + -e 'loadUrl|postUrl|loadDataWithBaseURL' \ + -e 'setJavaScriptEnabled\s*\(\s*true|addJavascriptInterface|postWebMessage|WebMessagePort' \ + -e 'CookieManager|Authorization|Bearer|getAuthHeaders|setCookie' \ + "$SRC" +``` + +Promote only when the chain has URL control, weak/missing scheme+host validation, and authenticated context or a sensitive bridge/action. URL control alone is `hardening_only` or `needs_route_map_validation`. + +### 2. Intent redirection / private component reachability + +Grounding: + +- Android intent redirection: https://developer.android.com/privacy-and-security/risks/intent-redirection +- Element Android writeup: https://www.shielder.com/blog/2024/04/element-android-cve-2024-26131-cve-2024-26132-never-take-intents-from-strangers/ + +Grep: + +```bash +rg -n \ + -e 'getParcelableExtra.*Intent|getSerializableExtra.*Intent|Bundle\.getParcelable|Intent\.parseUri' \ + -e 'startActivity|startService|bindService|sendBroadcast' \ + -e 'FLAG_GRANT_READ_URI_PERMISSION|FLAG_GRANT_WRITE_URI_PERMISSION|ClipData' \ + -e 'resolveActivity|IntentSanitizer|setPackage|setComponent' \ + "$SRC" +``` + +Promote only when an exported/no-permission component launches attacker-controlled nested intents without a strict component/package/data/flag allowlist. Distinguish “screen open” from a privileged action or grant leak. + +### 3. Dirty Stream / untrusted ContentProvider filename + +Grounding: + +- Microsoft Dirty Stream: https://www.microsoft.com/en-us/security/blog/2024/05/01/dirty-stream-attack-discovering-and-mitigating-a-common-vulnerability-pattern-in-android-apps/ +- Android ContentProvider filename guidance: https://developer.android.com/privacy-and-security/risks/untrustworthy-contentprovider-provided-filename + +Grep: + +```bash +rg -n \ + -e 'ACTION_SEND|ACTION_SEND_MULTIPLE|EXTRA_STREAM|EXTRA_TEXT|EXTRA_TITLE|ClipData' \ + -e 'OpenableColumns\.DISPLAY_NAME|MediaStore\.MediaColumns\.DISPLAY_NAME|getColumnIndex.*display' \ + -e 'openInputStream|openFileDescriptor|ContentResolver\.query' \ + -e 'FileOutputStream|Files\.copy|copyTo|new File\(|openFileOutput|writeBytes' \ + -e 'getCacheDir|getFilesDir|cacheDir|filesDir' \ + -e 'canonicalPath|getCanonicalPath|normalize|createTempFile|sanitize' \ + "$SRC" +``` + +Promote only when attacker-controlled filename/title/display name reaches a private-storage write without temp-file generation, basename sanitization, and canonical path enforcement. Impact requires a trusted later read/use of the overwritten file. + +### 4. Provider SQL injection / overbroad provider + +Grounding: + +- ownCloud Android provider issues: https://securitylab.github.com/advisories/GHSL-2022-059_GHSL-2022-060_Owncloud_Android_app/ + +Grep: + +```bash +rg -n \ + -e 'class .*ContentProvider|extends ContentProvider' \ + -e 'query\(|insert\(|update\(|delete\(|openFile\(' \ + -e 'rawQuery|execSQL|SQLiteQueryBuilder|selection|projection|sortOrder' \ + -e 'setStrict|setProjectionMap|appendWhere|UriMatcher' \ + "$SRC" +``` + +Promote only when a provider is exported or reachable with grants, lacks signature permission, and caller-controlled SQL/path inputs can reach sensitive rows/files. + +### 5. Deep link to account-state change without WebView + +Grep: + +```bash +rg -n \ + -e 'getQueryParameter\(.*(?:id|key|token|code|invite|pair|device|account)' \ + -e 'LaunchedEffect|init\s*\{|viewModelScope\.launch|lifecycleScope\.launch|onCreate' \ + -e 'accept|complete|register|recover|pair|join|claim|activate|verify|transfer' \ + -e 'Retrofit|OkHttp|enqueue|suspend fun|execute\(' \ + "$SRC" +``` + +Promote only when the route fires a server/account-state mutation before explicit user confirmation and backend authorization is absent or needs validation. + +## Capability implications + +- Component ranking should boost exported share/import targets in addition to BROWSABLE routes. +- `E_file_cloud`, `G_messenger`, `H_email`, and office/retail classes should get Dirty Stream/path-traversal search terms by default. +- Report taxonomy should distinguish `dirty_stream_file_overwrite`, `share_target_path_traversal`, `exported_provider_sqli`, `intent_redirection_uri_grant_leak`, and `deep_link_to_js_bridge` instead of flattening everything into “WebView” or “provider exposure.” +- Because the source set is public-writeup biased, corpus reports should state whether a pattern was selected due to historical signal or due to local corpus frequency. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/hybrid-bridge-to-backend.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/hybrid-bridge-to-backend.md new file mode 100644 index 0000000..22d0e1b --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/hybrid-bridge-to-backend.md @@ -0,0 +1,63 @@ +# Hybrid bridge to backend tracing + +Use for React Native, Flutter, Capacitor, Cordova, and WebView-heavy apps where external web/JS/Dart input may reach native API clients. This is the main APK -> web -> backend crossover path. + +## Trace shape + +```text +external input + -> deep link / WebView URL / postMessage / JS route / Dart route + -> native bridge / MethodChannel / EventChannel / Cordova plugin / Capacitor plugin + -> API client / request signer / token store + -> backend side effect +``` + +## Bridge inventory + +Java/Kotlin: + +```bash +rg -n \ + -e 'addJavascriptInterface|@JavascriptInterface|postWebMessage|WebMessagePort|onPostMessage' \ + -e 'MethodChannel|EventChannel|BasicMessageChannel|setMethodCallHandler|invokeMethod' \ + -e 'PluginMethod|CapacitorPlugin|CordovaPlugin|execute\(' \ + -e 'ReactContextBaseJavaModule|@ReactMethod|NativeModule|RCTDeviceEventEmitter' \ + "$SRC" > findings//rg-bridges.txt +``` + +JS/Hermes strings: + +```bash +rg -nN \ + -e 'postMessage|addEventListener|Linking|NativeModules|TurboModuleRegistry|bridge|invoke|emit' \ + -e 'fetch\(|axios|graphql|mutation|query|WebSocket|socket\.emit' \ + -e 'token|Authorization|deviceId|sessionId|accountId|tenantId' \ + "$JSDIR" > findings//rg-js-bridge-backend.txt +``` + +Dart/Flutter strings: + +```bash +rg -nN \ + -e 'MethodChannel|EventChannel|invokeMethod|setMethodCallHandler' \ + -e 'http\.|Dio\(|GraphQLClient|WebSocketChannel|FirebaseFirestore' \ + -e 'token|Authorization|deviceId|sessionId|accountId|tenantId' \ + findings//dart-analysis/pp.txt > findings//rg-dart-bridge-backend.txt +``` + +## High-risk bridge methods + +- token/session access: `getToken`, `refreshToken`, `getCookies`, `setCookie` +- signed requests: `sign`, `signedFetch`, `request`, `apiCall`, `hmac` +- account state: `acceptInvite`, `completeRecovery`, `verifyDevice`, `switchAccount` +- payments/orders: `checkout`, `pay`, `refund`, `redeem`, `applyCoupon` +- file APIs: `upload`, `import`, `share`, `openFile`, `download` +- device/IoT: `pair`, `bind`, `unlock`, `command`, `provision` + +## Promote when + +- web/JS/Dart-controlled data reaches a native method that performs an authenticated/signed API call, and +- origin/route/argument validation is absent, partial, or performed before an attacker-controllable redirect/frame/message boundary, and +- the backend action has account, payment, device, file, tenant, or privacy impact. + +Most findings are `webview_bridge_to_mobile_api_action` or `deep_link_to_js_bridge` and require route-map plus backend validation. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/impact-class-rg-profiles.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/impact-class-rg-profiles.md new file mode 100644 index 0000000..821e7e6 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/impact-class-rg-profiles.md @@ -0,0 +1,257 @@ +# Per-impact-class focused-rg profiles + +Use these as the starting `rg` pattern set after JADX decompile. They extend, not replace, the universal source/sink set in `android-semantic-vuln-hunting`. The point is to spend the first minute reading the lines a given app *class* is most likely to ship a bug in, rather than the universal set that's right for every app. + +Each profile assumes you have: + +```bash +SRC=findings//jadx/sources/ +``` + +and want one file of hits per APK. + +## A_ingress — QR / barcode / docscan / NFC + +The bug pattern: scanned data is rendered or executed without origin/sanity checks. Look for: + +- decoder output flowing into `Uri.parse` → `loadUrl`, `startActivity`, `Intent.parseUri` +- vCard / WiFi-config / contact / calendar URI parsing +- ML Kit / ZXing result callbacks pasted into the deep-link dispatcher +- camera-side intent filters that re-fire intents based on scanned content + +```bash +rg -n \ + -e 'decodeQrCode|parseQrPayload|decodeBarcode|onBarcodeDetected|onQrCodeDetected' \ + -e 'BarcodeFormat|BarcodeReader|MultiFormatReader|Result\.getText\(' \ + -e 'ZXing|com\.google\.mlkit\.vision\.barcode|BarcodeAnalyzer' \ + -e 'Uri\.parse\([^)]*(?:result|barcode|qr|payload|scanned)' \ + -e 'WifiConfig|MeCard|vCard|VCard|geo:|tel:|mailto:|sms:' \ + -e 'Intent\(.*Uri\.parse\(' -e 'startActivity\(.*Uri\.parse\(' \ + -e 'loadUrl\([^)]*(?:result|scanned|payload)' \ + -e 'getExtras\(\).get\(.*(?:result|payload|qr|barcode)' \ + -e 'onActivityResult.*(?:RequestCode.*QR|Scan|Barcode)' \ + "$SRC" > findings//rg-class-A.txt +``` + +## B_remote — TeamViewer / AnyDesk / AirDroid / RDP + +The bug pattern: pairing or session establishment trusts a token / code / URI from an attacker-influenced channel. Also exported services that handle pairing requests. Backend angle: remote-control/session APIs often expose device IDs, partner IDs, pairing workflows, relay endpoints, and websocket commands that need cross-account and replay review. + +```bash +rg -n \ + -e 'AccessibilityService|onAccessibilityEvent|performGlobalAction' \ + -e 'MediaProjection|VirtualDisplay|ScreenCapture|createScreenCaptureIntent' \ + -e 'pairingCode|partnerId|sessionToken|sessionId|connectionId|deviceId|accountId|tenantId' \ + -e 'startSession|joinSession|acceptInvite|inviteCode|approve|complete|bind|pair|link' \ + -e 'Retrofit|OkHttpClient|WebSocket|Socket|MQTT|grpc|protobuf|Authorization|Bearer|signature|nonce|timestamp' \ + -e 'getQueryParameter\(.*(?:code|token|session|partner|invite|pairing)' \ + -e 'startActivity\(.*FLAG_ACTIVITY_NEW_TASK' \ + -e 'addJavascriptInterface|loadUrl' \ + -e 'Permission\.SYSTEM_ALERT_WINDOW|TYPE_APPLICATION_OVERLAY|TYPE_SYSTEM_ALERT' \ + -e 'Settings\.canDrawOverlays|requestPermission.*overlay' \ + "$SRC" > findings//rg-class-B.txt +``` + +## C_wallet — crypto wallets, exchanges, DEX + +The bug pattern: dapp link → transaction signing without explicit per-call origin/contract gating; WalletConnect session hijack; weak chain/RPC validation; in-app browser WebView with JS bridge that exposes wallet state. Backend angle: exchange/wallet APIs often expose order, quote, KYC, device, subscription, webhook/callback, and request-signing mechanics that are difficult to map without the APK. + +```bash +rg -n \ + -e 'wc:|walletconnect|WalletConnect|WCSession|WCClient|RelayClient' \ + -e 'reown\.|Reown|com\.reown\.|dispatchEnvelope' \ + -e 'signTransaction|signMessage|signTypedData|personal_sign|eth_send|eth_sign' \ + -e 'sendTransaction|sendRawTransaction|broadcastTransaction' \ + -e 'getActiveSession|approveSession|rejectSession' \ + -e 'dappUrl|originUrl|requestUrl|metadata\.url|peerMetadata' \ + -e 'Web3|Web3Modal|RPC|provider\.send' \ + -e 'mnemonic|seed_phrase|seedPhrase|privateKey|keystore' \ + -e 'BiometricPrompt|FingerprintManager|KeyguardManager' \ + -e 'addJavascriptInterface|setJavaScriptEnabled' \ + -e 'QuestBrowser|DappBrowser|InAppBrowser|WebViewActivity' \ + -e 'shareData|getCurrentAppState|updateFingerprint' \ + -e 'Adjust\.processDeeplink|AdjustDeeplink|Branch\.initSession|RNBranchModule' \ + -e 'flutter_webview_plugin|MethodChannel.*[Ww]eb' \ + -e 'getQueryParameter\(.*(?:address|amount|chain|callback|tx)' \ + -e 'orderId|quoteId|paymentId|accountId|userId|kyc|tier|limit|price|amount|fee|discount' \ + -e 'Hmac|Mac\.getInstance|signature|nonce|timestamp|canonical|attestation|PlayIntegrity' \ + -e 'Retrofit|OkHttpClient|ApolloClient|GraphQL|/graphql|/api/|/v[0-9]+/|WebSocket' \ + "$SRC" > findings//rg-class-C.txt +``` + +Pattern recurrence across C_wallet targets (MetaMask, Trust Wallet, SafePal): every one hit on `addJavascriptInterface` + `WalletConnect`/`Reown` + one of {`RNBranchModule`, `Adjust.processDeeplink`, `flutter_webview_plugin`}. **The third-party deep-link consumer (Branch/Adjust) is consistently called *before* the app's own host allowlist runs** — worth a dedicated read on every C_wallet APK. + +## D_secret — password managers, 2FA, authenticators + +The bug pattern: autofill or accessibility service trusts the foreground app id without strict matching; clipboard exposure; URI handler dumps secrets into the wrong window. Backend angle: vault/password-manager APIs expose device registration, recovery, sharing, family/team membership, and item IDs that are prime BOLA/workflow-bypass candidates once mapped from the APK. + +```bash +rg -n \ + -e 'AutofillService|onFillRequest|FillResponse|AutofillId|AssistStructure' \ + -e 'AccessibilityService|onAccessibilityEvent|AccessibilityNodeInfo' \ + -e 'getInstalledApplications|getPackagesForUid|getApplicationInfo' \ + -e 'getRunningAppProcesses|getRunningTasks|UsageStatsManager' \ + -e 'ClipboardManager|setPrimaryClip|getPrimaryClip' \ + -e 'BiometricPrompt|BiometricManager|KeyGenParameterSpec' \ + -e 'KeyStore|MasterKey|EncryptedSharedPreferences|Cipher\.getInstance' \ + -e 'addJavascriptInterface|JavaScriptInterface|@JavascriptInterface' \ + -e 'getQueryParameter\(.*(?:totp|otp|secret|seed|token|code)' \ + -e 'Authenticator|TOTP|HOTP|generateOTP' \ + -e 'vaultId|itemId|secretId|familyId|orgId|teamId|deviceId|recovery|shareId|inviteId' \ + -e 'accept|approve|complete|activate|recover|reset|verify|bind|pair|migrate|transfer' \ + -e 'Retrofit|OkHttpClient|ApolloClient|GraphQL|/graphql|/api/|/v[0-9]+/|Hmac|signature|nonce' \ + "$SRC" > findings//rg-class-D.txt +``` + +## E_file_cloud — file managers, cloud sync + +The bug pattern: ContentProvider exports a too-broad ``, FileProvider grants flow into wrong intents, share-target accepts arbitrary `content://` URIs and re-broadcasts them with grants. Recent historical signal adds **Dirty Stream**: import/share targets trust attacker-controlled provider filenames (`OpenableColumns.DISPLAY_NAME`, `EXTRA_TITLE`) and overwrite app-private files. + +```bash +rg -n \ + -e 'ContentProvider|getContentResolver|openInputStream|openOutputStream|openFileDescriptor' \ + -e 'FileProvider|getUriForFile|FLAG_GRANT_READ_URI_PERMISSION|FLAG_GRANT_WRITE_URI_PERMISSION' \ + -e 'grantUriPermission|revokeUriPermission|takePersistableUriPermission' \ + -e 'ACTION_SEND|ACTION_SEND_MULTIPLE|ACTION_GET_CONTENT|ACTION_OPEN_DOCUMENT|EXTRA_STREAM|EXTRA_TEXT|EXTRA_TITLE|ClipData' \ + -e 'OpenableColumns\.DISPLAY_NAME|MediaStore\.MediaColumns\.DISPLAY_NAME|getColumnIndex.*display|DocumentsContract|DocumentFile\.fromSingleUri' \ + -e 'FileOutputStream|Files\.copy|copyTo|writeBytes|openFileOutput|new File\(' \ + -e 'getCacheDir|getFilesDir|cacheDir|filesDir|StorageVolume|Environment\.getExternalStorage' \ + -e 'canonicalPath|getCanonicalPath|normalize|createTempFile|sanitize|replace\("\.\."' \ + -e 'rawQuery|execSQL|SQLiteQueryBuilder|selection|projection|sortOrder|setStrict|setProjectionMap' \ + -e 'webdav|WebDAV|nextcloud|owncloud' \ + -e 'getQueryParameter\(.*(?:path|file|url|src|download)' \ + -e 'startActivity\(.*VIEW.*Uri' \ + -e '"\.\./|"\.\.\\\\\\\\' \ + "$SRC" > findings//rg-class-E.txt +``` + +## F_family — parental control, location tracking + +The bug pattern: exported services accept pairing codes that bind the device to a remote account; location streams have no origin check; admin commands can be triggered via deep link. + +```bash +rg -n \ + -e 'DeviceAdminReceiver|DevicePolicyManager|onPasswordChanged|onPasswordExpiring' \ + -e 'FusedLocationProvider|LocationManager|requestLocationUpdates' \ + -e 'AccessibilityService|onAccessibilityEvent' \ + -e 'pairingCode|invitationCode|familyCode|joinCode' \ + -e 'getQueryParameter\(.*(?:code|invite|family|child|parent|pair)' \ + -e 'AdminPin|adminPin|PARENT_PIN|parentPin' \ + -e 'Geofence|addGeofences|GeofencingClient' \ + -e 'BackgroundService|JobScheduler|WorkManager.*Periodic' \ + -e 'sendTextMessage|SmsManager' \ + "$SRC" > findings//rg-class-F.txt +``` + +## G_messenger + +The bug pattern: rich-link preview fetcher follows attacker-controlled URLs without scope; chat-protocol deep links populate auth context; file-attach view chain leaks grants; share/import handlers trust attachment filenames or nested intents. Backend angle: messaging APIs expose room/message/attachment/member IDs, invite workflows, link preview fetchers, and realtime event methods that map directly to BOLA/SSRF/workflow probes. + +```bash +rg -n \ + -e 'LinkPreview|OpenGraph|ogImage|ogDescription|fetchPreview' \ + -e 'addJavascriptInterface|setJavaScriptEnabled|shouldOverrideUrlLoading' \ + -e 'CookieManager|setCookie' \ + -e 'invitationLink|inviteLink|joinLink|tg:|sgnl:|threema:|line:|wickr:' \ + -e 'StickerProvider|stickers\.android' \ + -e 'getParcelableExtra\(.*EXTRA_STREAM|EXTRA_INTENT|Intent\.parseUri' \ + -e 'ACTION_SEND|ACTION_SEND_MULTIPLE|EXTRA_STREAM|EXTRA_TITLE|ClipData|OpenableColumns\.DISPLAY_NAME' \ + -e 'openInputStream|FileOutputStream|Files\.copy|new File\(|getCacheDir|getFilesDir' \ + -e 'startActivity\(.*Intent.*data' \ + -e 'getQueryParameter\(.*(?:invite|token|chat|user|room|server)' \ + -e 'roomId|messageId|attachmentId|memberId|userId|serverId|inviteId|groupId|channelId' \ + -e 'GraphQL|ApolloClient|/graphql|WebSocket|Socket\.IO|SignalR|EventSource|FirebaseFirestore|FirebaseDatabase' \ + -e 'preview|unfurl|OpenGraph|webhook|callback|redirect_uri|returnUrl|avatarUrl|imageUrl' \ + -e 'Notification.*setContentIntent|PendingIntent\.getActivity' \ + -e 'Linkify|spannable|URLSpan' \ + "$SRC" > findings//rg-class-G.txt +``` + +## H_email + +The bug pattern: HTML body rendering with insufficient sanitization; MIME parsing tricks; attachment open chains; mailto: handlers redirected to attacker WebViews; attachment import/export trusts caller-supplied filenames. + +```bash +rg -n \ + -e 'MimeMessage|MimeMultipart|MimeBodyPart|MimeUtility|MimeType' \ + -e 'WebView.*loadDataWithBaseURL|loadData|loadUrl' \ + -e 'setJavaScriptEnabled\s*\(\s*true|addJavascriptInterface' \ + -e 'MailTo|mailto:|message/rfc822' \ + -e 'CalendarContract|Events\.CONTENT_URI' \ + -e 'X-Originating-IP|Received: from|Return-Path' \ + -e 'S/MIME|PGP|PgpKey|Mailvelope' \ + -e 'getParcelableExtra\(.*EXTRA_STREAM|EXTRA_EMAIL|Intent\.parseUri' \ + -e 'ACTION_SEND|ACTION_SEND_MULTIPLE|EXTRA_STREAM|EXTRA_TITLE|ClipData|OpenableColumns\.DISPLAY_NAME' \ + -e 'FileProvider|getUriForFile|openInputStream|FileOutputStream|Files\.copy|new File\(' \ + -e 'getQueryParameter\(.*(?:subject|body|cc|bcc|to|attach)' \ + "$SRC" > findings//rg-class-H.txt +``` + +## I_browser + +The bug pattern: custom-tab intent forwarding hands out cookies/origins; URI-scheme dispatch loops; built-in settings pages with implicit JS bridges. + +```bash +rg -n \ + -e 'CustomTabsIntent|CustomTabsClient|CustomTabsSession|CustomTabsCallback' \ + -e 'addJavascriptInterface|@JavascriptInterface|setJavaScriptEnabled' \ + -e 'shouldOverrideUrlLoading|WebViewClient|WebChromeClient' \ + -e 'intent://|intent:.*S\.browser_fallback_url' \ + -e 'Intent\.parseUri|parseUri' \ + -e 'getCookie|setCookie|CookieManager' \ + -e 'getQueryParameter\(.*(?:url|src|next|return|redirect)' \ + -e 'TrustedWebActivity|TWA|setNavigationBarColor' \ + -e 'about:|chrome:|file:|content:|javascript:' \ + -e 'PWA|service-worker|manifest\.webmanifest' \ + "$SRC" > findings//rg-class-I.txt +``` + +## J_iot — smart home companion apps + +The bug pattern: LAN discovery + JSON-over-HTTP control with weak origin checks; OAuth-via-WebView; push tokens binding accounts to devices. Backend angle: IoT companion apps often reveal device ownership APIs, MQTT/WebSocket command channels, provisioning tokens, and cloud-to-LAN bridge methods that need tenant/device authorization review. + +```bash +rg -n \ + -e 'mDNS|NsdManager|MulticastSocket|SSDP|UPnP|Bonjour' \ + -e 'WifiManager|WifiNetworkSpecifier|WifiConfiguration|connectToWifi' \ + -e 'BluetoothGatt|BluetoothLeScanner|ScanCallback|scanResult' \ + -e 'addJavascriptInterface|setJavaScriptEnabled' \ + -e 'OAuth|oauth_token|authorize|redirect_uri' \ + -e 'mqtt|Mqtt|MqttClient|MqttAndroidClient|paho\.client\.mqttv3' \ + -e 'PushToken|FCM_TOKEN|registrationToken|onTokenRefresh' \ + -e 'deviceId|deviceUuid|deviceSecret|pairingToken|provisioningToken|homeId|tenantId|accountId|ownerId' \ + -e 'getQueryParameter\(.*(?:device|token|home|account|server)' \ + -e 'command|setState|unlock|arm|disarm|invite|shareDevice|transferOwner|bind|pair|provision' \ + -e 'Retrofit|OkHttpClient|WebSocket|MQTT|FirebaseFirestore|grpc|protobuf|Hmac|signature|nonce|timestamp' \ + -e 'http://(192|10|172)\.' \ + "$SRC" > findings//rg-class-J.txt +``` + +## How to use these + +```bash +# Read the per-APK class from the triage manifest +CLASS=$(jq -r --arg p "" 'select(.package==$p) | .impact_class' findings//triage/manifest.jsonl) + +# Then run the matching profile +case "$CLASS" in + A_ingress) bash scripts/run_class_rg.sh A "$SRC" findings// ;; + B_remote) bash scripts/run_class_rg.sh B "$SRC" findings// ;; + C_wallet) bash scripts/run_class_rg.sh C "$SRC" findings// ;; + D_secret) bash scripts/run_class_rg.sh D "$SRC" findings// ;; + E_file_cloud) bash scripts/run_class_rg.sh E "$SRC" findings// ;; + F_family) bash scripts/run_class_rg.sh F "$SRC" findings// ;; + G_messenger) bash scripts/run_class_rg.sh G "$SRC" findings// ;; + H_email) bash scripts/run_class_rg.sh H "$SRC" findings// ;; + I_browser) bash scripts/run_class_rg.sh I "$SRC" findings// ;; + J_iot) bash scripts/run_class_rg.sh J "$SRC" findings// ;; +esac +``` + +Or just paste the right block above into the shell. The profiles are deliberately small enough to read. + +## Notes on what's missing + +- These are **starting points**, not exhaustive. Add patterns as you find them in real APKs. +- Cross-class patterns (deep-link routers, JS bridges, FileProvider grants) still belong in the universal source/sink set in `android-semantic-vuln-hunting`. The per-class sets capture **what's unique to the class**, not what's common across all Android apps. +- For React Native / Capacitor / Cordova shells, complement with **Step 7.5** in `android-semantic-vuln-hunting`: pretty-print `assets/index.bundle` and grep the JS-side bridge surface. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/joern-recipes.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/joern-recipes.md new file mode 100644 index 0000000..cef955e --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/joern-recipes.md @@ -0,0 +1,161 @@ +# Joern recipes for Android semantic review + +Joern's role in this capability is **deep context on shortlisted methods**, not corpus scanning. By the time you reach Joern you have: + +- a decompiled JADX source tree +- specific files/methods from `rg`/Semgrep that look like real chains +- an exported BROWSABLE component or WebView entrypoint to anchor on + +Use Joern to answer: who calls this, where does this value come from, what does this bridge expose, does this validator actually run before that sink? + +Reference docs: + +- Joern overview and CPG concept: https://docs.joern.io/code-property-graph/ +- Quickstart and query language: https://docs.joern.io/quickstart/ +- Source repository and release notes: https://github.com/joernio/joern + +## Importing a JADX source tree + +JADX writes Java sources under `/sources/`. Joern's `javasrc2cpg` frontend handles that path directly. Memory usage scales with corpus size — for one large bank/wallet APK, 8–16 GB of heap is comfortable. + +Interactive shell: + +```scala +joern> importCode(inputPath = "findings/decompiled//sources", projectName = "") +joern> save +``` + +Headless script: + +```bash +cat > /tmp/import.sc <<'EOF' +@main def main(src: String, name: String) = { + importCode(inputPath = src, projectName = name) + save +} +EOF +joern --script /tmp/import.sc --param src=findings/decompiled//sources --param name= +``` + +Subsequent sessions open the saved project with `open("")` and skip the import cost. + +## Recipe 1 — exported activities reading attacker input + +For each Activity that the manifest exports (you already have this list from `androguard.json`), check whether its lifecycle methods read attacker-controlled input. + +```scala +val exported = Seq("com.target.LoginActivity", "com.target.DeepLinkActivity") // from androguard.json + +cpg.method + .where(_.typeDecl.fullNameExact(exported: _*)) + .name("(onCreate|onNewIntent|onStart|onResume)") + .where(_.callee.name("getIntent|getData|getStringExtra|getParcelableExtra|getQueryParameter")) + .map(m => (m.typeDecl.fullName.head, m.name, m.filename, m.lineNumber.getOrElse(0))) + .l +``` + +For each hit, follow the value with `cpg.call.name("loadUrl|startActivity|startService").where(_.argument.code(".*intent.*"))` to find sinks. + +## Recipe 2 — deep link source → WebView sink + +This is the canonical chain. The Android Developers page describes the unsafe-deep-link pattern: https://developer.android.com/privacy-and-security/risks/unsafe-use-of-deeplinks. + +```scala +def webviewSinks = cpg.call.name("loadUrl|postUrl|loadDataWithBaseURL") +def intentSources = cpg.call.name("getIntent|getData|getStringExtra|getQueryParameter") + +webviewSinks + .reachableByFlows(intentSources) + .take(25) + .map(_.elements.map(n => s"${n.method.fullName}:${n.lineNumber.getOrElse(0)}").l) + .l +``` + +If `reachableByFlows` returns too many false positives on obfuscated trees, pin the entrypoint: + +```scala +def entry = cpg.typeDecl.fullNameExact("com.target.DeepLinkActivity").method.l +webviewSinks + .reachableByFlows(intentSources.where(_.method.in(entry))) + .take(25) + .l +``` + +## Recipe 3 — what does the JavaScript bridge expose? + +CodeQL's unsafe-WebView-fetch query treats `addJavascriptInterface` as a high-trust boundary: https://codeql.github.com/codeql-query-help/java/java-android-unsafe-android-webview-fetch/. + +```scala +// Enumerate every class registered as a JavaScript interface. +cpg.call.name("addJavascriptInterface") + .argument + .isCall + .typeFullName + .l + +// For each bridge class, list its @JavascriptInterface-annotated public methods. +val bridgeTypes = Seq("com.target.bridge.NativeBridge") +cpg.typeDecl.fullNameExact(bridgeTypes: _*).method + .where(_.annotation.name("JavascriptInterface")) + .map(m => (m.fullName, m.parameter.l.map(_.code).mkString(", "))) + .l +``` + +Bridges that expose tokens, account state, file access, intent launch, or arbitrary URL navigation are the actual finding. + +## Recipe 4 — intent redirection / private component reachability + +CodeQL pattern: https://codeql.github.com/codeql-query-help/java/java-android-intent-redirection/. + +```scala +// Exported components forwarding a nested Intent obtained from getParcelableExtra/getSerializableExtra. +val nestedIntentSources = cpg.call.name("getParcelableExtra|getSerializableExtra|parseUri") +val componentLaunches = cpg.call.name("startActivity|startActivityForResult|startService|bindService|sendBroadcast") + +componentLaunches + .reachableByFlows(nestedIntentSources) + .take(25) + .l +``` + +Pair with `androguard.json` to confirm the containing class is `exported=true` and lacks a meaningful `permission`. + +## Recipe 5 — client-side auth/session decisions + +```scala +val localState = cpg.call.name("getSharedPreferences|getBoolean|getString|getInt") +val authDecisions = cpg.call.name("isLoggedIn|hasValidSession|checkAuth|canAccess|isPremium|isAdmin") + +authDecisions + .reachableByFlows(localState) + .take(25) + .l + +// Hardcoded keys that participate in token/account logic. +cpg.literal.code("\".{32,}\"") + .where(_.method.name(".*(token|reset|sign|verify|encrypt|decrypt|password).*")) + .map(l => (l.method.fullName, l.lineNumber.getOrElse(0), l.code)) + .l +``` + +## Recipe 6 — weak deep-link host validation + +```scala +// Calls to host validators on strings derived from intent data. +val hostsFromIntent = cpg.call.name("getHost|getQueryParameter").argument +val weakChecks = cpg.call.name("contains|startsWith|endsWith|matches") +weakChecks + .where(_.argument.reachableBy(hostsFromIntent)) + .map(c => (c.method.fullName, c.lineNumber.getOrElse(0), c.code)) + .take(50) + .l +``` + +Each hit is a candidate for host-confusion payloads such as `https://trusted.com.attacker.tld/` or `https://trusted.com@attacker.tld/`. + +## Operational notes + +- Increase Joern's heap when importing large APKs: `export JAVA_OPTS='-Xmx12g'` before launching. +- Save projects (`save`) and reopen (`open("")`) to avoid re-paying import cost. +- Joern is the *escalation* tier. If `rg` + `androguard.json` already proves the chain, skip Joern and write the finding. +- Capture useful query output to a file with `>` redirection inside Joern (`cpg.method.l #> "out.txt"`), then attach the file path to the finding's `evidence` array. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/leaked-host-triage.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/leaked-host-triage.md new file mode 100644 index 0000000..4aace94 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/leaked-host-triage.md @@ -0,0 +1,251 @@ +# Leaked-host triage + +Grounding: +- OWASP MASVS-CODE + MASVS-NETWORK: https://mas.owasp.org/MASVS/08-MASVS-CODE/ , https://mas.owasp.org/MASVS/04-MASVS-NETWORK/ +- MASTG-TECH-0025 Automated Static Analysis (use these heuristics alongside scanner output, not in place of it): https://mas.owasp.org/MASTG/techniques/android/MASTG-TECH-0025/ +- CWE-1188 Insecure Default Initialization of Resource: https://cwe.mitre.org/data/definitions/1188.html +- CWE-489 Active Debug Code (build-flag sub-pattern A1/A4): https://cwe.mitre.org/data/definitions/489.html +- CWE-540 Inclusion of Sensitive Information in Source Code: https://cwe.mitre.org/data/definitions/540.html + +Use this reference when a production APK ships a string literal in DEX that names a non-production hostname — QA, staging, sandbox, dev, preprod, internal, dogfood, canary, test, integration. The question this reference answers is **"can a third party reach a code path that swaps the production host for the leaked one, or is the string dead?"**. + +Empirically grounded on an internal corpus pass over 116 popular Google Play apps, 26 of which shipped at least one non-prod hostname in DEX (static analysis only; see `../../../references/sources.md` for provenance). Of the 14 P1+P2 packages triaged in depth: + +- **1 of 14 (~7%)** had a real reachable gate (Chase Mobile, server-flippable Split.io feature flag). +- **3 of 14 (~21%)** had a non-runtime build-pin gate (eBay Dagger, FitBit bootstrap initializer, ESPN build-flavor string compare). Production-safe in the released artifact. +- **10 of 14 (~71%)** had no reachable consumer at all — R8 retained the string inside a Kotlin top-level object with `@Metadata` but no DEX caller. Dead string. + +**Take the regex hit as a weak prior, not a strong one.** Most leaked-host strings in popular apps are R8/minification leftovers. Walk the consumer chain before treating the hit as a candidate finding. + +## When to use this reference + +- Inventory scan or `attack_surface.jsonl` flagged one or more non-prod host literals in a production APK. +- A focused `rg` against decompiled sources found the host in a base-URL constant, a setter argument, or a URL-construction string. +- You have time-bounded budget (target: ~5–15 minutes per package) and want a comparable outcome record across many packages. + +If the user is asking about a single APK they care about deeply, also load `android-targeted-assessment`; this reference is the methodology, the skill is the workflow shell. + +## Outcome schema + +Every package processed must resolve to exactly one of these: + +| outcome | meaning | grade | +|---|---|---| +| `build_config_gated` | `BuildConfig.DEBUG`, build-flavor string compare, Dagger compile-time module binding, or bootstrap initializer overrides the gate every cold start | production-safe; record and move on | +| `feature_flag_gated` | server-flippable feature flag controls the branch (ECS, LaunchDarkly, Split.io, FirebaseRemoteConfig, Statsig, Optimizely, custom config service) | **latent** — same shape as the Chase Mobile Split.io CDN-swap and Outlook FIC token observations (internal corpus research); promote to `confidence_tier=strong_static_chain`, `validation_tier=tier2_test_account_or_qa_backend` (because validation requires the vendor's flag console / authorized account) | +| `intent_extra_or_query_param_gated` | any local app can flip the gate via intent extra, deep-link query parameter, or a SharedPreferences write reachable from an exported settings activity | **real** — same shape as the Microsoft Teams `enableCanaryEndpointForMTService` canary-endpoint observation (internal corpus research); promote to `confidence_tier=strong_static_chain`, `validation_tier=tier1_local_device_no_live_backend` | +| `unreachable` | no production code path reaches the string; dead constant, third-party SDK config, OAuth scope literal, host-detection regex, or R8 leftover | record as `hardening_only` or skip | + +Only `feature_flag_gated` and `intent_extra_or_query_param_gated` are finding-shaped. The other two are negative-result closures. + +## Gate categories + +### A1 — `BuildConfig.DEBUG` / `isDebug()` + +Classic build-flag gate, runtime read but resolves to a build-time constant after R8. + +```bash +rg -n 'BuildConfig\.DEBUG|BuildConfig\.FLAVOR|isDebug\(\)|isDevDebug\(\)|isDev\(\)' "$SRC" +``` + +Production-safe. Record as `build_config_gated` / `outcome=production_safe`. + +### A2 — Feature flag (server-flippable) + +The gate reads a string/boolean/int from a feature-flag client. **This is the high-value finding shape this runbook is tuned for.** + +```bash +# ECS / Firebase / LD / Split.io / Optimizely / Statsig / custom-named +rg -n 'getEcsSetting|LDClient\.|boolVariation|stringVariation|FirebaseRemoteConfig|Statsig\.checkGate|Statsig\.getConfig' "$SRC" +rg -n 'io\.split\.android|mClient\.getTreatment|split\.getTreatment|Split\.' "$SRC" +rg -n 'OptimizelyClient|optimizelyManager|featureToggle\.a\(|featureToggles?\.' "$SRC" +rg -n 'configurationManager\.|configClient\.|getRemoteString\|getRemoteBoolean' "$SRC" +``` + +When you find the gate, name the flag key, the consumer, and the hosts reachable on each branch. Promote to a hypothesis matching the JSON template at the end of this file. + +**Validation tier:** `tier2_test_account_or_qa_backend` — confirming exploitability requires inspecting the vendor's flag-targeting console on an authorized account, not a local-device probe. + +### A3 — Intent extra / query param / exported pref-writer + +The gate reads `intent.getStringExtra(...)`, `getQueryParameter(...)`, or a SharedPreferences value that an exported settings-debug activity writes. + +```bash +rg -n 'getStringExtra\(|getBooleanExtra\(|getQueryParameter\(|intent\.getData\(\)\.getQueryParameter' "$SRC" +# Check whether SharedPreferences writes for the env key are reachable from exported components +rg -n '"environment"|"env"|"baseUrl"|"apiHost"|"server"' "$SRC" +``` + +Cross-reference any SharedPreferences writer activity against the manifest (`aapt2 dump xmltree --file AndroidManifest.xml `) — if it's `exported=true` or declares an intent-filter, the gate is third-party-reachable. + +**Validation tier:** `tier1_local_device_no_live_backend` — a helper app on the same device or an `adb shell am start` reproduces the chain. + +### A4 — Build-pin patterns + +Structurally distinct from A1 — these resolve at *compile* or *startup* time, not at every runtime read. Stronger guarantee than `BuildConfig.DEBUG` because there is no runtime read to flip. Documented from internal corpus research (provenance per `../../../references/sources.md`). + +| Sub-pattern | Detection | Illustrative example (corpus observation) | +|---|---|---| +| **Dagger compile-time pin** | `dagger.internal.Factory` whose `bindXxx` / `provideXxx` method returns a hard-wired enum/value | eBay: `EnvironmentRepositoryModule_BindEnvironmentRepositoryFactory.bindEnvironmentRepository() { return new FixedEnvironmentRepository(QaMode.PRODUCTION); }` | +| **Bootstrap-initializer override** | `EagerInitializer` / `Application.onCreate` / app-startup `Initializer` calls `setConfig(compileTimeValue)` with a flag that explicitly ignores persisted user overrides | FitBit: `HttpConfigInitializer.a(context)` calls `serverEnvironmentInitializerO.initializeEnvironmentConfig(wmoVarY, /*z=*/false)`. With `z=false`, `setServerConfig(compileTimeEnvironment)` runs every cold start, overwriting any prior write. | +| **Build-flavor string compare** | `if ("release" == "bet")` or similar — R8 replaces the literal per flavor; the production build evaluates to `false` | ESPN: `if ("release" == "bet") { ... setQa(true) ... }` — only the `bet` flavor sets the QA SharedPreferences key | + +```bash +# Dagger Factory binding a compile-time constant +rg -nE 'public static [A-Za-z<>]+ bind[A-Za-z]+\(\) \{\s+return new [A-Za-z]+\([A-Z_a-z\.]+\.[A-Z_]+\);' "$SRC" + +# Bootstrap initializer that ignores persisted override +rg -n 'EagerInitializer|androidx\.startup\.Initializer|HttpConfigInitializer|EnvironmentInitializer' "$SRC" + +# Build-flavor literal compare (post-R8 leftover) +rg -nE '"(release|debug|prod|qa|dev|staging|preprod)"\s*==\s*"' "$SRC" +rg -nE 'kotlin\.jvm\.internal\.Intrinsics\.areEqual\("[a-z]+"\s*,\s*"[a-z]+"' "$SRC" +``` + +Production-safe in the released artifact. Record as `build_config_gated` with `gate_evidence` naming the specific sub-pattern (A4-dagger, A4-bootstrap, A4-flavor) so the next pass can spot the same shape faster. + +### A0 — Unreachable (the default outcome) + +R8 + Kotlin Metadata retains the string but no DEX consumer reaches it. Common shapes: + +- **Public Kotlin top-level object** with `@Metadata(k=1|2, ...)` containing the string. The Metadata annotation is what keeps the object alive; the field itself has zero readers. Empirically dominant in corpus runs across airline, streaming, productivity, news, and creative-tool apps. +- **Allowlist / regex** that *mentions* the host as a string literal in a `Pattern.compile` or `contains` check. The literal is consumed by a regex compiler / `String.contains`, not by an HTTP client. Slack's `(slack|slack-gov|slack-gov-dev|slack-mil-dev)` workspace-URL detector is the canonical example. +- **OAuth scope literal** that ends in `.ReadWrite` / `.Read` / `.default` / `-Internal.ReadWrite`. The "internal" suffix names a token scope, not a host modifier. Outlook's `https://substrate.office.com/User-Internal.ReadWrite` is a scope URI, not an endpoint. +- **JSON test fixture** containing the host in a documentation/sample payload string — e.g. a `TestJson.kt` that ships a stage CDN URL inside an HTML article fixture used only by an in-app dev-menu preview path. +- **Sample / fixture / seed data** for an unused feature. + +To confirm a string is unreachable: + +```bash +# Find the file that contains the literal +rg -nl "$HOST" "$SRC" + +# Identify the enclosing class / object +# Walk back: does any other class import or reference this one? +grep -rn 'import .*' "$SRC" | head +rg -n '\b' "$SRC" | head + +# If zero consumers, the string is unreachable. +``` + +Record as `unreachable` / `outcome=production_safe`. Do not promote. + +## Per-package runbook (~5–15 minutes each) + +For each package on a workqueue, repeat this loop. Strict time-boxing matters — the negative-result rate is high, so spend most of the per-package budget *deciding to close*, not *deep-reading*. + +### Step 1 — decompile (cheap path) + +```bash +SRC=findings//decompiled//sources +if [ ! -d "$SRC" ]; then + APK=$(ls corpus/.../apks/*.apk | head -1) + # Heap tier by DEX class count — see android-semantic-vuln-hunting skill + JAVA_OPTS="-Xmx6g" jadx --show-bad-code --no-debug-info \ + -d "findings//decompiled/" "$APK" +fi +``` + +### Step 2 — locate consumers of the leaked host string + +```bash +TARGET='static2-qa1.chasecdn.com' # one host at a time +rg -nl "$TARGET" "$SRC" +``` + +Three shapes you will see: + +- **(A) String literal embedded in a class constant.** Walk back to find what reads the constant. Usually a `URLProvider` / `EnvironmentManager` / `ConfigManager` / `BackendEnvironment` enum that returns one of N URLs based on a `currentEnvironment` field. +- **(B) String passed into `setEndpoint(...)` / `setBaseUrl(...)` / `setHost(...)`.** Walk back to find the caller of the setter. The caller is the gate. +- **(C) String only appears in a logging statement, JSON fixture, comment, or `Pattern.compile`.** Dead branch from a removed feature or a regex consumer. Record as `unreachable`. + +### Step 3 — identify the gate + +For shape (A) — environment selector: + +```bash +# What populates the environment field? +rg -n 'currentEnvironment|envName|setEnvironment|getEnvironment|backendEnvironment|FitbitBackendEnvironment' "$SRC" +``` + +Classify the writer/source against the A1/A2/A3/A4 categories above. + +For shape (B) — setter call: + +```bash +rg -n 'setEndpoint\(|setBaseUrl\(|setHost\(|setServerConfig\(' "$SRC" +# For each call site, read 10 lines before to see what feeds the URL argument +``` + +Same A1/A2/A3/A4 classification on the caller. + +For shape (A) when you cannot find a writer: + +```bash +# Find any caller of the enclosing class/object +rg -n '\b' "$SRC" | grep -v 'kotlin\\.Metadata' | head +``` + +If the only references are inside the class itself or in `kotlin.Metadata` literals, the string is unreachable. + +### Step 4 — record outcome + +Append one line per package to `findings//leaked-host-workqueue.jsonl`: + +```json +{ + "package": "com.chase.sig.android", + "hosts": ["static2-qa1.chasecdn.com", "static2-qa2.chasecdn.com"], + "consumer_path": "fN/C5672.mo9740() — switch on Split.io flag; consumed by nu/C15513.tn0", + "gate_kind": "feature_flag_gated", + "gate_evidence": "InterfaceC36156.mo71972('mobl_AEMService_QAEnvironment', 'default') reads a Split.io string flag; q1/q2/q3/q4 return matching QA hosts.", + "outcome": "latent_flag", + "notes": "AEM CDN serves video + JSON content to the authenticated banking session." +} +``` + +Valid `gate_kind` values: `build_config_gated`, `feature_flag_gated`, `intent_extra_or_query_param_gated`, `unreachable`. + +Valid `outcome` values: + +- `production_safe` — `build_config_gated` (any A1/A4 sub-pattern) or `unreachable` +- `latent_flag` — `feature_flag_gated`; needs vendor-flag-console or authorized backend test to confirm exploitability +- `local_reachable` — `intent_extra_or_query_param_gated`; the finding shape worth promoting to a `tier1` hypothesis + +### Step 5 — promote (only if `latent_flag` or `local_reachable`) + +Append a hypothesis to `findings//hypotheses/hypotheses.jsonl` matching the schema in `output-schema.md`. The Chase Mobile feature-flag and Microsoft Teams intent-extra observations referenced above are template shapes — copy their structure (illustrated by the JSON sample further down) so `normalize_semantic_findings` produces a clean report. + +After the workqueue is processed, regenerate the report: + +```python +# Tool — emits deterministic Markdown. +normalize_semantic_findings( + inputs=["findings//hypotheses/hypotheses.jsonl"], + output_format="markdown", + out="findings//report.md", +) +``` + +## Biases to avoid + +Empirical, repeated across internal corpus passes: + +1. **Treating a regex hit as evidence of a real chain.** The leaked-host signal is ~7% strong-static-chain, ~21% build-pinned, ~71% dead. Cap per-package time accordingly — spend the *first* 2 minutes confirming the consumer exists at all. +2. **Stopping at A1.** "It's gated by `BuildConfig.DEBUG`" closes a chain; "it's gated by `BuildConfig.DEBUG` *and the writer SharedPref key is also exposed via an exported settings activity*" does not. Read the writer chain to the end before recording `build_config_gated`. +3. **Workqueue tunnel vision.** If a `local_reachable` or strong `latent_flag` finding lands in the first 3 packages, the temptation is to deep-read it before finishing the queue. Don't — the corpus-wide signal is the whole point. Finish the queue first; the strongest promoted hypothesis can be deep-read after. +4. **Cheap-path gravity (already-decompiled apps).** Apps you've already decompiled in a previous pass are tempting because the marginal cost is zero. Cap re-reads per pass; spend the freed time on a target class that hasn't been opened yet. +5. **Assuming the per-environment URLProvider is the only gate.** Some apps (eBay-class Dagger pin, FitBit-class bootstrap initializer) wire the environment value through Dagger / bootstrap-initializer paths that *override* any persisted/reachable value at startup. Always trace back to where the `currentEnvironment` field is *first written* in the process lifecycle. + +## Reference checklist + +- [ ] Pass uses inventory `attack_surface.jsonl` or equivalent regex over DEX strings to enumerate leaked hosts. +- [ ] Workqueue is finite, ranked, ~5–15 min per package, and explicitly *not* deep-read. +- [ ] Each package resolves to exactly one outcome. +- [ ] `build_config_gated` records distinguish A1 vs A4 sub-patterns. +- [ ] `feature_flag_gated` records name the flag key and the flag-service vendor. +- [ ] `intent_extra_or_query_param_gated` records name the intent extra / query param / SharedPref key and the exported component that writes it. +- [ ] `unreachable` records explain *why* the string is unreachable (R8-leftover, regex consumer, OAuth scope, JSON fixture). +- [ ] Hypotheses are appended only for the two promote-worthy outcomes. +- [ ] Final report regenerated via `normalize_semantic_findings`. +- [ ] Pass self-assessment records the outcome distribution and any new gate sub-pattern that appeared. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/output-schema.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/output-schema.md new file mode 100644 index 0000000..15290d8 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/output-schema.md @@ -0,0 +1,127 @@ +# Output schemas + +## Finding hypothesis JSONL + +Each line is one finding object. `class` is the canonical bug-class slug; the +normalizer (`normalize_semantic_findings` MCP tool) uses it to default +MASVS/CWE/MASWE tags when the record omits them. + +```json +{ + "title": "Deep link redirects authenticated WebView to attacker-controlled host", + "apk": "corpus/app.apk", + "package": "com.example", + "class": "deep_link_to_authenticated_webview", + "masvs": ["MASVS-PLATFORM", "MASVS-NETWORK"], + "cwe": ["CWE-939", "CWE-749"], + "maswe": ["MASWE-0058"], + "entrypoint": "com.example.DeepLinkActivity exported BROWSABLE myapp://open", + "source": "Intent.getData().getQueryParameter('next')", + "trust_boundary": "browser/other app controls next; app treats string prefix as first-party URL validation", + "sink": "WebView.loadUrl(next) with authenticated cookies", + "impact": "token theft or account takeover if auth context is sent to attacker-controlled URL", + "evidence": ["DeepLinkActivity.java:42", "WebRouter.java:88"], + "validation_plan": ["Use test device to launch myapp://open?next=https://trusted.com.attacker.tld/", "Observe WebView request host and cookies with authorized proxy"], + "scanner_gap": "generic warning only", + "confidence_tier": "needs_route_map_validation", + "validation_tier": "tier1_local_device_no_live_backend", + "missing_evidence": ["Confirm host-allowlist contents in the JS bundle (Step 7.5)"] +} +``` + +The `confidence_tier` enum is one of `confirmed_dynamic`, `strong_static_chain`, +`needs_backend_validation`, `needs_route_map_validation`, `hardening_only`, or +`generic_library_noise`. `validation_tier` is one of `tier0_static_only`, +`tier1_local_device_no_live_backend`, `tier2_test_account_or_qa_backend`, or +`tier3_explicit_production_authorization`. See the "Finding hypothesis contract" +section of `../SKILL.md` for the canonical contract; the normalizer also accepts +the legacy `risk` + `confidence` (`high`/`medium`/`low`) fields for back-compat, +but new hypotheses should use the tier enums. + +### Tag fields + +- **`masvs`** — OWASP MASVS control category (`MASVS-PLATFORM`, + `MASVS-NETWORK`, `MASVS-AUTH`, `MASVS-STORAGE`, `MASVS-CRYPTO`, + `MASVS-CODE`, `MASVS-RESILIENCE`, `MASVS-PRIVACY`). + Reference: . +- **`cwe`** — Common Weakness Enumeration IDs (e.g. `CWE-749`, `CWE-926`). + Reference: . +- **`maswe`** — OWASP Mobile Application Security Weakness Enumeration IDs. + MAS's weakness catalog tied to MASTG tests; currently in beta. + Reference: . **Where no MASWE cleanly maps + the bug class** (Dirty Stream, client-side trust, backend API abuse, + request-signing replay, leaked-host feature-flag gates) the field stays + empty rather than asserting an unrelated ID; CWE + MASVS carry the + grounding instead. + +All three are arrays, all three are optional in the source record — the +normalizer fills defaults from the `class` value if the record omits them. + +## Bug-class taxonomy + +Use these stable `class` slugs so corpus-level reports can compare patterns +across runs. Each class maps to a default MASVS / CWE / MASWE tag tuple in +`scripts/normalize_findings.py:CLASS_TAXONOMY` — when adding a new class, +update both this table and the dict in the same change. + +MASWE anchors used below: +- **MASWE-0058** — Insecure Deep Links (MASVS-PLATFORM) +- **MASWE-0064** — Insecure Content Providers (MASVS-PLATFORM) +- **MASWE-0066** — Insecure Intents (MASVS-PLATFORM) +- **MASWE-0068** — JavaScript Bridges in WebViews (MASVS-PLATFORM) + +| `class` | MASVS | CWE | MASWE | +|---|---|---|---| +| `deep_link_to_authenticated_webview` | PLATFORM, NETWORK | CWE-939, CWE-749 | MASWE-0058 | +| `deep_link_to_js_bridge` | PLATFORM | CWE-749, CWE-829 | MASWE-0058, 0068 | +| `custom_scheme_arbitrary_webview` | PLATFORM | CWE-939, CWE-079 | MASWE-0058 | +| `intent_redirection_private_component` | PLATFORM | CWE-926, CWE-940 | MASWE-0066 | +| `intent_redirection_uri_grant_leak` | PLATFORM | CWE-926, CWE-200 | MASWE-0066 | +| `dirty_stream_file_overwrite` | PLATFORM, STORAGE | CWE-22, CWE-73 | — | +| `share_target_path_traversal` | PLATFORM, STORAGE | CWE-22 | — | +| `exported_provider_sqli` | PLATFORM | CWE-89, CWE-926 | MASWE-0064 | +| `exported_provider_private_file_read` | PLATFORM, STORAGE | CWE-200, CWE-926 | MASWE-0064 | +| `provider_uri_grant_confusion` | PLATFORM | CWE-441, CWE-926 | MASWE-0064, 0066 | +| `deep_link_auto_account_state_change` | AUTH, PLATFORM | CWE-352, CWE-862 | MASWE-0058 | +| `client_state_auth_bypass` | AUTH | CWE-602, CWE-287 | — | +| `apk_discovered_backend_bola` | AUTH, NETWORK | CWE-639 | — *(OWASP API1)* | +| `apk_discovered_backend_workflow_bypass` | AUTH, NETWORK | CWE-841, CWE-863 | — *(OWASP API5)* | +| `apk_discovered_backend_mass_assignment` | NETWORK | CWE-915 | — *(OWASP API6)* | +| `apk_discovered_backend_ssrf_or_open_redirect` | NETWORK | CWE-918, CWE-601 | — *(OWASP API7)* | +| `apk_discovered_graphql_operation_abuse` | NETWORK | CWE-639, CWE-863 | — *(OWASP API1/API3)* | +| `apk_discovered_grpc_operation_abuse` | NETWORK | CWE-639, CWE-863 | — *(OWASP API1/API3)* | +| `webview_bridge_to_mobile_api_action` | PLATFORM | CWE-749, CWE-829 | MASWE-0068 | +| `mobile_request_signing_replay_or_confusion` | NETWORK, CRYPTO | CWE-345, CWE-294 | — | +| `leaked_host_feature_flag_gated` | NETWORK, CODE | CWE-1188 | — | +| `leaked_host_intent_extra_gated` | NETWORK, PLATFORM | CWE-1188, CWE-926 | MASWE-0066 | + +Tags are starting points, not contracts — override per-finding when the +specific chain warrants different categorization. The OWASP API Top 10 +annotations on `apk_discovered_backend_*` rows are aide-memoires for the +backend-side framework; they aren't yet emitted by the normalizer. + +## Corpus manifest JSONL + +```json +{ + "package": "com.example", + "version": "1.2.3", + "source": "androzoo|device|fdroid|user|mirror", + "sha256": "...", + "path": "corpus/com.example/base.apk", + "downloaded_at": "2026-05-15T16:00:00Z", + "provenance_url": "https://...", + "authorization_notes": "..." +} +``` + +## Report sections + +A good Markdown report has: + +1. Scope and corpus provenance. +2. Method summary and tool versions. +3. Top findings by impact/confidence. +4. Scanner-gap summary. +5. Validation queue grouped by required authorization tier. +6. Appendix with all normalized hypotheses. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/request-signing-and-attestation.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/request-signing-and-attestation.md new file mode 100644 index 0000000..9713c67 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/request-signing-and-attestation.md @@ -0,0 +1,65 @@ +# Request signing and attestation review + +Use when an APK contains custom API signing, encrypted bodies, device binding, Play Integrity/SafetyNet, certificate pinning, or backend endpoints that trust mobile-only headers. The goal is not to steal secrets; it is to understand what the backend thinks the mobile client proves. + +## Search terms + +```bash +rg -n \ + -e 'Hmac|Mac\.getInstance|SHA256|SHA-256|MessageDigest|Signature|getInstance\("Hmac' \ + -e 'signRequest|signature|x-signature|X-Signature|canonical|canonicalize|stringToSign' \ + -e 'nonce|timestamp|expires|ttl|replay|clockSkew|serverTime' \ + -e 'deviceId|installationId|androidId|Settings\.Secure|Build\.SERIAL|fingerprint' \ + -e 'SafetyNet|PlayIntegrity|IntegrityManager|IntegrityTokenRequest|attestation|attest' \ + -e 'CertificatePinner|TrustManager|HostnameVerifier|pinning|pinset' \ + -e 'encryptBody|encryptedPayload|Cipher\.getInstance|AES/GCM|RSA/ECB|publicKey' \ + "$SRC" > findings//rg-signing-attestation.txt +``` + +## Questions to answer + +### Signature coverage + +- Is the HTTP method signed? +- Is the path signed? +- Are query parameters signed after canonical sorting? +- Is the whole body signed, or only selected fields? +- Are object IDs (`accountId`, `tenantId`, `deviceId`, `orderId`) included? +- Are auth headers and tenant headers signed? +- Are file uploads/multipart parts signed consistently? + +### Replay controls + +- Is there a nonce? +- Is the timestamp server-validated? +- Does the server issue the nonce or does the client generate it? +- Is the nonce bound to account/device/session/path/body? +- What happens on retry/offline sync? + +### Device and attestation binding + +- Is Play Integrity/SafetyNet required or best-effort? +- Is there fallback for devices without Play Services? +- Are attestation failures logged but allowed? +- Is device ID client generated or server enrolled? +- Can a signed request be replayed from another device/account? + +### Pinning / transport + +- Is certificate pinning present only in release builds? +- Are debug/staging hosts unpinned? +- Do WebView/custom tabs share cookies or tokens with API clients? + +## High-value bug shapes + +- Signature excludes privilege-bearing fields (`tenantId`, `role`, `price`, `status`, `deviceId`). +- Signature is computed before later mutation of request body/map. +- Retry/offline queue reuses old signatures after object/account state changes. +- Nonce/timestamp is client-only and not server-enforced. +- Attestation fallback allows sensitive endpoints with only device ID. +- WebView/JS bridge can call native signing helper for attacker-controlled path/body. +- Mixed signed and unsigned endpoints share the same authorization token. + +## Evidence standard + +Static code can prove signing design and potential coverage gaps. It cannot prove backend acceptance. Use `mobile_request_signing_replay_or_confusion` with `needs_backend_validation` until authorized backend tests show replay, cross-device use, or field tampering works. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/scanners.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/scanners.md new file mode 100644 index 0000000..fb79597 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/scanners.md @@ -0,0 +1,154 @@ +# Android scanner baseline recipes + +Use these recipes after writing semantic hypotheses. Store outputs under `findings/baselines//`. The point is to label each hypothesis with `scanner_gap` (`exact` / `adjacent` / `generic` / `not found`). + +## Semgrep with the Android packs + +Public, free, fast, and explainable. The first scanner to run. + +```bash +PKG= +SRC=findings/decompiled/$PKG/sources + +semgrep --config p/security-audit \ + --config p/mobsfscan \ + --config r/java \ + --metrics=off \ + --json --output findings/baselines/$PKG/semgrep.json "$SRC" +``` + +The `p/mobsfscan` registry pack ships MobSF's rule set as Semgrep rules and covers Android-specific Java/Kotlin/XML checks (hardcoded secrets, weak crypto, exported components, WebView misuse). `p/security-audit` adds language-agnostic security audit rules; `r/java` adds general Java correctness/security rules. Smoke-tested on the Stone JADX tree (89 results, dominated by `mobsf.mobsfscan.android.*` rules). + +Useful flags: + +- `--severity=ERROR` to suppress informational rules during baselining +- `--include='*.java'` if Kotlin decompilation is noisy +- `--metrics=off` for air-gapped runs + +Interpretation: + +- Strong on explainable pattern findings (`addJavascriptInterface`, `setJavaScriptEnabled(true)`, exported components without permissions). +- Weak on app-specific routing semantics and multi-hop chains. +- Each Semgrep hit is a *lead*. Map it to the closest hypothesis or note it as a generic finding for the appendix. + +## MobSF (full static + dynamic platform) + +Use the REST API when a MobSF server is reachable. + +```bash +HASH=$(curl -sS -F "file=@app.apk" "$MOBSF_URL/api/v1/upload" \ + -H "Authorization:$MOBSF_API_KEY" | jq -r .hash) + +curl -sS -X POST "$MOBSF_URL/api/v1/scan" \ + -H "Authorization:$MOBSF_API_KEY" \ + -d "scan_type=apk&hash=$HASH" + +curl -sS "$MOBSF_URL/api/v1/report_json" \ + -H "Authorization:$MOBSF_API_KEY" \ + -d "hash=$HASH" \ + -o findings/baselines//mobsf.json +``` + +Interpretation: + +- Good broad MASVS-aligned baseline (network security config, certificate pinning, manifest flags, dangerous APIs). +- Output is noisy and compliance-flavoured — use as scanner_gap labelling, not as the report. + +## mobsfscan (CLI subset of MobSF rules) + +When you don't have a MobSF server. + +```bash +mobsfscan --json findings/decompiled//sources \ + --output findings/baselines//mobsfscan.json +``` + +Same rule provenance as MobSF static analysis; quicker to run, narrower coverage. Good for offline runs. + +## APKHunt + +When APKHunt and Go are installed. + +```bash +go run apkhunt.go -p app.apk -l > findings/baselines//apkhunt.txt +# Bulk: +# go run apkhunt.go -m corpus/apks -l > findings/baselines/apkhunt-corpus.txt +``` + +Interpretation: + +- MASVS-oriented static checks. +- Less structured output; preserve raw logs and summarize manually. + +## Focused ripgrep baseline + +Not a vulnerability detector. Useful to prove that semantic slices didn't miss obvious neighbourhoods. + +```bash +rg -n -e 'getParcelableExtra' -e 'Intent\.parseUri' -e 'startActivity\(' \ + -e 'loadUrl\(' -e 'addJavascriptInterface' \ + -e 'setJavaScriptEnabled\s*\(\s*true' \ + -e 'SharedPreferences' -e 'Authorization' -e 'Bearer' \ + findings/decompiled//sources \ + > findings/baselines//high-value-grep.txt +``` + +## Scanner-gap labels + +For each finding hypothesis, set `scanner_gap`: + +- `exact baseline finding` — scanner reported same entrypoint/source/sink/impact class +- `adjacent baseline finding` — scanner saw a related risky API/component but not the chain +- `generic warning only` — scanner reported a broad category without an actionable chain +- `not found` — no relevant scanner output + +For this capability, `adjacent` and `generic` are usually where the user's value lives. + +## Known coverage gaps (auto-label `scanner_gap=not found`) + +Bug-class shapes that the baseline scanners (`p/security-audit`, `p/mobsfscan`, `r/java`, MobSF, mobsfscan, APKHunt) do not currently cover. When a hypothesis matches one of these shapes, attach `scanner_gap=not found` without re-running the scanner — the absence is structural. + +Examples in the right column are internal corpus observations (static-analysis only; see `../../../references/sources.md` for provenance framing). They illustrate the shape; they are not vendor-confirmed advisories. + +| Bug class | Why scanners miss it | Illustrative example (corpus observation) | +|---|---|---| +| **Non-prod host swap via server-flippable feature flag** — production binary reads an ECS / LaunchDarkly / Split.io / FirebaseRemoteConfig / Optimizely / Statsig string and swaps a base URL to a QA/stage/dev host | No rule looks for `(feature-flag client call) → (string-switch with multiple URL constants) → (host substitution)`. The flag client itself is third-party SDK code, so noise rules suppress it. | Chase Mobile `mobl_AEMService_QAEnvironment` (Split.io), Outlook FIC token, Match P2P SDK (Mf.a featureToggle gating in-app toggle visibility). | +| **Non-prod host swap via intent extra without `BuildConfig.DEBUG`** — exported BROWSABLE activity reads a `boolean`/`String` extra and reaches a `setEndpoint` / `EndpointManager` sink | Semgrep rules for `addJavascriptInterface` and `intent.getData()` exist but none chain `getBooleanExtra → setEndpoint`. | Teams `enableCanaryEndpointForMTService`. | +| **Deep link to account-state-change (no WebView)** — `LaunchedEffect` on the destination Composable calls a server-side accept/complete/register API with deep-link args as inputs | All deep-link rules assume a WebView sink. | Dashlane `mplesslogin?id=&key=` (D_secret class). | +| **A4 build-pin patterns** — Dagger Factory binding a compile-time enum, bootstrap initializer overriding persisted env state, build-flavor literal compare | Scanners do not introspect dagger-generated factories or `androidx.startup.Initializer` startup graphs. | eBay `EnvironmentRepositoryModule_BindEnvironmentRepositoryFactory`, FitBit `HttpConfigInitializer`, ESPN `"release" == "bet"`. | +| **JS-bridge-to-native-API action** (RN / Capacitor / Cordova) — Java bridge wired but the auth-context-bearing argument flow is inside the JS/Dart bundle | Scanners do not read Hermes bytecode or Dart AOT. The Java side looks harmless. | Multiple C_wallet wallets observed in corpus runs. | +| **Request-signing or attestation bypass / fallback** — multi-tier signing canonicalization, HMAC-replay-safe-flag, attestation `IGNORE_ON_FAILURE` paths | Scanners look for hardcoded keys, not for fallback paths in signer code. | (no public empirical yet; pattern documented in `request-signing-and-attestation.md`). | + +Pre-labelling these gaps speeds up reporting on subsequent passes — once a shape is in this table, future hypotheses of the same shape attach `scanner_gap=not found` automatically. + +## Known-noise rules (auto-label `generic_library_noise`) + +These rules fire on third-party SDK code that is the **expected design pattern**, not a finding. Pre-classify hits on these `(rule_id, file path glob)` pairs as `generic_library_noise` so they stop polluting per-class reports. + +| Rule | File glob | Why it's noise | +|---|---|---| +| `mobsf.mobsfscan.webview.webview_javascript_interface` | `**/com/reactnativecommunity/webview/*.java` | RN `RNCWebView` registers `ReactNativeWebView` as its bridge by design; finding is the RN architecture | +| `mobsf.mobsfscan.webview.webview_debugging` | `**/com/reactnativecommunity/webview/*.java` | Debug-only call guarded by a build flag (`W6.a.*`); production builds are off | +| `mobsf.mobsfscan.android.secrets.hardcoded_api_key` | `**/nativeModules/NotificationModule.java`, `**/com/google/firebase/messaging/*.java` | FCM message-key constants (`google.message_id`, `gcm.notification.title`) trip the regex | +| `mobsf.mobsfscan.android.secrets.hardcoded_username/password` | `**/com/braze/**`, `**/com/intercom/**` | Hardcoded SDK field names match the secret-name regex | +| `java.lang.security.audit.unsafe-reflection.unsafe-reflection` | `**/kotlin/reflect/**`, `**/com/facebook/react/**` | Kotlin/RN runtime reflection is fundamental to the framework | +| `mobsf.mobsfscan.android.secrets.hardcoded_*` | `**/Dagger*HiltComponents*.java`, `**/*_Factory.java`, `**/*_MembersInjector.java`, `**/*_GeneratedInjector.java`, `**/R.java` | DI-generated companion classes and resource constants have field names like `username`, `password`, `apiKey` that match the secret-name regex but are never values | + +A small jq filter keeps the report clean: + +```bash +jq '.results | map(select( + (.check_id == "mobsf.mobsfscan.webview.webview_javascript_interface" and (.path | test("com/reactnativecommunity/webview"))) or + (.check_id == "mobsf.mobsfscan.android.secrets.hardcoded_api_key" and (.path | test("NotificationModule|firebase/messaging"))) or + (.check_id == "mobsf.mobsfscan.webview.webview_debugging" and (.path | test("com/reactnativecommunity/webview"))) or + ((.check_id | test("mobsf.mobsfscan.android.secrets.hardcoded_")) and (.path | test("Dagger.*HiltComponents|_Factory\\.java|_MembersInjector\\.java|_GeneratedInjector\\.java|/R\\.java$|/generated/"))) + | not +))' findings/baselines//semgrep.json > findings/baselines//semgrep-denoised.json +``` + +Empirical: + +- C_wallet pass (MetaMask, RN-Hermes): Semgrep returned 4 findings; 3 fell into the RN-noise rows. +- D_secret pass (Dashlane / Proton Pass / LastPass, all native Kotlin): Semgrep returned 365 / 1,022 / 111 findings. The `hardcoded_username|password|api_key` rules alone produced 1,200+ hits across the three apps, almost all on Dagger/Hilt generated classes and Kotlin field metadata — the new Dagger/_Factory row catches the majority. The real-signal rules in `D_secret` are `webview_javascript_interface` (LastPass account-recovery + share, Dashlane NitroSso), `xmlinputfactory_xxe` (LastPass serialization), and `webview_debugging` (LastPass — needs release-flag check). + +Extend the table as new noise patterns appear; the rule is that a finding belongs here only if it fires on **library code shipped by every app in the class** (or on framework-generated code) and the finding is **expected behaviour** for that library / generator. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/workflow-state-machines.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/workflow-state-machines.md new file mode 100644 index 0000000..edc7ffe --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/workflow-state-machines.md @@ -0,0 +1,70 @@ +# Workflow state-machine reconstruction + +Use when an APK reveals multi-step backend actions. The goal is to reconstruct the intended order, object ownership checks, and confirmation gates before planning backend validation. + +## Good candidate workflows + +- account recovery / password reset / magic link consume +- device registration / trusted device / device migration +- family/child/guardian invite and location sharing +- team/org/tenant invite and role assignment +- KYC / identity verification / document upload +- checkout / payment / refund / wallet transfer +- coupon / points / gift-card / subscription activation +- booking / ticket / trip modification +- password vault item sharing / emergency access +- IoT device bind/share/transfer-owner/command +- moderation/report/block/unblock/private media + +## Extraction steps + +1. Identify endpoint family and request DTOs from `api_map.jsonl`. +2. Find ViewModel/Presenter/UseCase/Repository methods that call the endpoints. +3. Order calls by UI state, coroutine chain, Rx chain, callback chain, or navigation route. +4. Record confirmation gates: unlock, OTP, biometric, KYC, user click, modal text, server challenge. +5. Record client-controlled object IDs and role/status fields at each step. +6. Record server-provided tokens/challenges and whether later steps bind to them. +7. Identify side-effecting endpoints that appear callable from code without earlier steps. + +## Grep + +```bash +rg -n \ + -e 'start|initiate|request|verify|confirm|approve|accept|complete|finalize|activate|claim|redeem|recover|reset|bind|pair|transfer|cancel|refund' \ + -e 'viewModelScope\.launch|lifecycleScope\.launch|LaunchedEffect|flatMap|switchMap|andThen|enqueue|suspend fun' \ + -e 'Otp|OTP|mfa|MFA|biometric|BiometricPrompt|challenge|nonce|token|verificationCode' \ + -e 'userId|accountId|tenantId|orgId|familyId|deviceId|inviteId|paymentId|orderId|subscriptionId' \ + -e 'status|state|role|verified|approved|completed|pending|expired' \ + "$SRC" > findings//rg-workflows.txt +``` + +## Output template + +```text +workflow: +entrypoints: + - deep link / screen / notification / API method +steps: + 1. -> fields=[...] + 2. -> fields=[...] + 3. -> fields=[...] +client_controlled_fields: + - ... +server_tokens_or_challenges: + - ... +confirmation_gates: + - ... +possible_bypass: + - complete endpoint appears callable with client-supplied deviceId and recoveryToken +impact_if_backend_accepts: + - ... +validation_plan: + - tier2/tier3 steps, starting with non-destructive negative checks +``` + +## Common mistakes + +- Do not equate a callable API client method with exploitability. Many methods are protected server-side. +- Do not ignore confirmation UI. A user click/modal can be a meaningful gate. +- Do not ignore server-issued challenge binding. If later steps require a server challenge bound to account/device, the hypothesis may be hardening-only. +- Do not probe destructive production flows without explicit scope. diff --git a/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/workflow.md b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/workflow.md new file mode 100644 index 0000000..3336e85 --- /dev/null +++ b/capabilities/android-apk-research/skills/android-semantic-vuln-hunting/references/workflow.md @@ -0,0 +1,118 @@ +# Workflow recipes + +This reference defines the corpus-scale and targeted workflows, validation tiers, and the tool-vs-CLI boundary. + +## Discoverability first + +Agents should not rely on memory for available APK-research utilities. Start with `agent-utility-index.md` when orienting, then choose the smallest applicable flow: + +```bash +cat agent-utility-index.md +``` + +This index lists corpus inventory/ranking, runtime detection, focused rg profiles, backend API map extraction, backend richness ranking, feature-flag mining, request-signing review, workflow reconstruction, hybrid bridge tracing, scanner baselines, and report normalization. + +## Corpus triage (1000s of APKs) + +```text +run_corpus_inventory (parallel, resumable) + → extract_components + rank_components for the component inbox + → jadx -d only for the top batch + → rg the focused source/sink set on first-party code + → semgrep p/android baseline + scanner_gap labelling + → joern for call/data context on the handful of methods that matter + → optional codeql / FlowDroid escalation + → finding JSONL → normalize_semantic_findings → report +``` + +Almost every APK in a corpus stops after `rank_components` or `rg`. JADX, Semgrep, Joern, CodeQL, and FlowDroid each cost real time; only escalate when the chain warrants it. + +## Targeted assessment (one APK) + +```text +run_corpus_inventory on the single APK + → read androguard.json + apkid.json (no decompilation yet) + → detect_protector (if APKiD flagged anything) → optional dexprotector_unpack + → detect_runtime_kind → JADX heap budget + JS/Dart follow-up routing + → jadx -d + → rg over first-party code + → semgrep p/android baseline + → joern for call/data context where needed + → optional codeql when path-precise evidence matters + → hypothesis JSONL → normalize_semantic_findings → report + → authorized adb validation +``` + +The pipeline is the same; the difference is depth on entrypoints, routers, and bridges. + +## Validation tiers + +Honest tiering matters more than confident-sounding writeups. + +- **`tier0_static_only`** — code/manifest evidence only. Default tier. +- **`tier1_local_device_no_live_backend`** — ADB launch, emulator, logcat capture, no backend mutation. Safe when the user has a test device and tells you so. +- **`tier2_test_account_or_qa_backend`** — test account or QA backend interaction in scope. +- **`tier3_explicit_production_authorization`** — only with written authorization; produce a minimum-impact proof and stop. + +Default to tier 0/1 unless the user has explicitly authorised higher tiers. + +## Tool vs CLI boundary + +The orchestration layer is wrapped in the `android-research` MCP. The methodology layer (decompile, search, scan, query, taint) stays in bash + canonical CLIs because the heap-tier table, rule-pack ensemble, query recipe, and pattern selection *are* the value — wrapping them as MCP tools would constrain the agent more than it enables. + +| MCP tool | Why a tool, not bash | +|---|---| +| `inventory_status` | One-shot environment probe: uv, apkid, aapt2, jadx, semgrep, joern, codeql, adb, hbctool, blutter. Cheap to call once at session start. | +| `run_corpus_inventory` | Process pool, per-APK timeouts, SHA256-keyed artifact layout, resume, Androguard + APKiD orchestration. Bash equivalents drop on the floor at scale. | +| `extract_components` | Streams every `androguard.json` under an inventory dir and falls back to `aapt2 dump xmltree` for the multi-dex APKs that break Androguard 4.1.3. Output is one JSONL row per `(apk, type, name)` — the input to `rank_components`. | +| `rank_components` | Applies the risk-prior table and emits the component inbox. Determinism + the read-budget tag matters for time-boxed corpus passes. | +| `detect_runtime_kind` | Returns a fixed enum (`native`, `react_native_hermes`, `flutter_aot`, …) driving JADX heap and Step 7.5 / 7.6 routing. One-second probe; perfect MCP surface. | +| `detect_protector` | Decides whether to load `android-protector-triage` and whether `dexprotector_unpack` will succeed. Pure-Python signal extraction; no Android device required. | +| `dexprotector_unpack` | Unicorn-backed static unpack of libdp.so for DexProtector-protected APKs (arm64-v8a). Heavy enough to deserve its own typed surface; conditional on `detect_protector`. | +| `extract_api_map` | Regex-mines API endpoints, generated clients, request-signing hints, feature flags, object IDs, and workflow verbs. Drives backend-rich APK triage. | +| `rank_backend_richness` | Sorts `backend_richness.json` summaries across a corpus. The next-targets-to-probe queue for backend hypotheses. | +| `normalize_semantic_findings` | Deterministic finding schema with MASVS/CWE/MASWE auto-tagging, dedup key, confidence/validation tier inference, Markdown/CSV/JSONL renderers. Stable contract across runs. | + +Everything else — JADX decompile, ripgrep, Semgrep, Joern, CodeQL, FlowDroid, adb — uses the canonical CLI directly. Read the matching skill for the recipes. + +### Why bash, not MCP — for JADX, Semgrep, Joern + +The "use bash" choice is not the absence of a decision; it's a deliberate one. The audit, with evidence: + +- **`semgrep mcp`** (semgrep 1.23.3 builtin; standalone `semgrep/mcp` repo archived 2025-10-28). The `semgrep_scan` MCP tool accepts only `code_files: [{path}]` — no `config` argument. It runs the default `auto` rule set. Our methodology *requires* the multi-pack ensemble (`--config p/security-audit --config p/mobsfscan --config r/java`); the MCP surface can't express it. Output also can't be redirected to `findings/baselines//semgrep.json` from the tool. Adopting it would regress the methodology to "scan with defaults." +- **`zinja-coder/jadx-mcp-server`** (632★) + **`jadx-ai-mcp` plugin** (2,171★). The README is explicit: *"Requires JADX-GUI with the JADX-AI-MCP plugin running — this is not a headless solution. The server communicates with an active decompiler instance via HTTP."* The 25-tool surface (`fetch_current_class`, `get_selected_text`, `xrefs_to_method`, `rename_variable`) is an analyst co-pilot for live GUI sessions. Our workflow is headless `jadx -d` across 86 APKs in parallel and then `rg` over the disk tree — wrong architectural fit. +- **`zinja-coder/apktool-mcp-server`** (same author as the jadx variant). Same GUI-coupled / interactive co-pilot architecture against `apktool d`. We only use apktool as a fallback path (`aapt2 dump xmltree` in `extract_corpus_components.py`), so the headless / multi-APK gap is the same as jadx-mcp-server. +- **`sfncat/mcp-joern`** (43★, Apr 2026, MIT) clears the recency floor but Joern is already our lowest-volume step ("only on shortlisted methods"). Marginal automation gain vs. a recipe in `joern-recipes.md`. Worth a separate spike later if Joern volume grows. +- **`JordyZomer/codeql-mcp`** (146★) ships without a LICENSE file — license-trap, not redistributable. + +If a new upstream MCP appears that *does* accept multi-pack configs (semgrep) or *can* drive headless decompile (jadx), revisit. Until then, the methodology lives in the skill prose where the heap-tier table, rule-pack choice, and Joern recipe can be taught. + +### Why no top-level agent + +Sibling capabilities (`ios-forensics`, `memory-forensics`, `web-security`, `bloodhound`) ship a coordinating agent that binds skills via frontmatter and gates tools. This capability deliberately does not. + +The four skills here cover non-overlapping intent shapes — corpus prep, broad semantic hunting (Mode A/B/C), single-APK depth, and protector triage — and each carries a precise `description:` trigger plus an `allowed-tools:` gate. The routing decision is *which skill to load*, and the skill frontmatter is enough to express that. An agent prompt layered on top would duplicate the Mode A/B/C selection logic already in `android-semantic-vuln-hunting/SKILL.md` without adding a `tools:` gate or `model:` pin a skill can't already express. + +Revisit if a future workflow needs persistent operating posture across multiple skills in one session (e.g. evidence rules + tool-priority order that span corpus prep and vuln hunting), or if a model pin distinct from the session default becomes the right choice for these workflows. + +## Operator-run scripts (large IO, not MCP tools) + +These stay scripts because they download bulk data the operator should supervise: + +- `scripts/androzoo_gp_metadata.py download` — ~1+ GB Google Play metadata +- `scripts/androzoo_download.py` — APK downloads with rate limiting and a download manifest +- `scripts/androzoo_to_parquet.py` — one-time conversion of the metadata sources to columnar Parquet +- `scripts/gplaydl_bulk.py` — anonymous Google Play bulk downloader (Aurora token dispenser) + +Their internals are unrelated to agent reasoning. Selection itself is `duckdb -c "SELECT ..."` against the Parquet files — see `android-corpus-prep` for the canonical recipe. + +## Scanner-gap labels (recap) + +Per finding hypothesis, attach `scanner_gap` after Semgrep/MobSF/APKHunt/`mobsfscan`: + +- `exact baseline finding` +- `adjacent baseline finding` +- `generic warning only` +- `not found` + +`adjacent` and `not found` are usually where the user's findings live. diff --git a/capabilities/android-apk-research/skills/android-targeted-assessment/SKILL.md b/capabilities/android-apk-research/skills/android-targeted-assessment/SKILL.md new file mode 100644 index 0000000..2affebf --- /dev/null +++ b/capabilities/android-apk-research/skills/android-targeted-assessment/SKILL.md @@ -0,0 +1,281 @@ +--- +name: android-targeted-assessment +description: "Use when assessing one Android APK deeply — inventory, decompile, ripgrep + Semgrep triage, Joern/CodeQL/FlowDroid escalation where warranted, hypothesis report, authorized validation plan." +allowed-tools: + - inventory_status + - run_corpus_inventory + - detect_runtime_kind + - extract_api_map + - normalize_semantic_findings + - bash + - read + - grep + - glob + - web_search + - web_extract + - report +license: MIT +--- + +# Android targeted APK assessment + +Use this skill when the user provides one APK (or a very small set) and wants depth over breadth. Output is a concise, evidence-backed report with semantic findings, scanner-gap labelling, and tiered validation plans. + +This skill rides the same pipeline as `android-semantic-vuln-hunting`, just compressed to one target. `android-semantic-vuln-hunting` is the canonical methodology — when the pipeline shape changes (new MCP tool, heap-tier update, new bug class), update both skills in the same commit. Read that skill first if you need methodology; this one is the recipe. + +## First: discover the available utilities + +If you are unsure which APK research flow applies, read the utility index first: + +```bash +cat ../android-semantic-vuln-hunting/references/agent-utility-index.md +``` + +For a single APK, the key optional branches are: + +- commercial protector detected → load `android-protector-triage` +- React Native / Flutter / hybrid runtime → JS/Dart route-map validation before grading +- rich backend client signs → run `scripts/extract_api_map.py` and use `backend-rich-apk-workflows.md` +- feature flags / request signing / workflow verbs / native bridges → use the matching references before writing hypotheses +## 0. Scope + tooling sanity + +Confirm with the user: + +- the APK path and authorization +- whether dynamic validation (ADB / emulator / device / test account / backend) is in scope + +Then verify tools (no custom tool needed): + +```bash +for t in jadx apktool aapt aapt2 adb androguard apkid semgrep joern codeql; do + printf '%-12s ' "$t"; command -v "$t" || echo MISSING +done +``` + +## 1. Inventory the APK + +Even for one APK, `run_corpus_inventory` gives a clean per-SHA artifact tree with Androguard-decoded manifest facts and APKiD packer detection — much faster than reading the binary manifest yourself. + +```bash +run_corpus_inventory(paths=["path/to.apk"], out_dir="findings/", include_apkid=true) +``` + +Read the produced `findings//apks//androguard.json` first. Look for: + +- `package`, `version_name`, `target_sdk` +- `browsable_components` — these are the externally-reachable entrypoints +- `components[*].exported`, `permission`, `intent_filters` — full attack surface +- `schemes`, `hosts` — deep link URI shapes +- `permissions` — what the app is allowed to do (dangerous perms suggest sensitive flows) + +Read `apkid.json` for packer/protector/obfuscator signal. Heavy obfuscation changes the strategy: deeper Joern reliance, less faith in `rg`, prefer dynamic validation. + +If APKiD flags a commercial protector OR `lib//libdpboot.so` / `libdexprotector.so` is present OR `assets/` contains opaque `.dat` blobs (`se.dat`, `classes.dex.dat`, `mm.dat`, `dp.mp3`, `resources.dat`, `ic.dat`, `ct.dat`, `rcdb.dat`), **stop here and load the `android-protector-triage` skill**. JADX output, ripgrep over decompiled sources, and Semgrep baselines are all incomplete under commercial protection; running them as if the APK were unprotected produces false confidence. The protector-triage skill includes detection, structural unpack (DexProtector → libdp.so today), and the protector-aware decompile/triage decisions. + +## 2. Detect runtime kind, then decompile + +A 1-second probe drives both the JADX heap setting **and** whether you need a JS-bundle or Dart-AOT trace later. Doing this before JADX saves a wasted OOM run. + +```python +# MCP tool — returns dict with `runtime_kind` in +# {native, react_native_js, react_native_hermes, flutter_aot, +# capacitor, cordova, unity, xamarin, maui}. +detect_runtime_kind(apk="path/to.apk") +``` + +For `react_native_*` and `flutter_aot`, default Java-side hypotheses to `confidence_tier=needs_route_map_validation` (see §4.5 / §4.6 below). + +```bash +SRC=findings//sources +# Heap tier by DEX class count (read from androguard.json, or `apkanalyzer dex packages`) +# <15k classes -> -Xmx2g (most apps) +# 15-30k classes -> -Xmx6g (wallets, messengers, large banking) +# >30k classes -> -Xmx8g +JAVA_OPTS="-Xmx6g" jadx --show-bad-code --no-debug-info -d "$SRC" "$APK" +``` + +Empirical (corpus observation): a wallet-class APK at ~27k classes / 83 MB (Trust Wallet) OOMs at `-Xmx2g` and runs clean at `-Xmx6g`; a smaller wallet at ~13k classes (MetaMask) is fine at `-Xmx2g`. **Start at the right heap; one retry burns 3-5 minutes.** + +The first-party tree usually lives under `$SRC/sources/`. Use that path for the rest of the work. + +## 3. ripgrep the focused source/sink set on first-party code + +If the APK has a known impact class (assigned by `android-corpus-prep`, e.g. `C_wallet`), run the per-class profile first — it captures patterns unique to that app type and is usually 3× faster to triage than the universal set: + +```bash +bash scripts/run_class_rg.sh "$APP" findings// +``` + +Profiles for all 10 classes live in `../android-semantic-vuln-hunting/references/impact-class-rg-profiles.md`. + +### Universal source/sink set + +```bash +APP=$SRC/sources/$(echo | tr . /) +rg -n \ + -e 'getIntent\(' -e 'getData\(' -e 'getStringExtra\(' \ + -e 'getParcelableExtra\(' -e 'getSerializableExtra\(' \ + -e 'Intent\.parseUri\(' -e 'Uri\.parse\(' -e 'getQueryParameter\(' \ + -e 'startActivity\(' -e 'startService\(' -e 'sendBroadcast\(' \ + -e 'WebView' -e 'loadUrl\(' -e 'postUrl\(' \ + -e 'addJavascriptInterface\(' -e 'setJavaScriptEnabled\s*\(\s*true' \ + -e 'shouldOverrideUrlLoading' -e 'CookieManager' \ + -e 'Authorization' -e 'Bearer ' \ + -e 'SharedPreferences' -e 'isLoggedIn' \ + -e 'FLAG_GRANT_READ_URI_PERMISSION' -e 'FileProvider' \ + "$APP" > findings//rg.txt +``` + +Read only the files that show **multiple** categories — especially `getIntent`/`getQueryParameter` together with `loadUrl`/`addJavascriptInterface` or with `startActivity`. Cross-reference against `androguard.json` exported/BROWSABLE components. + +### Backend-rich API map pass + +If the target appears to be a rich backend client (Retrofit/Apollo/gRPC/WebSocket/Firebase, request signing, feature flags, tenant/account/device/payment/order IDs), extract an API map before writing backend hypotheses: + +```python +extract_api_map( + src="$SRC", + out="findings//api_map.jsonl", + summary="findings//backend_richness.json", + dedupe=True, +) +``` + +Use `../android-semantic-vuln-hunting/references/backend-rich-apk-workflows.md` for APK→API validation planning. Static endpoint/DTO evidence is normally `needs_backend_validation`; do not probe live backend actions without explicit scope. +## 4. Semgrep scanner baseline + scanner_gap + +```bash +semgrep --config p/security-audit \ + --config p/mobsfscan \ + --config r/java \ + --metrics=off \ + --json --output findings//semgrep.json "$SRC" +``` + +When writing each hypothesis, attach `scanner_gap` (exact / adjacent / generic / not found). See `../android-semantic-vuln-hunting/references/scanners.md` for label definitions and other scanner options (MobSF, APKHunt, `mobsfscan`). + +## 4.5 React Native / hybrid: read the JS bundle before grading findings + +If the runtime probe in §2 reported `react_native_js` or `react_native_hermes`, the JS bundle owns the routing decision. Plain JS — prettier + rg. Hermes — `hbctool disasm` first (string table is the high-signal target): + +```bash +# Plain JS +BUNDLE=$SRC/resources/assets/index.bundle +mkdir -p findings//js-analysis +npx --yes prettier@3.3.3 --parser babel --print-width 120 "$BUNDLE" \ + > findings//js-analysis/index.pretty.js +JSDIR=findings//js-analysis + +# Hermes bytecode (file reports `data`, magic c6 1f bc 03 per facebook/hermes BCVersion.h) +pipx install hbctool 2>/dev/null +hbctool disasm $SRC/resources/assets/index.android.bundle findings//js-analysis/hbc +JSDIR=findings//js-analysis/hbc + +rg -n -e 'addEventListener\(.url.' -e 'Linking\.' -e 'emitNewIntentReceived' \ + -e '_PATHS_TO_SCREENS_MAP' -e 'DEEP_LINK' -e 'ALLOWED_HOSTS' "$JSDIR" +``` + +Trace each Java-side bridge symbol backwards from its callers to the URL/host/path validators in JS. A strict allowlist downgrades Java findings to `hardening_only`. A `contains`/`startsWith` or no check is a real bug. Pre-bundle gradings should default to `needs_route_map_validation`, not `strong_static_chain`. + +Full recipe and grading rules: `../android-semantic-vuln-hunting/SKILL.md` § Step 7.5. + +## 4.6 Flutter / Dart AOT: read libapp.so before grading findings + +If the runtime probe reported `flutter_aot`, the Dart code in `lib//libapp.so` owns routing. Java/Kotlin side is a MethodChannel relay; pre-bundle Java findings default to `needs_route_map_validation`. + +```bash +# Recover Dart classes / strings / route keys with blutter +# https://github.com/worawit/blutter +blutter $SRC/resources/lib/arm64-v8a/ findings//dart-analysis/ + +# Match Java MethodChannel names against Dart-side handlers +rg -n 'new MethodChannel\|new C6\.o\|new EventChannel' $SRC \ + | rg -o '"[^"]+"' | sort -u > findings//method-channels.txt +rg -nN -f findings//method-channels.txt findings//dart-analysis/ +``` + +Dart-side strict allowlist on URL/host -> `hardening_only`. Loose check or no check -> real bug. Without blutter, fall back to `strings -a libapp.so` filtered by route-key patterns; surfaces literals but loses structure. + +Full recipe: `../android-semantic-vuln-hunting/SKILL.md` § Step 7.6. + +## 5. Joern for call/data context — only on shortlisted methods + +For each surviving hypothesis where you need to know "who calls this", "where does this value come from", or "what does the bridge expose": + +```bash +joern --script <(cat </hypotheses.jsonl"], + output_format="markdown", + out="findings//report.md", +) +``` + +Build the ADB validation plan from `androguard.json` (decoded schemes/hosts/components): + +```bash +# Browsable deep link probe — non-destructive, local device only. +adb shell am start -a android.intent.action.VIEW \ + -d ':///?dn_probe=1' \ + +adb logcat -d | rg -i '|deep|webview' +``` + +For nested-Intent / Parcelable redirection, plan a tiny helper APK instead of trying `am start` payloads; record what it would do rather than building it unless authorized. + +## Depth-first review priorities + +For a single target, spend more time on: + +- exported + BROWSABLE activities and their `onCreate`/router methods +- internal deep-link dispatcher implementation (often a `*Router`, `*DeepLink*`, `*Navigator` class) +- WebView setup (`setJavaScriptEnabled`, `addJavascriptInterface`, `shouldOverrideUrlLoading`) +- auth/session/account-state checks (`SharedPreferences`, `isLoggedIn`, `access_token`, `Bearer`) +- password reset / invite / OAuth / magic link / payment flows +- exported `ContentProvider`s and `FileProvider` paths + +## Evidence standard + +A targeted report should not contain generic warnings. Each finding requires: + +- manifest or code entrypoint evidence +- attacker-controlled source +- specific trust-boundary mistake +- sensitive sink +- plausible impact +- a validation plan with an explicit authorization tier + +If impact depends on backend behaviour, set `confidence_tier=needs_backend_validation` and describe the test account/scope required. Do not claim exploitability from local bypass alone. + +## Report shape + +1. Scope and artifact identifiers (path, package, version, SHA256, JADX/scanner versions). +2. Decoded manifest summary (component counts, exported/BROWSABLE, schemes, hosts, dangerous permissions). +3. Top semantic findings, each with the full schema. +4. Scanner baseline summary and `scanner_gap` table. +5. Validation queue grouped by tier (`tier0_static_only` → `tier3_explicit_production_authorization`). +6. Appendix with normalized JSONL hypotheses.