diff --git a/.github/workflows/full-public-smokes.yml b/.github/workflows/full-public-smokes.yml index db496a89f..e9787a274 100644 --- a/.github/workflows/full-public-smokes.yml +++ b/.github/workflows/full-public-smokes.yml @@ -31,6 +31,9 @@ concurrency: group: full-public-smokes-${{ github.ref }} cancel-in-progress: true +env: + LOOPX_USAGE_PING: "0" + jobs: full-public-smokes: if: github.repository == 'loopx-project/loopx' diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 08d3a1db8..f362dd0d8 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -30,6 +30,9 @@ concurrency: group: python-tests-${{ github.ref }} cancel-in-progress: true +env: + LOOPX_USAGE_PING: "0" + jobs: changes: runs-on: ubuntu-latest diff --git a/docs/development/testing-and-quality.md b/docs/development/testing-and-quality.md index 444ec37c8..74b2b2862 100644 --- a/docs/development/testing-and-quality.md +++ b/docs/development/testing-and-quality.md @@ -73,6 +73,24 @@ golden 来让测试通过。 ## Pull-Request Baseline / PR 基线 +### Synthetic Runs Must Not Report Adoption / 合成运行不计入使用遥测 + +CI, pytest and canary smoke subprocesses disable usage collection with +`LOOPX_USAGE_PING=0`. A synthetic installer, benchmark profile or release +qualification that reconstructs its environment must set that opt-out itself, +including in Agent tool shells; filtering out `CI` must never restore collection. +Do not infer test provenance from OS, install channel or random installation IDs. +Telemetry transport tests may explicitly opt in only with isolated state and a +disposable local collector. Validate the actual child CLI and typed sender, and +assert zero HTTP requests for disabled profiles rather than only inspecting a +parent environment dictionary. + +CI、pytest 和 canary smoke 子进程通过 `LOOPX_USAGE_PING=0` 关闭遥测。合成安装、 +评测 profile 或发布资格验证重建环境时，必须自行设置关闭开关，并覆盖 Agent 工具 +shell；不能因为过滤了 `CI` 就恢复采集。不能按系统、安装渠道或随机 ID 推断测试 +来源。遥测传输专项测试只允许使用隔离状态和可丢弃的本地收集器显式开启。验证实际 +子进程 CLI 和类型化发送端，并断言关闭状态下 HTTP 请求为零，而非只检查父环境。 + ### Required Merge Check / 必需合并检查 `python-tests.yml` publishes `merge-gate` for every pull request. Code, diff --git a/docs/reference/usage-ping.md b/docs/reference/usage-ping.md index 776e3c272..2daca9c5f 100644 --- a/docs/reference/usage-ping.md +++ b/docs/reference/usage-ping.md @@ -105,6 +105,14 @@ settings also block all channels, even after explicit enable: - `DO_NOT_TRACK` set to a nonempty value other than `0` - `CI` set to a nonempty value other than `0|false` +Project Python CI and public smoke workflows explicitly set `LOOPX_USAGE_PING=0`. +Pytest and the canary smoke runner also disable collection for local validation. +Native Codex benchmark profiles force the same opt-out during installation, +runtime and Agent shell execution, even when a minimal environment removes `CI` +or the parent requests collection. Release qualification uses the same boundary. +These synthetic runs must not count as reporting installations. Telemetry tests +may explicitly enable collection only against a disposable local collector. + `LOOPX_USAGE_POLICY=consent_required` requires explicit enable; merely displaying the notice is insufficient. Default policy is `opt_out`; unknown policies fail closed. Distribution owners must choose the applicable policy before shipping; diff --git a/docs/reference/usage-ping.zh-CN.md b/docs/reference/usage-ping.zh-CN.md index 23d06d92d..bd8681251 100644 --- a/docs/reference/usage-ping.zh-CN.md +++ b/docs/reference/usage-ping.zh-CN.md @@ -87,6 +87,12 @@ App 不再要求首次点击启用。环境变量覆盖和 `consent_required` - `DO_NOT_TRACK` 非空且不是 `0` - `CI` 非空且不是 `0|false` +项目 Python CI 和公共 smoke 工作流显式设置 `LOOPX_USAGE_PING=0`；pytest 与 +canary smoke 执行器也在本地验证时关闭采集。Native Codex 评测 profile 的安装、 +运行和 Agent 工具 shell 都强制关闭，即使最小环境丢掉了 `CI` 或父进程要求开启， +也不会恢复采集。发布资格验证同样关闭。这些合成运行不能计作真实使用安装。 +遥测专项测试只能针对可丢弃的本地收集器显式开启。 + `LOOPX_USAGE_POLICY=consent_required` 要求明确开启，单纯显示告知不够。 默认策略为 `opt_out`，未知值拒绝发送。发行方必须按实际适用要求选择策略； 该配置不自动判断法律合规，不按 IP 猜测地区，也不能替代必要的同意。 diff --git a/loopx/canary/runner.py b/loopx/canary/runner.py index 46c754a0f..830346223 100644 --- a/loopx/canary/runner.py +++ b/loopx/canary/runner.py @@ -1,7 +1,8 @@ from __future__ import annotations -import shlex +import os import re +import shlex import subprocess import sys import time @@ -268,6 +269,7 @@ def _run_check( completed = subprocess.run( normalized["argv"], cwd=REPO_ROOT, + env={**os.environ, "LOOPX_USAGE_PING": "0"}, text=True, encoding="utf-8", errors="replace", stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/loopx/capabilities/benchmark_toolkit/native_codex_profile.py b/loopx/capabilities/benchmark_toolkit/native_codex_profile.py index d499179c1..09f295d60 100644 --- a/loopx/capabilities/benchmark_toolkit/native_codex_profile.py +++ b/loopx/capabilities/benchmark_toolkit/native_codex_profile.py @@ -210,6 +210,9 @@ def _formal_install_environment( "LOOPX_PYTHON": python_executable, "LOOPX_PROMOTE_DEFAULT": "1", "LOOPX_INSTALL_CANARY": "0", + # Synthetic profiles must not become adoption samples, even when + # rebuilding the environment drops the supervisor's CI/opt-out flags. + "LOOPX_USAGE_PING": "0", "LOOPX_BIN_DIR": str(paths["bin_dir"]), "LOOPX_RELEASES_DIR": str(paths["release_root"].parent), "LOOPX_RELEASE_ID": release_id, @@ -249,6 +252,7 @@ def native_codex_profile_environment( "TMP": str(profile.home), "TEMP": str(profile.home), "CODEX_HOME": str(profile.codex_home), + "LOOPX_USAGE_PING": "0", "PATH": f"{profile.bin_dir}{os.pathsep}{inherited_path}", } ) @@ -284,6 +288,8 @@ def native_codex_app_server_shell_policy_args( f"shell_environment_policy.include_only={json.dumps(_AGENT_SHELL_ENV_INCLUDE_ONLY)}", "-c", f"shell_environment_policy.exclude={json.dumps(normalized)}", + "-c", + 'shell_environment_policy.set.LOOPX_USAGE_PING="0"', ) diff --git a/scripts/qualify-native-goal-release.py b/scripts/qualify-native-goal-release.py index b729496ee..9af62a2ea 100644 --- a/scripts/qualify-native-goal-release.py +++ b/scripts/qualify-native-goal-release.py @@ -31,6 +31,7 @@ def host_environment(root: Path, launcher: Path) -> dict[str, str]: "PATH": str(launcher.parent) + os.pathsep + os.environ.get("PATH", os.defpath), "HOME": str(home), "TMPDIR": str(temporary), "SHELL": "/bin/sh", "LANG": "C.UTF-8", "PYTHONPATH": str(REPO), + "LOOPX_USAGE_PING": "0", "XDG_CONFIG_HOME": str(home / ".config"), "XDG_CACHE_HOME": str(home / ".cache"), } diff --git a/skills/loopx-self-repair/references/repair-patterns.md b/skills/loopx-self-repair/references/repair-patterns.md index e00ca3309..4252df3ce 100644 --- a/skills/loopx-self-repair/references/repair-patterns.md +++ b/skills/loopx-self-repair/references/repair-patterns.md @@ -5,6 +5,7 @@ teaches a reusable control-plane lesson. | Pattern | Symptoms | Evidence To Read | Likely Root | Durable Repair | | --- | --- | --- | --- | --- | +| `synthetic_profile_telemetry_leak` | Reporting installation IDs rise during tests, isolated installs or benchmark preparation. | Synthetic state with send receipts, collector matches when authorized, child environment builders and Agent shell policy. | Environment allowlists drop CI and telemetry opt-outs while fresh homes generate new IDs. | Force collection off at synthetic install/runtime/tool-shell boundaries and test/smoke entrypoints; exercise real child CLI and typed sender against a local collector with zero-request assertions. Keep normal user collection unchanged; do not infer or delete historical test samples from platform/channel heuristics. | | `authority_cold_read_starves_runtime` | Unrelated pure Todo rules and ping time out while warm reads are fast. | Same-byte isolated cold/warm/alternating-Goal probes, original response budget, CPU profile and exact provider revision. | Synchronous retained-history verification monopolizes the shared event loop; allocation-heavy canonical key sorting amplifies it. | Preserve byte-level proof semantics, reduce codec allocations, yield between complete transaction proofs and share only identical in-flight proofs. Test real socket concurrency and late-history corruption; do not raise timeouts, weaken integrity or replay ambiguous writes. | | `native_todo_status_index_schema_gap` | A Goal shows “status load failed / invalid response” after native Todos appear, while the scoped status endpoint returns valid JSON. | Exact scoped status response, Zod issue paths, Todo `todo_id` and `index` fields, native presentation contract, packaged Goal load. | The dashboard still requires every Todo to have a numeric source index, but native Todos deliberately use stable `todo_id` with absent or null index. One row rejects the entire Goal snapshot. | Accept nullable/absent index only when a nonempty stable Todo ID exists; keep numeric legacy indexes and reject anonymous or malformed rows. Render and act by Todo ID, then prove a mixed native/legacy scoped snapshot loads in the packaged UI. | | `capability_catalog_editor_kind_drift` | Machine or Goal settings report an empty capability list even though the configuration API returns registered capabilities. | Live API catalog IDs and editor kinds, the dashboard's accepted field-kind schema, and the page's load-error state. | One new descriptor emits an unsupported field kind; strict validation rejects the shared catalog and the machine page presents the failed load as an empty registry. | Keep the published editor vocabulary aligned with the browser contract, check every built-in descriptor together, and show a retryable error when catalog loading or validation fails. Only a successfully loaded empty catalog may show the empty state. | diff --git a/tests/canary/test_telemetry_isolation.py b/tests/canary/test_telemetry_isolation.py new file mode 100644 index 000000000..9bfe26090 --- /dev/null +++ b/tests/canary/test_telemetry_isolation.py @@ -0,0 +1,22 @@ +"""The smoke runner suppresses telemetry in actual child processes.""" +import json + +from loopx.canary import runner + + +def test_smoke_subprocess_overrides_parent_telemetry_enable(tmp_path, monkeypatch): + examples = tmp_path / "examples" + examples.mkdir() + (examples / "environment.py").write_text( + "import json,os\nprint(json.dumps({k:os.environ.get(k) for k in " + "['LOOPX_USAGE_PING','CI','SYNTHETIC_VALUE']}))\n", encoding="utf-8", + ) + monkeypatch.setattr(runner, "REPO_ROOT", tmp_path) + monkeypatch.setenv("LOOPX_USAGE_PING", "1") + monkeypatch.setenv("SYNTHETIC_VALUE", "preserved") + monkeypatch.delenv("CI", raising=False) + result = runner._run_check({"command": "python examples/environment.py"}, timeout_seconds=10) + assert result["ok"], result + assert json.loads(result["stdout_tail"]) == { + "LOOPX_USAGE_PING": "0", "CI": None, "SYNTHETIC_VALUE": "preserved", + } diff --git a/tests/capabilities/test_native_codex_profile_skill_versions.py b/tests/capabilities/test_native_codex_profile_skill_versions.py index 4889bc532..27c6a6835 100644 --- a/tests/capabilities/test_native_codex_profile_skill_versions.py +++ b/tests/capabilities/test_native_codex_profile_skill_versions.py @@ -68,11 +68,14 @@ def doctor(profile, *args): assert profiles[0].source_revision == profiles[1].source_revision for profile in profiles: assert doctor(profile, "--deep")["typescript_control_plane"]["ready"] + assert not (profile.home / ".codex/loopx/usage-ping.json").exists() stopped = doctor(profiles[0], "--restart-runtime")["effect_runtime_restart"] assert stopped["status"] == "stopped" assert Path(stopped["info_path"]).is_relative_to(profiles[0].home) other = doctor(profiles[1])["typescript_control_plane"] assert other["runtime_lifecycle"]["state"] == "running" + for profile in profiles: + assert not (profile.home / ".codex/loopx/usage-ping.json").exists() finally: for profile in profiles: assert doctor(profile, "--restart-runtime")["effect_runtime_restart"][ diff --git a/tests/capabilities/test_native_codex_profile_telemetry.py b/tests/capabilities/test_native_codex_profile_telemetry.py new file mode 100644 index 000000000..946097f37 --- /dev/null +++ b/tests/capabilities/test_native_codex_profile_telemetry.py @@ -0,0 +1,93 @@ +"""Synthetic profile environments never contribute adoption telemetry.""" +from __future__ import annotations + +import os +import subprocess +import sys +import threading +import tomllib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from loopx.capabilities.benchmark_toolkit.native_codex_profile import ( + _formal_install_environment, + native_codex_app_server_shell_policy_args, + native_codex_profile_environment, +) + + +def profile_environments(root, base): + paths = {key: root / key for key in ( + "home", "codex_home", "bin_dir", "release_root", "man_root", "shell_profile", "skills_dir", + )} + return [ + _formal_install_environment(paths=paths, python_executable=sys.executable, + release_id="fixture", base_env=base), + native_codex_profile_environment(SimpleNamespace(**paths), base_env=base), + ] + + +@pytest.mark.parametrize("base", [{}, {"CI": "true"}, {"LOOPX_USAGE_PING": "1"}, + {"DO_NOT_TRACK": "1", "LOOPX_USAGE_PING": "0"}]) +def test_install_and_runtime_environments_disable_even_without_inherited_ci(tmp_path, base): + for env in profile_environments(tmp_path, base): + assert env.get("LOOPX_USAGE_PING") == "0" + + +def test_agent_shell_reconstruction_explicitly_disables_telemetry(): + arguments = native_codex_app_server_shell_policy_args(excluded_env_keys=("RUNNER_SENTINEL",)) + policy = tomllib.loads("\n".join(arguments[1::2]))["shell_environment_policy"] + assert policy["set"]["LOOPX_USAGE_PING"] == "0" + assert "RUNNER_SENTINEL" in policy["exclude"] + # The explicit override survives even if the shell inherits no parent keys. + result = subprocess.run([sys.executable, "-c", "import os; print(os.environ['LOOPX_USAGE_PING'])"], + env=policy["set"], capture_output=True, text=True, check=True) + assert result.stdout.strip() == "0" + + +def test_profile_cli_and_typed_sender_cannot_override_disabled_collection(tmp_path): + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + requests.append(self.path) + self.send_response(204) + self.end_headers() + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + env = profile_environments(tmp_path, {"PATH": os.environ["PATH"], "LOOPX_USAGE_PING": "1"})[1] + # Any regression is contained by a disposable collector, never production. + env["LOOPX_USAGE_PING_ENDPOINT"] = f"http://127.0.0.1:{server.server_port}/v1/ping" + code = ''' +import json +from loopx import usage_ping +from loopx.cli_runtime import main +enabled = usage_ping.control("enable") +assert enabled["blocked_by"] == "LOOPX_USAGE_PING" +assert not enabled["sending"] +assert main(["version", "--format", "json"]) == 0 +assert main(["version", "--format", "json"]) == 0 +state = json.loads(usage_ping.state_path().read_text()) +for action, fields in [("start", {}), ("observe", dict(feature="todo", outcome="ok", error="none", elapsed_ms=1))]: + result = usage_ping.control(action, generation=state["generation"], **fields) + assert result == {"sent": False, "reason": "blocked"} +after = json.loads(usage_ping.state_path().read_text()) +assert after == state +assert "last_attempt_day" not in after and "counters" not in after +''' + result = subprocess.run([sys.executable, "-c", code], env=env, + cwd=Path(__file__).resolve().parents[2], + capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stderr + assert requests == [] + finally: + server.shutdown() + server.server_close() diff --git a/tests/conftest.py b/tests/conftest.py index d79da298a..156af8272 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,14 @@ from __future__ import annotations +import os import sys from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] +# Suppress collection before imports and in inherited test subprocesses. +# Telemetry transport tests explicitly opt in against disposable collectors. +os.environ["LOOPX_USAGE_PING"] = "0" if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) diff --git a/tests/test_native_goal_release_qualification.py b/tests/test_native_goal_release_qualification.py index b0376e1ae..6d476d349 100644 --- a/tests/test_native_goal_release_qualification.py +++ b/tests/test_native_goal_release_qualification.py @@ -43,10 +43,12 @@ class InspectedSpawn(Exception): def inspect(command, **kwargs): assert len(prompt_loads) == 1 env = kwargs["env"] + assert env["LOOPX_USAGE_PING"] == "0" assert "UNRELATED_AUTH_TOKEN" not in env and "SSH_AUTH_SOCK" not in env settings = tomllib.loads((Path(env["CODEX_HOME"]) / "config.toml").read_text()) policy = settings["shell_environment_policy"] assert policy["inherit"] == "none" + assert policy["set"]["LOOPX_USAGE_PING"] == "0" assert "LOOPX_CODEX_QUALIFICATION_API_KEY" not in policy["set"] child = subprocess.run([sys.executable, "-c", "import os,json; print(json.dumps(dict(os.environ)))"], env=policy["set"], capture_output=True, text=True, check=True) diff --git a/tests/test_usage_goal.py b/tests/test_usage_goal.py index 110d5ad37..3584cd30b 100644 --- a/tests/test_usage_goal.py +++ b/tests/test_usage_goal.py @@ -11,11 +11,19 @@ def test_telemetry_failure_cannot_replace_host_exception(tmp_path, monkeypatch): monkeypatch.setattr(usage_ping, "DEFAULT_RUNTIME_ROOT", tmp_path) usage_ping.state_path().write_text(json.dumps({"generation": "fixture", "consent": "enabled"})) - monkeypatch.delenv("CI", raising=False) - monkeypatch.setattr(usage_ping, "_detach", lambda _: (_ for _ in ()).throw(OSError("fixture failure"))) + for name in ("CI", "DO_NOT_TRACK", "LOOPX_USAGE_PING"): + monkeypatch.delenv(name, raising=False) + attempts = [] + + def fail_transport(request): + attempts.append(request) + raise OSError("fixture failure") + + monkeypatch.setattr(usage_ping, "_detach", fail_transport) with pytest.raises(ValueError, match="host failure"): with usage_goal.observe_goal_execution(tmp_path, "fixture-goal"): raise ValueError("host failure") + assert attempts, "the test must exercise transport failure, not an environment opt-out" def test_disabled_observer_starts_no_worker_or_process(tmp_path, monkeypatch):