From f858cc871323450b294a572f2f480600b4a16211 Mon Sep 17 00:00:00 2001 From: Matt Miller Date: Sat, 1 Aug 2026 16:48:22 -0700 Subject: [PATCH 1/4] feat(telemetry): emit partner_nodes_detected + stamp caller_kind on every event (BE-5633) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Partner-node detection already ran on every local `comfy run` but never reached telemetry, and caller attribution (caller.py) never reached event props. Wire both through tracking.py so partner-API usage is measurable and attributable. - `comfy run` (local) fires `partner_nodes_detected` whenever the workflow uses partner-API nodes, before the partner_node_requires_credential rejection so that funnel is counted (credential_present marks it). - `_dispatch` stamps `caller_kind` (user/pipe/agent/claude-code/custom COMFY_USER_AGENT label) on every event, alongside cli_version/tracing_id. Telemetry only — no change to run semantics. --- comfy_cli/command/run/__init__.py | 77 +++++++----- comfy_cli/tracking.py | 13 +- tests/comfy_cli/command/test_run.py | 179 ++++++++++++++++++++++++++++ tests/comfy_cli/test_tracking.py | 46 ++++++- 4 files changed, 285 insertions(+), 30 deletions(-) diff --git a/comfy_cli/command/run/__init__.py b/comfy_cli/command/run/__init__.py index ad299286..d4362ebd 100644 --- a/comfy_cli/command/run/__init__.py +++ b/comfy_cli/command/run/__init__.py @@ -22,7 +22,7 @@ WebSocketTimeoutException, ) -from comfy_cli import cancellation, execution_errors, jobs_state +from comfy_cli import cancellation, execution_errors, jobs_state, tracking # Re-exports — names patched by tests live at this namespace. from comfy_cli.command.run.credentials import _resolve_partner_credential as _resolve_partner_credential @@ -291,32 +291,55 @@ def execute( extra_data: dict | None = None if api_key: extra_data = {"api_key_comfy_org": api_key} - # Only resolve an injected credential when an explicit --api-key hasn't - # already satisfied the partner node: the resolver may perform a network - # OAuth refresh, so skipping it here keeps an explicit-key run network-free. - if partner_nodes and not extra_data: - cred = _resolve_partner_credential() - if cred is None: - msg = ( - "Workflow uses partner-API node(s) that need an `api_key_comfy_org` " - "credential the local server doesn't have: " + ", ".join(partner_nodes) + "." - ) - renderer.error( - code="partner_node_requires_credential", - message=msg, - hint=( - "run: comfy cloud login (or set COMFY_API_KEY in the environment, " - "or persist a key with `comfy cloud set-key --key …`; " - "cloud runs auto-inject via --where cloud)" - ), - details={ - "partner_nodes": partner_nodes, - "host": host, - "port": port, - }, - ) - raise typer.Exit(code=1) - extra_data = {cred[0]: cred[1]} + if partner_nodes: + # Only resolve an injected credential when an explicit --api-key hasn't + # already satisfied the partner node: the resolver may perform a network + # OAuth refresh, so skipping it here keeps an explicit-key run network-free. + # Resolved once — the result feeds both the telemetry prop below and the + # credential gate that follows. + cred = _resolve_partner_credential() if not extra_data else None + # Fired BEFORE the reject-for-missing-credential branch so runs that are + # turned away are still counted: that funnel is exactly what the metric + # is for, and `credential_present: False` marks them. class_types are + # node names, not PII — the same data `workflow_unknown_nodes` reports. + # It does sit AFTER the BE-4326 spend gate, so a run refused for lack of + # `--allow-spend` emits no event: the gate deliberately precedes any + # credential resolution (a refusal must not trigger a network OAuth + # refresh), and `credential_present` needs that resolution. The + # spend-declined funnel wants its own event rather than an early + # resolve here. + tracking.track_event( + "partner_nodes_detected", + { + # Cap defends against pathological graphs; the count stays exact. + "partner_nodes": partner_nodes[:20], + "partner_node_count": len(partner_nodes), + "where": "local", + "credential_present": bool(api_key) or cred is not None, + }, + ) + if not extra_data: + if cred is None: + msg = ( + "Workflow uses partner-API node(s) that need an `api_key_comfy_org` " + "credential the local server doesn't have: " + ", ".join(partner_nodes) + "." + ) + renderer.error( + code="partner_node_requires_credential", + message=msg, + hint=( + "run: comfy cloud login (or set COMFY_API_KEY in the environment, " + "or persist a key with `comfy cloud set-key --key …`; " + "cloud runs auto-inject via --where cloud)" + ), + details={ + "partner_nodes": partner_nodes, + "host": host, + "port": port, + }, + ) + raise typer.Exit(code=1) + extra_data = {cred[0]: cred[1]} # Pre-submit validation via pure-Python CQL engine (checks class_types + input shapes). _preflight_validate(renderer, workflow, object_info, target_label="server") diff --git a/comfy_cli/tracking.py b/comfy_cli/tracking.py index bddb5549..a2798200 100644 --- a/comfy_cli/tracking.py +++ b/comfy_cli/tracking.py @@ -14,6 +14,7 @@ import typer from comfy_cli import constants, logging, ui +from comfy_cli.caller import detect_caller from comfy_cli.config_manager import ConfigManager from comfy_cli.workspace_manager import WorkspaceManager @@ -124,6 +125,10 @@ def _scrub_value(value: object) -> object: user_id = config_manager.get(constants.CONFIG_KEY_USER_ID) # tracking all events for a single command tracing_id = str(uuid.uuid4()) +# Who is driving this process: "user" | "pipe" | "agent" | "claude-code" | a +# lowercased custom COMFY_USER_AGENT label. Computed once at import, matching +# the cli_version/tracing_id pattern above, so we don't re-run isatty per event. +_caller_kind = detect_caller().kind workspace_manager = WorkspaceManager() # Process-scoped opt-in used when running non-interactively before the @@ -350,12 +355,16 @@ def disable(): def _dispatch( event_name: str, properties: dict[str, Any], *, distinct_id: str | None, mixpanel_name: str | None = None ): - """Fan an event out to every provider. Enriches with cli_version/tracing_id. + """Fan an event out to every provider. Enriches with cli_version/tracing_id/caller_kind. This is the shared send path; callers above own the gating (consent for passive telemetry, env-only for feedback). + + ``caller_kind`` lands on EVERY event (execution_*, partner_nodes_detected, + feedback, …) — that is the point: it makes agent-vs-human analytics possible + across the whole stream. Purely additive, so no existing dashboard breaks. """ - properties = {**properties, "cli_version": cli_version, "tracing_id": tracing_id} + properties = {**properties, "cli_version": cli_version, "tracing_id": tracing_id, "caller_kind": _caller_kind} for provider in _get_providers(): provider_event_name = ( mixpanel_name if (mixpanel_name is not None and isinstance(provider, MixpanelProvider)) else event_name diff --git a/tests/comfy_cli/command/test_run.py b/tests/comfy_cli/command/test_run.py index 5c19b9aa..bff99cac 100644 --- a/tests/comfy_cli/command/test_run.py +++ b/tests/comfy_cli/command/test_run.py @@ -1243,6 +1243,185 @@ def test_non_partner_workflow_skips_preflight(self, workflow_file, monkeypatch): MockExec.assert_called_once() +class TestPartnerNodesDetectedTelemetry: + """Partner-node detection runs on every local `comfy run`; this is the + telemetry that makes partner-API usage measurable. It fires whenever the + workflow has partner nodes — including runs that are then rejected for a + missing credential, which is exactly the funnel the metric is for.""" + + PARTNER_WF = { + "1": {"class_type": "SomePartnerNode", "inputs": {"prompt": "x"}}, + "2": {"class_type": "PreviewAny", "inputs": {"source": ["1", 0]}}, + } + # The authoritative signal is `api_node: true` (category prefix is a fallback). + OBJECT_INFO = { + "SomePartnerNode": { + "category": "image", + "api_node": True, + "output": ["IMAGE"], + "output_name": ["IMAGE"], + }, + "PreviewAny": {"category": "image", "output": [], "output_name": [], "output_node": True}, + } + + def _wf_file(self, tmp_path, workflow=None): + path = tmp_path / "partner-telemetry.json" + path.write_text(json.dumps(self.PARTNER_WF if workflow is None else workflow)) + return str(path) + + @staticmethod + def _partner_events(mock_track): + """Props of every ``partner_nodes_detected`` call on the mock.""" + return [ + call.args[1] for call in mock_track.call_args_list if call.args and call.args[0] == "partner_nodes_detected" + ] + + def _no_credentials(self, monkeypatch): + monkeypatch.delenv("COMFY_CLOUD_API_KEY", raising=False) + from comfy_cli.auth import store as auth_store + + monkeypatch.setattr(auth_store, "get", lambda _: None) + monkeypatch.setattr(auth_store, "get_cloud_session", lambda: None) + + def test_fires_with_credential_present_when_api_key_supplied(self, tmp_path, monkeypatch): + wf_file = self._wf_file(tmp_path) + self._no_credentials(monkeypatch) + + with ( + patch("comfy_cli.command.run.check_comfy_server_running", return_value=True), + patch("comfy_cli.command.run._fetch_object_info", return_value=self.OBJECT_INFO), + patch("comfy_cli.command.run.ExecutionProgress"), + patch("comfy_cli.command.run.WorkflowExecution") as MockExec, + patch("comfy_cli.tracking.track_event") as mock_track, + ): + mock_exec = MagicMock() + MockExec.return_value = mock_exec + mock_exec.outputs = [] + execute(wf_file, host="127.0.0.1", port=8188, wait=True, timeout=30, api_key="k", allow_spend=True) + + events = self._partner_events(mock_track) + assert len(events) == 1 + assert events[0] == { + "partner_nodes": ["SomePartnerNode"], + "partner_node_count": 1, + "where": "local", + "credential_present": True, + } + + def test_fires_with_credential_present_when_env_key_available(self, tmp_path, monkeypatch): + wf_file = self._wf_file(tmp_path) + monkeypatch.setenv("COMFY_CLOUD_API_KEY", "test-key-abc") + from comfy_cli.auth import store as auth_store + + monkeypatch.setattr(auth_store, "get", lambda _: None) + + with ( + patch("comfy_cli.command.run.check_comfy_server_running", return_value=True), + patch("comfy_cli.command.run._fetch_object_info", return_value=self.OBJECT_INFO), + patch("comfy_cli.command.run.ExecutionProgress"), + patch("comfy_cli.command.run.WorkflowExecution") as MockExec, + patch("comfy_cli.tracking.track_event") as mock_track, + ): + mock_exec = MagicMock() + MockExec.return_value = mock_exec + mock_exec.outputs = [] + execute(wf_file, host="127.0.0.1", port=8188, wait=True, timeout=30, allow_spend=True) + + events = self._partner_events(mock_track) + assert len(events) == 1 + assert events[0]["credential_present"] is True + assert events[0]["partner_nodes"] == ["SomePartnerNode"] + + def test_does_not_fire_for_partner_free_workflow(self, workflow_file): + with ( + patch("comfy_cli.command.run.check_comfy_server_running", return_value=True), + patch( + "comfy_cli.command.run._fetch_object_info", + return_value={ + "EmptyLatentImage": {"category": "latent", "output": ["LATENT"], "output_name": ["LATENT"]}, + "PreviewAny": {"category": "image", "output": [], "output_name": [], "output_node": True}, + }, + ), + patch("comfy_cli.command.run.ExecutionProgress"), + patch("comfy_cli.command.run.WorkflowExecution") as MockExec, + patch("comfy_cli.tracking.track_event") as mock_track, + ): + mock_exec = MagicMock() + MockExec.return_value = mock_exec + mock_exec.outputs = [] + execute(workflow_file, host="127.0.0.1", port=8188, wait=True, timeout=30) + + assert self._partner_events(mock_track) == [] + + def test_fires_even_when_run_is_rejected_for_missing_credential(self, tmp_path, monkeypatch): + """The rejected-for-missing-credential funnel is what this metric is + for — the event must precede the error branch, with + ``credential_present: False`` marking those runs.""" + wf_file = self._wf_file(tmp_path) + self._no_credentials(monkeypatch) + + with ( + patch("comfy_cli.command.run.check_comfy_server_running", return_value=True), + patch("comfy_cli.command.run._fetch_object_info", return_value=self.OBJECT_INFO), + patch("comfy_cli.command.run.WorkflowExecution") as MockExec, + patch("comfy_cli.tracking.track_event") as mock_track, + ): + with pytest.raises(typer.Exit) as exc_info: + execute(wf_file, host="127.0.0.1", port=8188, wait=True, timeout=30, allow_spend=True) + assert exc_info.value.exit_code == 1 + MockExec.assert_not_called() + + events = self._partner_events(mock_track) + assert len(events) == 1 + assert events[0]["credential_present"] is False + assert events[0]["partner_node_count"] == 1 + + def test_does_not_fire_when_the_spend_gate_refuses(self, tmp_path, monkeypatch): + """Documents the one funnel this event does NOT cover: the BE-4326 spend + gate refuses before any credential resolution (so a refusal never + triggers a network OAuth refresh), and ``credential_present`` depends on + that resolution — so a run declined for lack of ``--allow-spend`` emits + nothing. That funnel needs its own event, not an early resolve here.""" + wf_file = self._wf_file(tmp_path) + self._no_credentials(monkeypatch) + monkeypatch.setattr("comfy_cli.command.run.sys.stdin.isatty", lambda: False, raising=False) + + with ( + patch("comfy_cli.command.run.check_comfy_server_running", return_value=True), + patch("comfy_cli.command.run._fetch_object_info", return_value=self.OBJECT_INFO), + patch("comfy_cli.command.run._resolve_partner_credential") as MockCred, + patch("comfy_cli.command.run.WorkflowExecution"), + patch("comfy_cli.tracking.track_event") as mock_track, + ): + with pytest.raises(typer.Exit): + execute(wf_file, host="127.0.0.1", port=8188, wait=True, timeout=30) + MockCred.assert_not_called() + + assert self._partner_events(mock_track) == [] + + def test_partner_nodes_list_is_capped_but_count_is_exact(self, tmp_path, monkeypatch): + """A pathological graph must not ship an unbounded property; the count + stays exact so the cap never distorts the metric.""" + workflow = {str(i): {"class_type": f"PartnerNode{i:02d}", "inputs": {}} for i in range(30)} + object_info = {f"PartnerNode{i:02d}": {"category": "image", "api_node": True} for i in range(30)} + wf_file = self._wf_file(tmp_path, workflow) + self._no_credentials(monkeypatch) + + with ( + patch("comfy_cli.command.run.check_comfy_server_running", return_value=True), + patch("comfy_cli.command.run._fetch_object_info", return_value=object_info), + patch("comfy_cli.command.run.WorkflowExecution"), + patch("comfy_cli.tracking.track_event") as mock_track, + ): + with pytest.raises(typer.Exit): + execute(wf_file, host="127.0.0.1", port=8188, wait=True, timeout=30, allow_spend=True) + + events = self._partner_events(mock_track) + assert len(events) == 1 + assert events[0]["partner_node_count"] == 30 + assert events[0]["partner_nodes"] == [f"PartnerNode{i:02d}" for i in range(20)] + + class TestExecuteSpendGate: """`comfy run` gates partner-API (paid) workflows on `--allow-spend` (BE-4326), mirroring `comfy run-template`'s spend gate. A partner-node diff --git a/tests/comfy_cli/test_tracking.py b/tests/comfy_cli/test_tracking.py index b79319ca..271556f2 100644 --- a/tests/comfy_cli/test_tracking.py +++ b/tests/comfy_cli/test_tracking.py @@ -1,3 +1,4 @@ +import os from unittest.mock import MagicMock, patch import pytest @@ -80,7 +81,7 @@ def test_properties_default_to_empty_dict(self, tracking_module): tracking_module.track_event("some_event") tracking_module.provider.track.assert_called_once() _, _, properties = _last_track_call(tracking_module.provider) - assert set(properties.keys()) == {"cli_version", "tracing_id"} + assert set(properties.keys()) == {"cli_version", "tracing_id", "caller_kind"} def test_swallows_provider_errors(self, tracking_module): tracking_module.config_manager.set(constants.CONFIG_KEY_ENABLE_TRACKING, "True") @@ -89,6 +90,49 @@ def test_swallows_provider_errors(self, tracking_module): tracking_module.provider.track.assert_called_once() +class TestCallerKindEnrichment: + """``_dispatch`` stamps every event with the caller kind (human vs agent), + the same way it stamps cli_version/tracing_id — that is what makes + agent-vs-human analytics possible across the whole event stream.""" + + def test_track_event_carries_caller_kind(self, tracking_module): + tracking_module.config_manager.set(constants.CONFIG_KEY_ENABLE_TRACKING, "True") + tracking_module.track_event("some_event", {"k": "v"}) + _, _, properties = _last_track_call(tracking_module.provider) + assert properties["caller_kind"] == tracking_module._caller_kind + assert isinstance(properties["caller_kind"], str) and properties["caller_kind"] + + @pytest.mark.skipif( + bool(os.environ.get("COMFY_USER_AGENT")), + reason="a custom COMFY_USER_AGENT label legitimately replaces the intrinsic kinds", + ) + def test_caller_kind_is_one_of_the_known_kinds_by_default(self, tracking_module): + # Without a COMFY_USER_AGENT override the module-scope value must be one + # of the four intrinsic kinds detect_caller() can return. + assert tracking_module._caller_kind in {"user", "pipe", "agent", "claude-code"} + + def test_feedback_carries_caller_kind(self, tracking_module): + # Feedback rides the same _dispatch path, so it is enriched too. + tracking_module.submit_feedback("nice tool") + _, _, properties = _last_track_call(tracking_module.provider) + assert properties["caller_kind"] == tracking_module._caller_kind + + def test_explicit_user_agent_label_flows_through(self, tracking_module): + """An explicit ``COMFY_USER_AGENT`` label reaches the provider verbatim + (lowercased by detect_caller). Patched onto the module because + ``_caller_kind`` is evaluated once at import, not per event.""" + from comfy_cli.caller import detect_caller + + kind = detect_caller(env={"COMFY_USER_AGENT": "My-Harness"}, is_tty=True).kind + assert kind == "my-harness" + + tracking_module.config_manager.set(constants.CONFIG_KEY_ENABLE_TRACKING, "True") + with patch.object(tracking_module, "_caller_kind", kind): + tracking_module.track_event("some_event") + _, _, properties = _last_track_call(tracking_module.provider) + assert properties["caller_kind"] == "my-harness" + + class TestSubmitFeedback: def test_sends_even_when_passive_consent_disabled(self, tracking_module): # Feedback is explicit/user-initiated: it ignores the passive-telemetry From 453d3e1623f739b5490703cc2d77d94c52459ef2 Mon Sep 17 00:00:00 2001 From: Matt Miller Date: Sat, 1 Aug 2026 17:30:40 -0700 Subject: [PATCH 2/4] fix(telemetry): guard stdout probe + bound caller_kind and partner-node names (BE-5633) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the Cursor review panel findings on #647. - caller: `detect_caller()` called `sys.stdout.isatty()` unguarded, but `_caller_kind` now evaluates it at `comfy_cli.tracking` import — which happens during CLI startup for every command. Under pythonw/detached processes `sys.stdout` is None (AttributeError) or already closed (ValueError), so a telemetry detail became an import-time crash for even `comfy --help` with tracking disabled. Guarded in `detect_caller` itself rather than at the tracking call site, so the pre-existing renderer path (`output/renderer.py`) is fixed by the same change. Falls through to kind="pipe" — a process with no usable stdout is by definition not a human at a terminal. Mirrors `_stdin_is_interactive`. - tracking: `caller_kind` can be an arbitrary `COMFY_USER_AGENT` label that detect_caller only lowercases, and it now rides every event, including `feedback_submitted` which dispatches even when passive telemetry consent is off. Scrub URL query strings and cap to 64 chars before it ships, the same treatment command kwargs get. - run: `partner_nodes[:20]` bounded the element count but not each element, and class_type strings come verbatim from untrusted workflow JSON — one multi-megabyte class name still shipped whole. Cap each name to 64 chars; `partner_node_count` stays exact. The same cap now bounds the node list echoed in the missing-credential error prose, with an "and N more" suffix; `details.partner_nodes` stays complete since that is the machine-readable field JSON consumers read. Co-Authored-By: Claude Opus 5 --- comfy_cli/caller.py | 28 +++++++++++- comfy_cli/command/run/__init__.py | 24 +++++++++-- comfy_cli/tracking.py | 23 +++++++++- tests/comfy_cli/command/test_run.py | 62 +++++++++++++++++++++++++++ tests/comfy_cli/output/test_caller.py | 45 +++++++++++++++++++ tests/comfy_cli/test_tracking.py | 34 +++++++++++++++ 6 files changed, 210 insertions(+), 6 deletions(-) diff --git a/comfy_cli/caller.py b/comfy_cli/caller.py index 0b8c4da9..37d6dbe5 100644 --- a/comfy_cli/caller.py +++ b/comfy_cli/caller.py @@ -9,7 +9,7 @@ 1. ``COMFY_USER_AGENT=