From 54f8564b7df227ea7d3890348444559af921be51 Mon Sep 17 00:00:00 2001 From: stephamie7 <1223696150@qq.com> Date: Thu, 10 Sep 2026 16:17:36 +0800 Subject: [PATCH 1/7] fix(project): tolerate unavailable historical worktrees --- flocks/project/project.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flocks/project/project.py b/flocks/project/project.py index 37ca39158..41922fc6a 100644 --- a/flocks/project/project.py +++ b/flocks/project/project.py @@ -299,7 +299,16 @@ def registry_path(cls, owner_id: str) -> Path: @staticmethod def _normalized_worktree(worktree: str) -> str: - return os.path.normcase(str(Path(worktree).expanduser().resolve(strict=True))) + """Compare registry paths even after their directories become unavailable.""" + + path = Path(worktree).expanduser() + try: + path = path.resolve(strict=False) + except (OSError, RuntimeError): + # Unreadable paths and symlink loops must not block other projects. + # New/restored worktrees are still checked by validate_worktree. + path = Path(os.path.abspath(path)) + return os.path.normcase(str(path)) @staticmethod def _directory_context(directory: str) -> Tuple[Path, Path, Optional[str]]: From fe5e7f562183fc635abc16dc2e63ff67d7601936 Mon Sep 17 00:00:00 2001 From: John Yin <10972267+john-yin2333@user.noreply.gitee.com> Date: Fri, 11 Sep 2026 14:52:47 +0800 Subject: [PATCH 2/7] Revert "Merge pull request #738 from AgentFlocks/codex/soc-dashboard-data-integrity" This reverts commit 4321667a3a52b16940390206bf4f165cda5f91ad, reversing changes made to fcaf9a6381022aca54137510165f6cd2fa38ac5a. --- .../soc_ui/soc_dashboard/api/handlers.py | 833 ++---------------- .../soc_ui/soc_dashboard/api/routes.yaml | 5 - .../webuis/soc_ui/soc_dashboard/src/Page.tsx | 511 +++-------- .../soc_dashboard/src/severityValues.ts | 15 +- .../stream_alert_denoise/workflow.json | 2 +- flocks/workflow/store.py | 320 ------- tests/hub/test_soc_dashboard_schema.py | 698 --------------- tests/workflow/test_workflow_store.py | 364 -------- .../utils/socDashboardPageRuntime.test.tsx | 218 ----- 9 files changed, 181 insertions(+), 2785 deletions(-) diff --git a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/handlers.py b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/handlers.py index 69627de44..240bd4286 100644 --- a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/handlers.py +++ b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/handlers.py @@ -32,7 +32,6 @@ WORKFLOW_DB = Path.home() / ".flocks" / "data" / "workflow.db" WORKFLOW_SNAPSHOT_TABLE = "soc_dashboard_workflow_stats_samples" -WORKFLOW_METRIC_ROLLUP_SCHEMA_VERSION = 3 TASK_DB = Path.home() / ".flocks" / "data" / "tasks.db" USAGE_DB = Path.home() / ".flocks" / "data" / "flocks.db" SOC_PINNED_WORKFLOW_NAMES = { @@ -905,194 +904,9 @@ def _empty_workflow_denoise_stats(): "seriesUnique": [], "timelineLabels": [], "timelineWindow": "", - "metricsAvailable": False, - "dataAvailable": True, - "dataQuality": "legacy", - "unavailableReason": "", - "coverageComplete": False, - "coverageStartedAt": 0, - "sourceCoverageRate": 0, - "sourceMetricsAvailable": False, - "invalidExecutionCount": 0, - "unprocessedInputCount": 0, - "dataSource": "workflow.db.workflow_stats.call_count", } -def _unavailable_workflow_denoise_stats(reason): - result = { - **_empty_workflow_denoise_stats(), - "dataAvailable": False, - "dataQuality": "unavailable", - "unavailableReason": str(reason or "workflow_metrics_unavailable"), - "dataSource": "unavailable", - } - for key in ( - "callCount", - "successCount", - "errorCount", - "earliestStartedAt", - "latestStartedAt", - "rawCount", - "normalizedCount", - "afterFilterCount", - "uniqueCount", - "filterRemovedCount", - "duplicateCount", - "reducedCount", - "reductionRate", - "dedupRate", - "sourceCoverageRate", - "invalidExecutionCount", - "unprocessedInputCount", - ): - result[key] = None - return result - - -def _get_workflow_metric_rollups(workflow_name, start_time, end_time): - if not WORKFLOW_DB.is_file(): - return None - start_ms = max(_safe_int(start_time), 0) * 1000 - end_ms = max(_safe_int(end_time), 0) * 1000 - try: - with sqlite3.connect(f"file:{WORKFLOW_DB}?mode=ro", uri=True, timeout=1.0) as conn: - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA query_only = ON") - if not ( - _table_exists(conn, "workflow_metric_rollups") - and _table_exists(conn, "workflow_metric_meta") - ): - return None - meta = conn.execute( - "SELECT coverage_started_at, updated_at FROM workflow_metric_meta " - "WHERE workflow_id = ?", - (workflow_name,), - ).fetchone() - if meta is None: - return None - verified_row = conn.execute( - "SELECT MIN(bucket_start) FROM workflow_metric_rollups " - "WHERE workflow_id = ? AND schema_version >= ?", - (workflow_name, WORKFLOW_METRIC_ROLLUP_SCHEMA_VERSION), - ).fetchone() - verified_started_at = _safe_int(verified_row[0] if verified_row else 0) - if verified_started_at <= 0: - # Existing v2 rows predate input/output reconciliation and may - # contain false zero ingress counts. Keep them out of the exact - # path until a v3 contribution establishes verified coverage. - return None - earliest_execution_ms = 0 - if _table_exists(conn, "workflow_executions"): - earliest_execution_row = conn.execute( - "SELECT MIN(started_at) FROM workflow_executions WHERE workflow_id = ?", - (workflow_name,), - ).fetchone() - earliest_execution_ms = _safe_int( - earliest_execution_row[0] if earliest_execution_row else 0 - ) - query = ( - "SELECT * FROM workflow_metric_rollups " - "WHERE workflow_id = ? AND schema_version >= ?" - ) - query_params = [workflow_name, WORKFLOW_METRIC_ROLLUP_SCHEMA_VERSION] - if start_ms > 0 and end_ms > 0: - query += " AND bucket_start >= ? AND bucket_start <= ?" - query_params.extend((start_ms - (start_ms % 60000), end_ms)) - query += " ORDER BY bucket_start" - rows = conn.execute(query, query_params).fetchall() - except Exception: - return None - - result = _empty_workflow_denoise_stats() - source_counts = Counter() - for row in rows: - parsed_sources = _safe_json_object(row["source_counts"]) - if isinstance(parsed_sources, dict): - for key, value in parsed_sources.items(): - source_counts[_norm(key)] += max(_safe_int(value), 0) - raw_count = sum(max(_safe_int(row["raw_count"]), 0) for row in rows) - normalized_count = sum(max(_safe_int(row["normalized_count"]), 0) for row in rows) - after_filter_count = sum(max(_safe_int(row["after_filter_count"]), 0) for row in rows) - unique_count = sum(max(_safe_int(row["unique_count"]), 0) for row in rows) - filter_removed_count = sum(max(_safe_int(row["filter_removed_count"]), 0) for row in rows) - duplicate_count = sum(max(_safe_int(row["duplicate_count"]), 0) for row in rows) - success_count = sum(max(_safe_int(row["success_count"]), 0) for row in rows) - error_count = sum(max(_safe_int(row["error_count"]), 0) for row in rows) - invalid_count = sum(max(_safe_int(row["invalid_count"]), 0) for row in rows) - source_covered_count = sum(max(_safe_int(row["source_covered_count"]), 0) for row in rows) - coverage_started_at = max( - _safe_int(meta["coverage_started_at"]), - verified_started_at, - 0, - ) - requested_start_ms = start_ms or earliest_execution_ms - # An unbounded query is complete only when execution history proves that - # verified rollups cover the workflow's first execution. Without that - # lower bound, treating recent v3 rows as all-time history would silently - # undercount older executions. - complete_window = requested_start_ms > 0 and coverage_started_at <= requested_start_ms - source_complete = source_covered_count == raw_count - metrics_complete = invalid_count == 0 and error_count == 0 - quality = ( - "complete" - if complete_window and metrics_complete and source_complete - else "partial" - ) - - first_bucket = _safe_int(rows[0]["bucket_start"]) if rows else start_ms - last_bucket = _safe_int(rows[-1]["bucket_start"]) if rows else end_ms - bucket_start, bucket_seconds, bucket_count, labels, window = _timeline_spec( - [], - start_time or first_bucket // 1000, - end_time or max(last_bucket // 1000, first_bucket // 1000), - ) - series_raw = [0] * bucket_count - series_unique = [0] * bucket_count - for row in rows: - index = int(((_safe_int(row["bucket_start"]) // 1000) - bucket_start) / bucket_seconds) - if 0 <= index < bucket_count: - series_raw[index] += max(_safe_int(row["raw_count"]), 0) - series_unique[index] += max(_safe_int(row["unique_count"]), 0) - - result.update( - { - "callCount": success_count + error_count, - "successCount": success_count, - "errorCount": error_count, - "earliestStartedAt": first_bucket, - "latestStartedAt": last_bucket, - "rawCount": raw_count, - "normalizedCount": normalized_count, - "afterFilterCount": after_filter_count, - "uniqueCount": unique_count, - "filterRemovedCount": filter_removed_count, - "duplicateCount": duplicate_count, - "reducedCount": max(raw_count - unique_count, 0), - "reductionRate": _ratio(max(raw_count - unique_count, 0), raw_count), - "dedupRate": _ratio(duplicate_count, after_filter_count), - "sourceCounts": dict(source_counts), - "seriesRaw": series_raw, - "seriesUnique": series_unique, - "timelineLabels": labels, - "timelineWindow": window, - "metricsAvailable": metrics_complete, - "sourceMetricsAvailable": source_complete, - "dataQuality": quality, - "coverageComplete": complete_window, - "coverageStartedAt": coverage_started_at, - # Sources describe ingress volume on the dashboard. Use the raw - # denominator so failed/invalid executions cannot silently vanish - # from coverage while still contributing to the raw total. - "sourceCoverageRate": _ratio(min(source_covered_count, raw_count), raw_count), - "invalidExecutionCount": invalid_count, - "unprocessedInputCount": max(raw_count - normalized_count, 0), - "dataSource": "workflow.db.workflow_metric_rollups", - } - ) - return result - - def _get_workflow_denoise_stats( workflow_name: str, start_time: int = 0, @@ -1103,25 +917,15 @@ def _get_workflow_denoise_stats( now = time.time() cache_key = f"denoise:{workflow_name}:{start_time or 0}:{end_time or 0}" - # Source liveness is part of the quality contract. Do not let a fresh - # process-local cache make a disappeared database look healthy. - if not WORKFLOW_DB.is_file(): - return _unavailable_workflow_denoise_stats("workflow_db_missing") - with _cache_lock: cached = _workflow_stats_cache.get(cache_key) if not force and cached and now - float(cached.get("updatedAt") or 0) < _CACHE_TTL: _workflow_stats_cache.move_to_end(cache_key) return cached["value"] - rollup_result = _get_workflow_metric_rollups(workflow_name, start_time, end_time) - if rollup_result is not None and rollup_result.get("coverageComplete"): - with _cache_lock: - _workflow_stats_cache[cache_key] = {"updatedAt": now, "value": rollup_result} - _workflow_stats_cache.move_to_end(cache_key) - while len(_workflow_stats_cache) > _WORKFLOW_CACHE_MAX: - _workflow_stats_cache.popitem(last=False) - return rollup_result + empty = _empty_workflow_denoise_stats() + if not WORKFLOW_DB.is_file(): + return empty try: with sqlite3.connect(WORKFLOW_DB) as conn: @@ -1186,22 +990,6 @@ def _get_workflow_denoise_stats( result_dict["timelineLabels"] = labels result_dict["timelineWindow"] = window - if rollup_result is not None: - # A newly-created rollup cannot represent the part of a requested - # window that predates metric collection. Keep the legacy values - # visible until the selected window is fully covered, and expose - # the coverage state so the UI does not present them as exact. - result_dict.update( - { - "dataQuality": "legacy-partial", - "coverageComplete": False, - "coverageStartedAt": rollup_result.get("coverageStartedAt", 0), - "invalidExecutionCount": rollup_result.get("invalidExecutionCount", 0), - "sourceCoverageRate": rollup_result.get("sourceCoverageRate", 0), - "shadowMetricsAvailable": True, - } - ) - with _cache_lock: _workflow_stats_cache[cache_key] = {"updatedAt": now, "value": result_dict} _workflow_stats_cache.move_to_end(cache_key) @@ -1211,13 +999,7 @@ def _get_workflow_denoise_stats( except Exception: with _cache_lock: cached = _workflow_stats_cache.get(cache_key) - if cached: - return { - **cached["value"], - "dataQuality": "stale", - "unavailableReason": "workflow_db_query_failed", - } - return _unavailable_workflow_denoise_stats("workflow_db_query_failed") + return cached["value"] if cached else empty def _get_workflow_progress( @@ -1468,10 +1250,6 @@ async def get_task_center(ctx, request): return await asyncio.to_thread(_get_task_center, include_mock) -async def get_ai_tasks(ctx, request): - return await asyncio.to_thread(_get_ai_tasks) - - def _table_exists(conn, table_name): return bool( conn.execute( @@ -2134,340 +1912,6 @@ def _get_task_center(include_mock=False): } -def _workflow_task_metric(stats, key): - if key not in stats: - return None, "missing" - value = stats.get(key) - if isinstance(value, bool): - return None, "invalid" - try: - parsed = int(value) - except (TypeError, ValueError, OverflowError): - return None, "invalid" - if parsed < 0: - return None, "invalid" - return parsed, "complete" - - -def _workflow_task_metrics(output_text, status): - output = _safe_json_object(output_text) - stats = output.get("stats") if isinstance(output.get("stats"), dict) else {} - values = {} - qualities = [] - for field, key in ( - ("raw", "raw_count"), - ("normalized", "normalized_count"), - ("afterFilter", "after_filter_count"), - ("unique", "after_dedup_count"), - ): - values[field], quality = _workflow_task_metric(stats, key) - qualities.append(quality) - if all(quality == "complete" for quality in qualities): - ordered = ( - values["raw"], - values["normalized"], - values["afterFilter"], - values["unique"], - ) - if ordered[0] >= ordered[1] >= ordered[2] >= ordered[3] >= 0: - quality = "complete" - else: - # Running executions may expose node-local intermediate output. - # Treat an impossible stage order as pending until completion; - # persisted finished output with the same shape is invalid. - quality = "pending" if status in WORKFLOW_RUNNING_STATUSES else "invalid" - elif "invalid" in qualities: - quality = "invalid" - elif status in WORKFLOW_RUNNING_STATUSES: - quality = "pending" - else: - quality = "missing" - return values, quality - - -def _workflow_task_input_count(inputs): - # Match stream_alert_denoise's receive node exactly: a decodable syslog - # payload wins over every batch field. - value = inputs.get("syslog_message") or inputs.get("syslog") - if isinstance(value, dict) and value.get("message"): - try: - parsed_syslog = json.loads(str(value["message"])) - except (TypeError, ValueError, json.JSONDecodeError): - pass - else: - if isinstance(parsed_syslog, dict): - return 1 - - def sequence_count(value): - if isinstance(value, str): - try: - value = json.loads(value) - except (TypeError, ValueError, json.JSONDecodeError): - return None - if isinstance(value, list): - return len(value) - if isinstance(value, dict): - if value.get("_type") in {"list", "tuple", "set"}: - count, quality = _workflow_task_metric(value, "count") - return count if quality == "complete" else None - if "data" in value: - value = value.get("data") - return len(value) if isinstance(value, list) else (1 if value else 0) - return 1 if value else 0 - - # `alerts` shadows `alert_list` even when it is empty, matching - # inputs.get('alerts', inputs.get('alert_list', [])) in the workflow. - for key in ("alerts", "alert_list"): - marker_key = f"_{key}_count" - if key not in inputs and marker_key not in inputs: - continue - materialized_count = sequence_count(inputs.get(key)) if key in inputs else None - if materialized_count is not None: - return materialized_count - marker, quality = _workflow_task_metric(inputs, marker_key) - if quality == "complete": - return marker - return None - - # Compatibility fallbacks for older/compacted execution rows. They are - # considered only when no canonical workflow input is present. - for marker_key in ("_raw_alerts_count", "raw_count"): - marker, quality = _workflow_task_metric(inputs, marker_key) - if quality == "complete": - return marker - if "raw_alerts" in inputs: - return sequence_count(inputs.get("raw_alerts")) - if inputs.get("alert") not in (None, "", {}): - return 1 - return None - - -def _workflow_task_row(row, workflow_id, effective_status): - output_text = row["output_results"] - input_text = row["input_params"] - output = _safe_json_object(output_text) - inputs = _safe_json_object(input_text) - metrics = _workflow_execution_metrics(output_text, input_text) - counts, data_quality = _workflow_task_metrics(output_text, effective_status) - input_count = _workflow_task_input_count(inputs) - output_raw_count = counts["raw"] - raw_count_source = "workflow_output" if output_raw_count is not None else "pending" - if input_count is not None: - counts["raw"] = input_count - raw_count_source = "workflow_input" - if output_raw_count is not None and output_raw_count != input_count: - data_quality = ( - "pending" if effective_status in WORKFLOW_RUNNING_STATUSES else "invalid" - ) - elif input_count == 0 and data_quality in {"complete", "pending", "missing"}: - data_quality = "empty-input" - elif effective_status in WORKFLOW_RUNNING_STATUSES and output_raw_count == 0: - # A zero-initialized output is not proof that an active task received - # no alerts. Keep it unknown until input or completed output verifies it. - counts["raw"] = None - raw_count_source = "pending" - data_quality = "pending" - preview = metrics["preview"] - stage = "triage" if workflow_id in TRIAGE_WORKFLOW_IDS else "denoise" - title = _workflow_latest_alert_name(workflow_id, output_text, input_text) - if stage == "denoise" and not preview: - raw_count = counts["raw"] - title = ( - "降噪批次 · 空输入" - if data_quality == "empty-input" - else f"降噪批次 · 原始 {raw_count} 条" - if raw_count is not None - else "降噪批次 · 原始条数待生成" - ) - session_id, message_id = _workflow_link_context(row) - total_steps = max(_workflow_node_count(workflow_id), _safe_int(row["step_count"])) - current_step = max(_safe_int(row["current_step_index"]), 0) - if effective_status == "running" and total_steps > 0: - progress = { - "mode": "steps", - "current": min(max(current_step, 1), total_steps), - "total": total_steps, - "percent": _ratio(min(max(current_step, 1), total_steps), total_steps), - "label": f"第 {min(max(current_step, 1), total_steps)}/{total_steps} 步", - } - else: - progress = { - "mode": "waiting" if effective_status in {"queued", "pending"} else "none", - "current": current_step, - "total": total_steps, - "percent": None, - "label": "等待调度" if effective_status in {"queued", "pending"} else "", - } - source_type = metrics["sourceType"] - return { - "taskId": f"workflow-execution:{row['id']}", - "workflowId": workflow_id, - "executionId": str(row["id"]), - "stage": stage, - "status": effective_status, - "startedAt": _safe_int(row["started_at"]), - "updatedAt": _safe_int(row["updated_at"]), - "finishedAt": _safe_int(row["finished_at"]), - "currentPhase": str(row["current_phase"] or ""), - "title": title, - "sourceType": source_type, - "srcIp": preview.get("sip") or preview.get("src_ip") or preview.get("net_real_src_ip"), - "dstIp": preview.get("dip") or preview.get("dst_ip") or preview.get("net_dest_ip"), - "counts": counts, - "dataQuality": data_quality, - "rawCountSource": raw_count_source, - "emptyBatch": data_quality == "empty-input", - "emptyInput": data_quality == "empty-input", - "progress": progress, - "sessionId": session_id, - "messageId": message_id, - "error": str(row["error_message"] or ""), - "inputMode": str(output.get("input_mode") or inputs.get("input_mode") or ""), - } - - -def _get_ai_tasks(): - empty_summary = { - "active": 0, - "running": 0, - "waiting": 0, - "stale": 0, - "disabled": 0, - "returned": 0, - "truncated": False, - } - if not WORKFLOW_DB.is_file(): - return { - "generatedAt": datetime.now().isoformat(timespec="seconds"), - "connection": "unavailable", - "reason": "workflow_db_missing", - "summary": empty_summary, - "tasks": [], - } - workflow_ids = tuple(SOC_PINNED_WORKFLOW_NAMES) - try: - with sqlite3.connect(f"file:{WORKFLOW_DB}?mode=ro", uri=True, timeout=1.0) as conn: - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA query_only = ON") - if not _table_exists(conn, "workflow_executions"): - return { - "generatedAt": datetime.now().isoformat(timespec="seconds"), - "connection": "unavailable", - "reason": "workflow_executions_missing", - "summary": empty_summary, - "tasks": [], - } - columns = { - row[1] - for row in conn.execute("PRAGMA table_info(workflow_executions)").fetchall() - } - updated_expr = "updated_at" if "updated_at" in columns else "started_at" - freshness_expr = f"COALESCE(NULLIF({updated_expr}, 0), started_at)" - latest_select = ", ".join( - [ - "id", - "workflow_id", - "status", - "started_at", - _workflow_execution_column_expr(columns, "finished_at", "0"), - _workflow_execution_column_expr(columns, "updated_at", "0"), - _workflow_execution_column_expr(columns, "current_phase", "''"), - _workflow_execution_column_expr(columns, "current_step_index", "0"), - _workflow_execution_column_expr(columns, "step_count", "0"), - _workflow_execution_column_expr(columns, "output_results", "'{}'"), - _workflow_execution_column_expr(columns, "input_params", "'{}'"), - _workflow_execution_column_expr(columns, "payload", "'{}'"), - _workflow_execution_column_expr(columns, "error_message", "''"), - ] - ) - now_ms = int(time.time() * 1000) - tasks = [] - summary = dict(empty_summary) - for workflow_id in workflow_ids: - trigger_state = _workflow_trigger_state(conn, workflow_id) - cutoff = now_ms - trigger_state["timeoutSeconds"] * 1000 - if trigger_state["hasConfig"] and not trigger_state["enabled"]: - summary["disabled"] += _safe_int( - conn.execute( - "SELECT COUNT(*) FROM workflow_executions " - "WHERE workflow_id = ? AND status IN ('running', 'queued', 'pending')", - (workflow_id,), - ).fetchone()[0] - ) - continue - active_count = _safe_int( - conn.execute( - "SELECT COUNT(*) FROM workflow_executions " - "WHERE workflow_id = ? AND status IN ('running', 'queued', 'pending') " - f"AND {freshness_expr} >= ?", - (workflow_id, cutoff), - ).fetchone()[0] - ) - running_count = _safe_int( - conn.execute( - "SELECT COUNT(*) FROM workflow_executions " - "WHERE workflow_id = ? AND status = 'running' " - f"AND {freshness_expr} >= ?", - (workflow_id, cutoff), - ).fetchone()[0] - ) - waiting_count = _safe_int( - conn.execute( - "SELECT COUNT(*) FROM workflow_executions " - "WHERE workflow_id = ? AND status IN ('queued', 'pending') " - f"AND {freshness_expr} >= ?", - (workflow_id, cutoff), - ).fetchone()[0] - ) - stale_count = _safe_int( - conn.execute( - "SELECT COUNT(*) FROM workflow_executions " - "WHERE workflow_id = ? AND status IN ('running', 'queued', 'pending') " - f"AND {freshness_expr} < ?", - (workflow_id, cutoff), - ).fetchone()[0] - ) - summary["active"] += active_count - summary["running"] += running_count - summary["waiting"] += waiting_count - summary["stale"] += stale_count - rows = conn.execute( - f"SELECT {latest_select} FROM workflow_executions " - "WHERE workflow_id = ? AND status IN ('running', 'queued', 'pending') " - f"AND {freshness_expr} >= ? " - f"ORDER BY CASE WHEN status = 'running' THEN 0 ELSE 1 END, {freshness_expr} DESC " - "LIMIT 50", - (workflow_id, cutoff), - ).fetchall() - for row in rows: - effective_status = str(row["status"] or "").lower() - tasks.append(_workflow_task_row(row, workflow_id, effective_status)) - tasks.sort( - key=lambda item: ( - 0 if item["status"] == "running" else 1, - -max(item["updatedAt"], item["startedAt"]), - ) - ) - tasks = tasks[:50] - summary["returned"] = len(tasks) - summary["truncated"] = summary["active"] > len(tasks) - return { - "generatedAt": datetime.now().isoformat(timespec="seconds"), - "connection": "online", - "reason": "", - "summary": summary, - "tasks": tasks, - } - except Exception as exc: - return { - "generatedAt": datetime.now().isoformat(timespec="seconds"), - "connection": "error", - "reason": str(exc), - "summary": empty_summary, - "tasks": [], - } - - def _get_activity(params): _ensure_sqlite_schema() _maybe_prune_activity() @@ -2542,13 +1986,6 @@ def _get_activity(params): last_row_id = max(_safe_int(cursor.get("lastRowId")), 0) last_activity_id = max(_safe_int(cursor.get("lastActivityId")), 0) - previous_polled_at = max(_safe_int(cursor.get("polledAt")), 0) - current_polled_at = int(time.time() * 1000) - poll_window_ms = ( - max(current_polled_at - previous_polled_at, 1) - if previous_polled_at - else ACTIVITY_WINDOW_MS - ) if last_row_id > latest_row_id or last_activity_id > latest_activity_id: last_row_id = 0 last_activity_id = 0 @@ -2561,7 +1998,6 @@ def _get_activity(params): latest_row_id=latest_row_id, latest_activity_id=latest_activity_id, limit=limit, - window_ms=poll_window_ms, ) except Exception as exc: return { @@ -2609,11 +2045,9 @@ def _activity_response( workflow_stats=None, workflow_events=None, ): - generated_at = datetime.now().astimezone() - polled_at = int(generated_at.timestamp() * 1000) return { - "cursor": _encode_activity_cursor(last_row_id, last_activity_id, polled_at), - "generatedAt": generated_at.isoformat(timespec="seconds"), + "cursor": _encode_activity_cursor(last_row_id, last_activity_id), + "generatedAt": datetime.now().astimezone().isoformat(timespec="seconds"), "events": events, "recentEvents": recent_events or [], "overflowCount": 0, @@ -2640,12 +2074,11 @@ def _empty_activity_batch(): } -def _encode_activity_cursor(last_row_id, last_activity_id, polled_at=None): +def _encode_activity_cursor(last_row_id, last_activity_id): payload = json.dumps( { "lastRowId": max(_safe_int(last_row_id), 0), "lastActivityId": max(_safe_int(last_activity_id), 0), - "polledAt": max(_safe_int(polled_at), 0), }, separators=(",", ":"), ).encode("utf-8") @@ -2684,7 +2117,6 @@ def _activity_rows( latest_row_id, latest_activity_id, limit, - window_ms=ACTIVITY_WINDOW_MS, ): summary = _activity_insert_summary(conn, settings, last_row_id, latest_row_id) new_count = summary["receivedCount"] @@ -2726,11 +2158,11 @@ def _activity_rows( if new_count > ACTIVITY_SURGE_LIMIT else "burst" if new_count > ACTIVITY_NORMAL_LIMIT else "normal" ), - "windowMs": max(_safe_int(window_ms), 1), + "windowMs": ACTIVITY_WINDOW_MS, "triageUpdatedCount": updated_count, "sampledCount": sampled_count, "suppressedCount": max(new_count - sampled_count, 0), - "ratePerSecond": round(new_count / (max(_safe_int(window_ms), 1) / 1000), 1), + "ratePerSecond": round(new_count / (ACTIVITY_WINDOW_MS / 1000), 1), } return rows, max(new_count + updated_count - len(rows), 0), batch @@ -2974,13 +2406,7 @@ async def get_stats(ctx, request): def _get_stats(params): - try: - _ensure_sqlite_schema() - except Exception: - # Source quality is resolved by the read-only query below. Keeping the - # endpoint alive lets the UI distinguish an unavailable SOC database - # from a healthy database whose selected window genuinely has zero rows. - pass + _ensure_sqlite_schema() time_window = _normalize_time_window( params.get("startTime"), params.get("endTime"), @@ -3014,16 +2440,6 @@ def _get_stats(params): cache_key, _stats_cache_ttl(range_start_time, range_end_time), ) - if cached is not None: - cached_status = cached.get("sourceStatus") or {} - cached_assets = cached_status.get("assets") or {} - cached_workflow = cached_status.get("metricQuality") or {} - source_liveness_changed = bool(cached_assets.get("exists")) != _active_source_exists() - workflow_liveness_changed = bool(cached_workflow.get("dataAvailable", True)) != bool( - WORKFLOW_DB.is_file() - ) - if source_liveness_changed or workflow_liveness_changed: - cached = None if cached is not None: return { **cached, @@ -3036,12 +2452,7 @@ def _get_stats(params): denoise_files, denoise_locations = [], [] triage_files, triage_locations = [], [] - asset_files, triage_quality = _find_asset_files_with_quality( - start_date, - end_date, - start_time, - end_time, - ) + asset_files = _find_asset_files(start_date, end_date, start_time, end_time) asset_denoise_files = [path for path in asset_files if _asset_file_role(path) == "denoise"] asset_triage_files = [path for path in asset_files if _asset_file_role(path) == "triage"] sample_mode = bool(asset_denoise_files or asset_triage_files) @@ -3055,59 +2466,37 @@ def _get_stats(params): range_end_time, force=force_refresh, ) - denoise = _read_denoise(denoise_files, workflow_stats.get("callCount") or 0) + denoise = _read_denoise(denoise_files, workflow_stats.get("callCount", 0)) soc_unique_count = denoise["totalUnique"] soc_unique_series = denoise["seriesUnique"] timeline_labels = workflow_stats["timelineLabels"] or denoise.get("_timelineLabels", []) timeline_window = workflow_stats["timelineWindow"] or denoise.get("_timelineWindow", "") workflow_series_raw = workflow_stats["seriesRaw"] - metrics_available = bool(workflow_stats.get("metricsAvailable")) - if metrics_available: - processed_total = workflow_stats["rawCount"] - normalized_total = workflow_stats["normalizedCount"] - after_filter_total = workflow_stats["afterFilterCount"] - unique_total = workflow_stats["uniqueCount"] - filter_removed_count = workflow_stats["filterRemovedCount"] - duplicate_count = workflow_stats["duplicateCount"] - workflow_series_unique = workflow_stats["seriesUnique"] - else: - # Legacy/unavailable data is kept only as an internal compatibility - # fallback. The quality contract tells the UI not to present it as an - # authoritative metric; unavailable fields themselves remain null in - # sourceStatus.workflowStats. - processed_total = max(_safe_int(workflow_stats.get("callCount")), 0) - normalized_total = processed_total - after_filter_total = processed_total - unique_total = soc_unique_count - filter_removed_count = 0 - duplicate_count = max(processed_total - soc_unique_count, 0) - workflow_series_unique = soc_unique_series - if not workflow_series_raw and workflow_series_unique: - workflow_series_raw = [0] * len(workflow_series_unique) - reduced_count = max(processed_total - unique_total, 0) + if not workflow_series_raw and soc_unique_series: + workflow_series_raw = [0] * len(soc_unique_series) + processed_total = workflow_stats["callCount"] + reduced_count = max(processed_total - soc_unique_count, 0) reduction_rate = _ratio(reduced_count, processed_total) denoise.update( { "totalRaw": processed_total, - "totalNormalized": normalized_total, - "afterFilter": after_filter_total, - "totalUnique": unique_total, - "filterRemoved": filter_removed_count, - "dedupRemoved": duplicate_count, + "totalNormalized": processed_total, + "afterFilter": processed_total, + "totalUnique": soc_unique_count, + "filterRemoved": 0, + "dedupRemoved": reduced_count, "duplicates": reduced_count, "duplicateRate": reduction_rate, - "dedupRate": _ratio(duplicate_count, after_filter_total), - "uniqueRate": _ratio(min(unique_total, processed_total), processed_total), - "files": workflow_stats.get("callCount"), + "dedupRate": reduction_rate, + "uniqueRate": _ratio(min(soc_unique_count, processed_total), processed_total), + "files": processed_total, "sourceCounter": Counter(workflow_stats["sourceCounts"]), "seriesRaw": workflow_series_raw, - "seriesUnique": workflow_series_unique, + "seriesUnique": soc_unique_series, "_timelineLabels": timeline_labels, "_timelineWindow": timeline_window, - "workflowCallCount": workflow_stats["callCount"], - "socPersistedUnique": soc_unique_count, - "dataSource": workflow_stats.get("dataSource"), - "dataQuality": workflow_stats.get("dataQuality"), + "workflowCallCount": processed_total, + "dataSource": "workflow.db.workflow_stats.call_count + soc.db.unique", } ) triage = _read_triage(triage_files) @@ -3120,65 +2509,6 @@ def _get_stats(params): available_dates = _available_asset_dates() date_range = _build_date_range(start_date, end_date, asset_files, available_dates) event_range = _build_event_range(date_range, denoise, triage) - triage_payload = _without_counters(triage) - pipeline_payload = dict(pipeline) - closed_loop_payload = dict(closed_loop) - attack_profile = _build_attack_profile(denoise, triage) - verdicts = [ - {"key": "attack_success", "label": "攻击成功", "value": triage["attackSuccess"], "color": "#ff4d6d"}, - {"key": "attack", "label": "攻击行为", "value": triage["attack"], "color": "#ffb020"}, - {"key": "attack_failed", "label": "攻击失败", "value": triage["attackFailed"], "color": "#2ee6a6"}, - {"key": "non_attack", "label": "非攻击", "value": triage["benign"], "color": "#58a6ff"}, - {"key": "unknown", "label": "未知", "value": triage["unknown"], "color": "#9b8cff"}, - ] - top_threat_types = _counter_items( - triage["threatTypeCounter"] or denoise["threatTypeCounter"], - 14, - ) - severity_levels = _counter_items( - _profile_counter(denoise, triage, "severityCounter"), - 8, - ) - risk_levels = _counter_items(triage["riskCounter"], 5) - triage_series_total = triage["seriesTotal"] - triage_series_attack = triage["seriesAttack"] - if not triage_quality["metricsAvailable"]: - for key in ( - "totalRecords", "batchTotal", "newTriaged", "cacheHit", "triageFailed", - "followersReused", "attackTotal", "attackSuccess", "attack", "attackFailed", - "benign", "unknown", "attackRate", "successRate", "cacheRate", "coverageRate", - "avgTriageMs", "headers", "files", "parseErrors", - ): - triage_payload[key] = None - for key in ( - "triageTotal", "attackTotal", "llmSaved", "workloadReuseRate", - "attackRate", "successRate", - ): - pipeline_payload[key] = None - closed_loop_payload = {key: None for key in closed_loop_payload} - verdicts = [{**item, "value": None} for item in verdicts] - attack_profile = [] - top_threat_types = [] - severity_levels = [] - risk_levels = [] - triage_series_total = [] - triage_series_attack = [] - - missing_sources = [ - item - for item in denoise_locations + triage_locations - if not item["exists"] or item["fileCount"] == 0 - ] - if not triage_quality["dataAvailable"]: - missing_sources.append( - { - "kind": "soc", - "path": _display_path(DEFAULT_SQLITE_DB), - "exists": DEFAULT_SQLITE_DB.is_file(), - "fileCount": 0, - "reason": triage_quality["unavailableReason"], - } - ) result = { "date": start_date, @@ -3198,20 +2528,6 @@ def _get_stats(params): }, "workflowStatsDb": _display_path(WORKFLOW_DB), "workflowStats": workflow_stats, - "triageQuality": triage_quality, - "metricQuality": { - "status": workflow_stats.get("dataQuality", "legacy"), - "dataAvailable": workflow_stats.get("dataAvailable", True), - "unavailableReason": workflow_stats.get("unavailableReason", ""), - "coverageComplete": workflow_stats.get("coverageComplete", False), - "coverageStartedAt": workflow_stats.get("coverageStartedAt", 0), - "sourceCoverageRate": workflow_stats.get("sourceCoverageRate", 0), - "sourceMetricsAvailable": workflow_stats.get("sourceMetricsAvailable", False), - "invalidExecutionCount": workflow_stats.get("invalidExecutionCount", 0), - "unprocessedInputCount": workflow_stats.get("unprocessedInputCount", 0), - "errorExecutionCount": workflow_stats.get("errorCount", 0), - "metricsAvailable": metrics_available, - }, "sampleMode": sample_mode, "sampleFile": ", ".join(_source_label(path) for path in asset_files) if sample_mode else "", "assets": { @@ -3228,18 +2544,34 @@ def _get_stats(params): "triage": triage_locations, "denoiseFiles": [_file_brief(path) for path in denoise_files], "triageFiles": [_file_brief(path) for path in triage_files], - "missing": missing_sources, + "missing": [] if sample_mode else [ + item + for item in denoise_locations + triage_locations + if not item["exists"] or item["fileCount"] == 0 + ], }, "denoise": _without_counters(denoise), - "triage": triage_payload, - "pipeline": pipeline_payload, + "triage": _without_counters(triage), + "pipeline": pipeline, "sources": sources, - "closedLoop": closed_loop_payload, - "attackProfile": attack_profile, - "verdicts": verdicts, - "topThreatTypes": top_threat_types, - "severityLevels": severity_levels, - "riskLevels": risk_levels, + "closedLoop": closed_loop, + "attackProfile": _build_attack_profile(denoise, triage), + "verdicts": [ + {"key": "attack_success", "label": "攻击成功", "value": triage["attackSuccess"], "color": "#ff4d6d"}, + {"key": "attack", "label": "攻击行为", "value": triage["attack"], "color": "#ffb020"}, + {"key": "attack_failed", "label": "攻击失败", "value": triage["attackFailed"], "color": "#2ee6a6"}, + {"key": "non_attack", "label": "非攻击", "value": triage["benign"], "color": "#58a6ff"}, + {"key": "unknown", "label": "未知", "value": triage["unknown"], "color": "#9b8cff"}, + ], + "topThreatTypes": _counter_items( + triage["threatTypeCounter"] or denoise["threatTypeCounter"], + 14, + ), + "severityLevels": _counter_items( + _profile_counter(denoise, triage, "severityCounter"), + 8, + ), + "riskLevels": _counter_items(triage["riskCounter"], 5), "tokenUsage": _read_token_usage(), "timeline": { "labels": denoise.get("_timelineLabels") @@ -3248,8 +2580,8 @@ def _get_stats(params): or _timeline_window(start_date, end_date, len(denoise["seriesRaw"])), "denoiseRaw": denoise["seriesRaw"], "denoiseUnique": denoise["seriesUnique"], - "triageTotal": triage_series_total, - "triageAttack": triage_series_attack, + "triageTotal": triage["seriesTotal"], + "triageAttack": triage["seriesAttack"], }, } result["cacheHit"] = False @@ -3291,17 +2623,7 @@ def _date_span(start_date, end_date): def _find_asset_files(start_date, end_date, start_time=0, end_time=0): - sources, _ = _find_sqlite_sources_with_quality( - start_date, - end_date, - start_time, - end_time, - ) - return sources - - -def _find_asset_files_with_quality(start_date, end_date, start_time=0, end_time=0): - return _find_sqlite_sources_with_quality(start_date, end_date, start_time, end_time) + return _find_sqlite_sources(start_date, end_date, start_time, end_time) def _asset_file_date(path): @@ -3345,31 +2667,10 @@ def _active_source_exists(): def _find_sqlite_sources(start_date, end_date, start_time=0, end_time=0): - sources, _ = _find_sqlite_sources_with_quality( - start_date, - end_date, - start_time, - end_time, - ) - return sources - - -def _soc_source_quality(*, available, reason="", record_count=0): - return { - "status": "complete" if available else "unavailable", - "dataAvailable": bool(available), - "metricsAvailable": bool(available), - "unavailableReason": "" if available else str(reason or "soc_metrics_unavailable"), - "recordCount": max(_safe_int(record_count), 0) if available else None, - "dataSource": "soc.db.soc_dashboard_alert_facts" if available else "unavailable", - } - - -def _find_sqlite_sources_with_quality(start_date, end_date, start_time=0, end_time=0): settings = _sqlite_settings() db_path = settings["db_path"] if not db_path.is_file(): - return [], _soc_source_quality(available=False, reason="soc_db_missing") + return [] time_clause = "" query_params = [start_date, end_date] @@ -3385,24 +2686,9 @@ def _find_sqlite_sources_with_quality(start_date, end_date, start_time=0, end_ti ) try: with sqlite3.connect(db_path) as conn: - conn.execute("PRAGMA query_only = ON") - required_tables = (DEFAULT_SQLITE_TABLE, FACTS_TABLE, META_TABLE) - if not all(_table_exists(conn, table_name) for table_name in required_tables): - return [], _soc_source_quality( - available=False, - reason="soc_dashboard_schema_unavailable", - ) - schema_row = conn.execute( - f"SELECT meta_value FROM {META_TABLE} WHERE meta_key='schema_version'" - ).fetchone() - if not schema_row or str(schema_row[0]) != SCHEMA_VERSION: - return [], _soc_source_quality( - available=False, - reason="soc_dashboard_schema_unavailable", - ) rows = conn.execute(query, query_params).fetchall() except Exception: - return [], _soc_source_quality(available=False, reason="soc_db_query_failed") + return [] sources = [] for asset_date, record_count in rows: @@ -3420,10 +2706,7 @@ def _find_sqlite_sources_with_quality(start_date, end_date, start_time=0, end_ti end_time=end_time, ) ) - return sources, _soc_source_quality( - available=True, - record_count=sum(source.record_count for source in sources), - ) + return sources def _available_sqlite_dates(): diff --git a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/routes.yaml b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/routes.yaml index db37b7a93..020897392 100644 --- a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/routes.yaml +++ b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/routes.yaml @@ -9,11 +9,6 @@ routes: handler: handlers.get_activity timeoutMs: 5000 description: Incremental alert denoise and triage activity - - method: GET - path: /ai-tasks - handler: handlers.get_ai_tasks - timeoutMs: 5000 - description: Authoritative SOC workflow execution task snapshot - method: GET path: /task-center handler: handlers.get_task_center diff --git a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/Page.tsx b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/Page.tsx index d14c473f4..c08e3d499 100644 --- a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/Page.tsx +++ b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/Page.tsx @@ -24,56 +24,56 @@ const EMPTY_STATS = { eventRange: { start: '', end: '', label: '', source: '' }, generatedAt: '', latencyMs: 0, - sourceStatus: { workflowRoot: '', denoise: [], triage: [], denoiseFiles: [], triageFiles: [], missing: [], metricQuality: {}, triageQuality: {} }, + sourceStatus: { workflowRoot: '', denoise: [], triage: [], denoiseFiles: [], triageFiles: [], missing: [] }, denoise: { - totalRaw: null, - totalNormalized: null, - afterFilter: null, - totalUnique: null, - filterRemoved: null, - dedupRemoved: null, - duplicates: null, - duplicateRate: null, - dedupRate: null, - uniqueRate: null, - files: null, - parseErrors: null, + totalRaw: 0, + totalNormalized: 0, + afterFilter: 0, + totalUnique: 0, + filterRemoved: 0, + dedupRemoved: 0, + duplicates: 0, + duplicateRate: 0, + dedupRate: 0, + uniqueRate: 0, + files: 0, + parseErrors: 0, }, triage: { - totalRecords: null, - newTriaged: null, - cacheHit: null, - triageFailed: null, - followersReused: null, - attackTotal: null, - attackSuccess: null, - attack: null, - attackFailed: null, - benign: null, - unknown: null, - attackRate: null, - successRate: null, - cacheRate: null, - coverageRate: null, - avgTriageMs: null, - files: null, - parseErrors: null, + totalRecords: 0, + newTriaged: 0, + cacheHit: 0, + triageFailed: 0, + followersReused: 0, + attackTotal: 0, + attackSuccess: 0, + attack: 0, + attackFailed: 0, + benign: 0, + unknown: 0, + attackRate: 0, + successRate: 0, + cacheRate: 0, + coverageRate: 0, + avgTriageMs: 0, + files: 0, + parseErrors: 0, }, pipeline: { - raw: null, - unique: null, - triageTotal: null, - attackTotal: null, - reductionSaved: null, - llmSaved: null, - uniqueRate: null, - workloadReuseRate: null, - coverageRate: null, - attackRate: null, - successRate: null, + raw: 0, + unique: 0, + triageTotal: 0, + attackTotal: 0, + reductionSaved: 0, + llmSaved: 0, + uniqueRate: 0, + workloadReuseRate: 0, + coverageRate: 0, + attackRate: 0, + successRate: 0, }, sources: [], - closedLoop: { autoClosed: null, resolved: null, manualDecision: null, pending: null, resolutionRate: null }, + closedLoop: { autoClosed: 0, resolved: 0, manualDecision: 0, pending: 0, resolutionRate: 0 }, tokenUsage: { totalTokens: 0, todayTokens: 0, todayRequests: 0, dailySeries: [], dailyLabels: [], source: '' }, verdicts: [], attackProfile: [], @@ -382,69 +382,6 @@ function createTaskCenterState() { }; } -function createAiTasksState() { - return { - connection: 'initializing', - generatedAt: '', - reason: '', - summary: { - active: 0, - running: 0, - waiting: 0, - stale: 0, - disabled: 0, - returned: 0, - truncated: false, - }, - tasks: [], - }; -} - -function createMockAiTasksState() { - const now = Date.now(); - const tasks = [ - { - taskId: 'workflow:stream_alert_triage:mock-triage-run-002', - workflowId: 'stream_alert_triage', - executionId: 'mock-triage-run-002', - stage: 'triage', - status: 'running', - startedAt: now - 6200, - title: '远程命令执行攻击(Mock)', - sourceType: 'tdp', - counts: { raw: null }, - dataQuality: 'pending', - progress: { mode: 'steps', current: 2, total: 4, percent: 0.5, label: '2 / 4 步' }, - }, - { - taskId: 'workflow:stream_alert_denoise:mock-denoise-run-004', - workflowId: 'stream_alert_denoise', - executionId: 'mock-denoise-run-004', - stage: 'denoise', - status: 'queued', - startedAt: now - 18000, - title: '端口扫描聚类(Mock)', - sourceType: 'qingteng', - counts: { raw: 12 }, - dataQuality: 'complete', - progress: { mode: 'queued', current: 0, total: 4, percent: null, label: '等待执行' }, - }, - ]; - return { - ...createAiTasksState(), - connection: 'online', - generatedAt: new Date(now).toISOString(), - summary: { - ...createAiTasksState().summary, - active: tasks.length, - running: 1, - waiting: 1, - returned: tasks.length, - }, - tasks, - }; -} - function createMockTaskCenterState() { const now = Date.now(); const startedAt = now - 7 * 60 * 1000; @@ -775,91 +712,30 @@ function refreshLabel(value) { } function mergeStats(raw) { - const sourceStatus = { - ...EMPTY_STATS.sourceStatus, - ...((raw || {}).sourceStatus || {}), - metricQuality: { - ...(EMPTY_STATS.sourceStatus.metricQuality || {}), - ...((raw || {}).sourceStatus?.metricQuality || {}), - }, - triageQuality: { - ...(EMPTY_STATS.sourceStatus.triageQuality || {}), - ...((raw || {}).sourceStatus?.triageQuality || {}), - }, - }; - const metricQuality = sourceStatus.metricQuality || {}; - const triageQuality = sourceStatus.triageQuality || {}; - const denoiseMetricsUnavailable = metricQuality.metricsAvailable === false; - const triageMetricsUnavailable = triageQuality.metricsAvailable === false; - const sourceMetricsUnavailable = denoiseMetricsUnavailable - || metricQuality.sourceMetricsAvailable === false; const denoise = { ...EMPTY_STATS.denoise, ...((raw || {}).denoise || {}) }; - if (denoiseMetricsUnavailable) { - for (const key of [ - 'totalRaw', 'totalNormalized', 'afterFilter', 'totalUnique', - 'filterRemoved', 'dedupRemoved', 'duplicates', 'duplicateRate', - 'dedupRate', 'uniqueRate', - ]) denoise[key] = null; - if (metricQuality.dataAvailable === false) denoise.files = null; - } - const triage = { ...EMPTY_STATS.triage, ...((raw || {}).triage || {}) }; - if (triageMetricsUnavailable) { - for (const key of Object.keys(EMPTY_STATS.triage)) triage[key] = null; - } - const pipeline = { ...EMPTY_STATS.pipeline, ...((raw || {}).pipeline || {}) }; - if (denoiseMetricsUnavailable) { - for (const key of ['raw', 'unique', 'reductionSaved', 'uniqueRate', 'coverageRate']) { - pipeline[key] = null; - } - } - if (triageMetricsUnavailable) { - for (const key of [ - 'triageTotal', 'attackTotal', 'llmSaved', 'workloadReuseRate', - 'attackRate', 'successRate', - ]) pipeline[key] = null; - } - const timeline = { ...EMPTY_STATS.timeline, ...((raw || {}).timeline || {}) }; - if (denoiseMetricsUnavailable) { - timeline.denoiseRaw = []; - timeline.denoiseUnique = []; - } - if (triageMetricsUnavailable) { - timeline.triageTotal = []; - timeline.triageAttack = []; - } - const sources = Array.isArray((raw || {}).sources) ? raw.sources : []; - const closedLoop = { ...EMPTY_STATS.closedLoop, ...((raw || {}).closedLoop || {}) }; - if (triageMetricsUnavailable) { - for (const key of Object.keys(EMPTY_STATS.closedLoop)) closedLoop[key] = null; - } + const processedTotal = Math.max(Number(denoise.totalRaw || 0), 0); + denoise.totalNormalized = processedTotal; return { ...EMPTY_STATS, ...(raw || {}), - sourceStatus, + sourceStatus: { ...EMPTY_STATS.sourceStatus, ...((raw || {}).sourceStatus || {}) }, denoise, - triage, - pipeline, - closedLoop, + triage: { ...EMPTY_STATS.triage, ...((raw || {}).triage || {}) }, + pipeline: { ...EMPTY_STATS.pipeline, ...((raw || {}).pipeline || {}) }, + closedLoop: { ...EMPTY_STATS.closedLoop, ...((raw || {}).closedLoop || {}) }, tokenUsage: { ...EMPTY_STATS.tokenUsage, ...((raw || {}).tokenUsage || {}) }, dateRange: { ...EMPTY_STATS.dateRange, ...((raw || {}).dateRange || {}) }, eventRange: { ...EMPTY_STATS.eventRange, ...((raw || {}).eventRange || {}) }, - timeline, - sources: sourceMetricsUnavailable - ? sources.map((item) => ({ ...item, value: null, rate: null, active: false })) - : sources, - verdicts: triageMetricsUnavailable - ? ((raw || {}).verdicts || []).map((item) => ({ ...item, value: null })) - : ((raw || {}).verdicts || []), - attackProfile: triageMetricsUnavailable ? [] : ((raw || {}).attackProfile || []), - topThreatTypes: triageMetricsUnavailable ? [] : ((raw || {}).topThreatTypes || []), - severityLevels: triageMetricsUnavailable ? [] : ((raw || {}).severityLevels || []), - riskLevels: triageMetricsUnavailable ? [] : ((raw || {}).riskLevels || []), + timeline: { ...EMPTY_STATS.timeline, ...((raw || {}).timeline || {}) }, + sources: [ + { key: 'ndr', label: 'NDR', value: processedTotal, rate: processedTotal > 0 ? 1 : 0, active: processedTotal > 0 }, + { key: 'other', label: '其他接入', value: 0, rate: 0, active: false }, + ], }; } function fullNumber(value) { - if (value === null || value === undefined || !Number.isFinite(Number(value))) return '--'; - const n = Number(value); + const n = Number(value || 0); return new Intl.NumberFormat('zh-CN').format(n); } @@ -895,8 +771,7 @@ function workflowDenoiseActivity(callCount, delta, generatedAt, workflowEvent) { } function compactNumber(value) { - if (value === null || value === undefined || !Number.isFinite(Number(value))) return '--'; - const n = Number(value); + const n = Number(value || 0); if (Math.abs(n) >= 100000000) return `${trim(n / 100000000)}亿`; if (Math.abs(n) >= 10000) return `${trim(n / 10000)}万`; return fullNumber(n); @@ -910,8 +785,7 @@ function formatTokenVolume(value) { function AnimatedNumber({ value, format, tag = 'span', className, duration = 900 }) { const { useEffect, useRef, useState } = getReact(); - const unavailable = value === null || value === undefined || !Number.isFinite(Number(value)); - const target = unavailable ? 0 : Number(value); + const target = Number(value || 0); const current = useRef(0); const [display, setDisplay] = useState(0); const formatter = format || ((number) => compactNumber(Math.round(number))); @@ -939,8 +813,8 @@ function AnimatedNumber({ value, format, tag = 'span', className, duration = 900 return h(tag, { className: cx('animated-number', className), - title: unavailable ? '数据不可用' : formatter(target), - }, unavailable ? '--' : formatter(display)); + title: formatter(target), + }, formatter(display)); } function trim(value) { @@ -948,8 +822,7 @@ function trim(value) { } function pct(value) { - if (value === null || value === undefined || !Number.isFinite(Number(value))) return '--'; - return `${Math.round(Number(value) * 1000) / 10}%`; + return `${Math.round(Number(value || 0) * 1000) / 10}%`; } function ratio(part, total) { @@ -1041,9 +914,7 @@ function SourceColumn({ stats }) { return h('div', { className: 'column left-col' }, [ h(Panel, { key: 'sources', title: '多源告警接入', meta: `${stats.denoise.files || 0} 个降噪批次` }, [ h('div', { className: 'source-list', key: 'list' }, stats.sources.map((item) => { - const width = item.value === null || item.value === undefined - ? '0%' - : `${Math.max(4, Math.round((item.rate || 0) * 100))}%`; + const width = `${Math.max(4, Math.round((item.rate || 0) * 100))}%`; return h('div', { className: 'source-row', key: item.key }, [ h('div', { className: cx('source-node', item.active && 'active'), key: 'node' }), h('div', { className: 'source-main', key: 'main' }, [ @@ -1891,9 +1762,6 @@ function laneLinkStatus(kind, event, peerLane) { } function triageContextText(stats) { - if (stats?.sourceStatus?.triageQuality?.metricsAvailable === false) { - return '窗口研判数据不可用'; - } const triage = stats?.triage || EMPTY_STATS.triage; const total = Math.max(Number(triage.totalRecords || 0), 0); const newTriaged = Math.max(Number(triage.newTriaged || 0), 0); @@ -1973,10 +1841,8 @@ function CommandGraph({ stats, activity }) { const activeSeverityTone = severityToneFor(activity.triage.current); const recentSeverityTone = severityToneFor(activity.triage.last); const activeSources = [...(stats.sources || [])].sort((a, b) => Number(b.value || 0) - Number(a.value || 0)).slice(0, 2); - while (activeSources.length < 2) activeSources.push({ key: `source-${activeSources.length}`, label: activeSources.length ? '备用数据源' : '告警数据源', value: null }); - const triageMetricsAvailable = Boolean(stats.generatedAt) - && stats.sourceStatus?.triageQuality?.metricsAvailable !== false; - const severities = severityRows(stats, triageMetricsAvailable); + while (activeSources.length < 2) activeSources.push({ key: `source-${activeSources.length}`, label: activeSources.length ? '备用数据源' : '告警数据源', value: 0 }); + const severities = severityRows(stats); return h('section', { className: cx('command-graph', denoiseActive && 'denoise-running', triageActive && 'triage-running', `load-${activity.mode}`) }, [ h(CommandConnections, { key: 'links' }), h('div', { className: 'source-stack', key: 'sources' }, activeSources.map((source) => h('div', { className: 'command-source', key: source.key }, [ @@ -2064,7 +1930,7 @@ function CommandMetrics({ stats }) { return h('section', { className: 'command-metrics' }, [ h(CommandMetric, { label: '原始告警量', value: stats.denoise.totalRaw, sub: `${compactNumber(stats.denoise.totalUnique)} 条进入研判`, values: stats.timeline.denoiseRaw, color: '#2e72ff', key: 'raw' }), h(CommandMetric, { label: '安全事件量', value: stats.triage.attackTotal, sub: `${compactNumber(stats.triage.attackSuccess)} 条攻击成功`, values: stats.timeline.triageAttack, color: '#23ca8e', key: 'events' }), - h(CommandMetric, { label: '降噪率', value: stats.denoise.duplicateRate === null ? null : stats.denoise.duplicateRate * 100, format: (value) => `${trim(value)}%`, sub: `${compactNumber(stats.denoise.duplicates)} 条告警已过滤/收敛`, values: stats.timeline.denoiseUnique, color: '#21d8a3', key: 'rate' }), + h(CommandMetric, { label: '降噪率', value: stats.denoise.duplicateRate * 100, format: (value) => `${trim(value)}%`, sub: `${compactNumber(stats.denoise.duplicates)} 条告警已过滤/收敛`, values: stats.timeline.denoiseUnique, color: '#21d8a3', key: 'rate' }), h(TokenUsageMetric, { tokenUsage, key: 'tokens' }), ]); } @@ -2548,83 +2414,56 @@ function CommandTaskCenterPanel({ taskCenter }) { ]); } -function WorkflowTaskProgress({ task }) { - const progress = task.progress || {}; - if (progress.mode !== 'steps' || !Number.isFinite(Number(progress.percent))) { - return h('div', { className: 'event-rail-progress indeterminate', 'aria-label': 'AI 任务执行中' }, [ - h('span', { className: 'event-rail-progress-track', key: 'track' }, [h('i', { key: 'fill' })]), - h('small', { key: 'label' }, progress.label || '执行中'), - ]); - } - const percent = Math.max(Math.min(Number(progress.percent), 1), 0); - return h('div', { className: 'event-rail-progress', 'aria-label': 'AI 任务步骤进度' }, [ - h('span', { className: 'event-rail-progress-track', style: { '--queue-progress': percent }, key: 'track' }, [h('i', { key: 'fill' })]), - h('small', { key: 'label' }, progress.label || `${Math.round(percent * 100)}%`), - ]); -} - -function CommandAiTaskPanel({ aiTasks }) { - const summary = { ...createAiTasksState().summary, ...(aiTasks.summary || {}) }; - const visibleTasks = (aiTasks.tasks || []).slice(0, EVENT_RAIL_TASK_LIMIT); - const banner = aiTasks.connection === 'error' || aiTasks.connection === 'unavailable' - ? '处理任务数据不可用,正在重试' - : summary.running && summary.waiting - ? `正在处理 ${summary.running} 个,等待 ${summary.waiting} 个` - : summary.running - ? `AI 正在处理 ${summary.running} 个任务` - : summary.waiting - ? `${summary.waiting} 个任务等待处理` - : summary.stale - ? `发现 ${summary.stale} 个失联任务,已移出活跃列表` - : '当前没有运行或等待中的 AI 任务'; +function CommandAiTaskPanel({ activity, timeFilter }) { + const tasks = buildEventQueueTasks(activity, timeFilter); + const filterTransitionKey = [timeFilter.mode, timeFilter.range, timeFilter.start, timeFilter.end].join('|'); + const visibleTasks = useAnimatedTaskWindow( + tasks.filter((task) => task.state !== 'completed'), + filterTransitionKey, + ); + const counts = { + processing: visibleTasks.filter((task) => task.state === 'processing').length, + waiting: visibleTasks.filter((task) => task.state === 'waiting').length, + }; + const queueCount = visibleTasks.length; + const banner = activity.connection === 'error' + ? '处理任务连接异常,正在重试' + : counts.processing + ? `AI 正在并行处理 ${counts.processing} 个任务` + : counts.waiting ? '最新 10 条待处理任务' : '等待新的降噪或研判任务'; return [ - h('div', { className: cx('event-update-banner', ['error', 'unavailable'].includes(aiTasks.connection) && 'warn'), key: 'banner' }, banner), - summary.truncated - ? h('small', { className: 'event-rail-limit-note', key: 'limit' }, `共 ${summary.active} 条活跃任务,当前展示 ${visibleTasks.length} 条`) - : null, + h('div', { className: cx('event-update-banner', activity.connection === 'error' && 'warn'), key: 'banner' }, banner), h('div', { className: 'event-rail-list', key: 'list' }, visibleTasks.length ? visibleTasks.map((task) => { - const processing = task.status === 'running'; - const waiting = ['queued', 'pending'].includes(task.status); - const title = task.title || (task.stage === 'triage' ? '研判任务' : '降噪任务'); + const event = task.event; + const sampleCount = Math.max(Number(task.denoise?.sampleCount || 1), 1); + const title = `${event?.alert?.threatName || '未知告警'}${sampleCount > 1 ? ` × ${sampleCount}` : ''}`; const stageLabel = task.stage === 'triage' - ? waiting ? '待研判' : '智能研判' - : waiting ? '待降噪' : '智能降噪'; - const stateLabel = processing ? '处理中' : '等待处理'; - const unverifiedZero = task.stage === 'denoise' - && Number(task.counts?.raw) === 0 - && !task.emptyInput - && task.rawCountSource !== 'workflow_input'; - const qualityDetail = unverifiedZero - ? ' · 原始条数待校验' - : task.dataQuality === 'invalid' - ? ' · 指标格式异常' - : task.dataQuality === 'empty-input' - ? ' · 空输入任务' - : task.dataQuality === 'missing' - ? ' · 原始条数未知' - : task.dataQuality === 'pending' && (task.counts?.raw === null || task.counts?.raw === undefined) - ? ' · 原始条数待生成' - : ''; - const rawDetail = task.stage === 'denoise' && !task.emptyInput && !unverifiedZero && task.counts?.raw !== null && task.counts?.raw !== undefined - ? ` · 原始 ${task.counts.raw} 条` - : ''; - const detail = task.stage === 'triage' - ? processing ? '研判工作流执行中' : '等待研判工作流调度' - : processing ? '降噪工作流执行中' : '等待降噪工作流调度'; - const hasExecution = Boolean(task.workflowId && task.executionId); + ? task.state === 'waiting' ? '待研判' : '智能研判' + : task.state === 'waiting' ? '待降噪' : '智能降噪'; + const stateLabel = task.state === 'processing' + ? '处理中' + : '等待处理'; + const detail = event?.triggerSource === 'workflow_execution' + ? task.stage === 'triage' + ? task.state === 'processing' ? '研判工作流处理中' : '研判工作流待处理' + : event.result?.isDuplicate ? '重复告警已收敛' : '降噪处理完成' + : task.state === 'processing' + ? task.stage === 'triage' ? '证据关联与结论生成中' : '特征提取与相似聚类中' + : '等待 AI 处理'; + const hasExecution = Boolean(workflowIdFromEvent(event) && executionIdFromWorkflowEvent(event)); const handleOpen = () => { - if (hasExecution) openWorkflowExecution(task.workflowId, task.executionId); + if (hasExecution) openWorkflowExecutionFromEvent(event); }; const handleKeyDown = (keyboardEvent) => { if (!hasExecution) return; if (keyboardEvent.key === 'Enter' || keyboardEvent.key === ' ') { keyboardEvent.preventDefault(); - openWorkflowExecution(task.workflowId, task.executionId); + openWorkflowExecutionFromEvent(event); } }; return h('article', { - className: cx('event-rail-item', processing ? 'state-processing' : 'state-waiting', `kind-${task.stage}`, hasExecution && 'clickable'), - key: task.taskId, + className: cx('event-rail-item', `state-${task.state}`, `kind-${task.stage}`, `motion-${task.motion || 'stable'}`, hasExecution && 'clickable'), + key: task.key, role: hasExecution ? 'button' : undefined, tabIndex: hasExecution ? 0 : undefined, title: hasExecution ? '打开执行详情' : undefined, @@ -2634,19 +2473,20 @@ function CommandAiTaskPanel({ aiTasks }) { h('div', { className: 'event-rail-meta', key: 'meta' }, [ h('span', { className: cx('event-queue-kind', `kind-${task.stage}`), key: 'kind' }, stageLabel), h('span', { className: 'event-stage', key: 'stage' }, stateLabel), - h('time', { key: 'time' }, eventTimeLabel(task.startedAt)), + h('time', { key: 'time' }, eventTimeLabel(event?.occurredAt)), ]), h('strong', { title, key: 'title' }, title), - h('span', { key: 'endpoint' }, [task.srcIp, task.dstIp].filter(Boolean).join(' → ') || '未提供网络端点'), - h('small', { key: 'result' }, `${detail}${rawDetail}${qualityDetail}${hasExecution ? ' · 查看执行' : ''}`), - processing ? h(WorkflowTaskProgress, { task, key: 'progress' }) : null, + h('span', { title: eventEndpoint(event), key: 'endpoint' }, eventEndpoint(event)), + h('small', { key: 'result' }, hasExecution ? `${detail} · 查看执行` : detail), + task.state === 'processing' ? h(EventQueueProgress, { event, key: 'progress' }) : null, ]); - }) : h('div', { className: 'event-rail-empty' }, '暂无活跃的降噪或研判任务')), + }) : h('div', { className: 'event-rail-empty' }, '等待新的降噪或研判任务')), ]; } -function CommandEventRail({ aiTasks, taskCenter, view, onViewChange, collapsed, onToggle, railWidth, onResizeStart, onResizeKeyDown }) { - const queueCount = Math.max(Number(aiTasks.summary?.active || 0), 0); +function CommandEventRail({ activity, timeFilter, taskCenter, view, onViewChange, collapsed, onToggle, railWidth, onResizeStart, onResizeKeyDown }) { + const tasks = buildEventQueueTasks(activity, timeFilter); + const queueCount = tasks.filter((task) => task.state !== 'completed').length; const taskCenterCount = Number(taskCenter.sessionCount || 0); const content = collapsed ? [] : [ h('div', { className: 'event-rail-head rail-view-head', key: 'head' }, [ @@ -2672,7 +2512,7 @@ function CommandEventRail({ aiTasks, taskCenter, view, onViewChange, collapsed, ]), view === 'taskCenter' ? h(CommandTaskCenterPanel, { taskCenter, key: 'taskCenterContent' }) - : h(CommandAiTaskPanel, { aiTasks, key: 'aiTaskContent' }), + : h(CommandAiTaskPanel, { activity, timeFilter, key: 'aiTaskContent' }), ]; return h('aside', { className: cx('command-event-rail', collapsed && 'collapsed') }, [ collapsed ? null : h('div', { @@ -2714,7 +2554,6 @@ export default function Page() { const [loading, setLoading] = useState(true); const [error, setError] = useState(''); const [activity, setActivity] = useState(createActivityState); - const [aiTasks, setAiTasks] = useState(createAiTasksState); const [taskCenter, setTaskCenter] = useState(createTaskCenterState); const activityCursor = useRef(''); const workflowProgressByFilter = useRef(new Map()); @@ -2901,50 +2740,6 @@ export default function Page() { }; }, [mockDashboardEnabled]); - useEffect(() => { - let stopped = false; - let timer = 0; - - const schedule = (delay) => { - if (!stopped) timer = window.setTimeout(() => void poll(), delay); - }; - - const poll = async () => { - if (stopped) return; - if (document.hidden) { - schedule(ACTIVITY_POLL_MS); - return; - } - try { - const response = await getApi().page.get('/ai-tasks', { params: {} }); - const payload = response.data || {}; - if (!stopped) { - setAiTasks({ - ...createAiTasksState(), - ...payload, - summary: { ...createAiTasksState().summary, ...(payload.summary || {}) }, - tasks: Array.isArray(payload.tasks) ? payload.tasks : [], - }); - } - } catch (aiTaskError) { - if (!stopped) { - setAiTasks((previous) => ({ - ...previous, - connection: 'error', - reason: aiTaskError instanceof Error ? aiTaskError.message : 'ai tasks api failed', - })); - } - } - schedule(ACTIVITY_POLL_MS); - }; - - void poll(); - return () => { - stopped = true; - window.clearTimeout(timer); - }; - }, []); - useEffect(() => { let stopped = false; let timer = 0; @@ -2992,8 +2787,7 @@ export default function Page() { : (payload.events || []); const incomingEvents = rawIncomingEvents.filter((event) => event?.stage !== 'denoise'); const workflowEvents = Array.isArray(payload.workflowEvents) ? payload.workflowEvents : []; - const activeWorkflowEvents = workflowEvents.filter(isRunningWorkflowEvent); - for (const workflowEvent of activeWorkflowEvents) { + for (const workflowEvent of workflowEvents) { const hasExecution = Boolean(workflowIdFromEvent(workflowEvent) && executionIdFromWorkflowEvent(workflowEvent)); if (hasExecution) { incomingEvents.push(workflowEvent); @@ -3025,9 +2819,9 @@ export default function Page() { workflowProgressByFilter.current.set(workflowFilterKey, { callCount, latestStartedAt }); } const incomingRecentEvents = bootstrap - ? [...(payload.recentEvents || []), ...activeWorkflowEvents] + ? [...(payload.recentEvents || []), ...workflowEvents] : workflowChanged - ? [...rawIncomingEvents, ...activeWorkflowEvents] + ? [...rawIncomingEvents, ...workflowEvents] : rawIncomingEvents; setActivity((previous) => enqueueActivity( previous, @@ -3130,14 +2924,6 @@ export default function Page() { ), [mockDashboardEnabled, taskCenter], ); - const displayAiTasks = useMemo( - () => ( - mockDashboardEnabled && !aiTasks.tasks.length - ? createMockAiTasksState() - : aiTasks - ), - [aiTasks, mockDashboardEnabled], - ); const displayActivityBusy = Boolean( displayActivity.denoise.current || displayActivity.triage.current @@ -3146,46 +2932,6 @@ export default function Page() { || displayActivity.batch?.receivedCount || displayActivity.batch?.triageUpdatedCount ); - const metricQuality = stats.sourceStatus?.metricQuality || {}; - const triageQuality = stats.sourceStatus?.triageQuality || {}; - const metricIssues = []; - if (Number(metricQuality.invalidExecutionCount || 0) > 0) { - metricIssues.push(`${metricQuality.invalidExecutionCount} 次指标格式异常`); - } - if (Number(metricQuality.errorExecutionCount || 0) > 0) { - metricIssues.push(`${metricQuality.errorExecutionCount} 次执行失败`); - } - if (Number(metricQuality.unprocessedInputCount || 0) > 0) { - metricIssues.push(`${metricQuality.unprocessedInputCount} 条输入未完成归一化`); - } - const sourceCoverageRate = Number(metricQuality.sourceCoverageRate); - if (metricQuality.metricsAvailable && Number.isFinite(sourceCoverageRate) && sourceCoverageRate < 1) { - metricIssues.push(`来源覆盖 ${Math.round(sourceCoverageRate * 1000) / 10}%`); - } - const metricQualityWarning = !stats.generatedAt - ? '' - : metricQuality.dataAvailable === false || metricQuality.status === 'unavailable' - ? '降噪统计数据源不可用,相关数字已隐藏;系统正在重试' - : metricQuality.status === 'stale' - ? '降噪统计查询失败,当前显示上次缓存数据,可能已过期' - : metricQuality.status === 'partial' && !metricQuality.metricsAvailable - ? `降噪指标校验未通过,相关数字已隐藏${metricIssues.length ? `:${metricIssues.join(';')}` : ''}` - : metricQuality.metricsAvailable - ? metricQuality.status === 'partial' - ? `降噪指标部分可用${metricIssues.length ? `:${metricIssues.join(';')}` : ''}${metricQuality.coverageStartedAt ? `;完整采集始于 ${taskCenterTimeLabel(metricQuality.coverageStartedAt)}` : ''}` - : metricIssues.length - ? `降噪指标存在异常:${metricIssues.join(';')}` - : '' - : metricQuality.status === 'legacy-partial' - ? `精确降噪指标尚未覆盖当前时间范围,相关数字已隐藏;完整采集始于 ${taskCenterTimeLabel(metricQuality.coverageStartedAt)}` - : '精确降噪指标尚不可用,相关数字已隐藏;新指标链路产生数据后将自动显示'; - const triageQualityWarning = !stats.generatedAt || triageQuality.metricsAvailable !== false - ? '' - : triageQuality.unavailableReason === 'soc_db_missing' - ? 'SOC 事件数据库不可用,研判、事件与闭环数字已隐藏;系统正在重试' - : triageQuality.unavailableReason === 'soc_dashboard_schema_unavailable' - ? 'SOC 统计结构尚未就绪,研判、事件与闭环数字已隐藏;系统正在重试' - : 'SOC 研判统计查询失败,研判、事件与闭环数字已隐藏;系统正在重试'; return h('div', { className: cx('adtd-root command-root', displayActivityBusy && 'command-is-processing', eventRailCollapsed && 'event-rail-is-collapsed'), @@ -3207,8 +2953,6 @@ export default function Page() { activity: displayActivity, }), error ? h('div', { className: 'error-banner', key: 'error' }, `统计接口异常:${error}`) : null, - metricQualityWarning ? h('div', { className: 'quality-banner', key: 'quality' }, metricQualityWarning) : null, - triageQualityWarning ? h('div', { className: 'quality-banner', key: 'triage-quality' }, triageQualityWarning) : null, h('main', { className: cx('command-shell', eventRailCollapsed && 'event-rail-collapsed'), key: 'main', @@ -3219,7 +2963,8 @@ export default function Page() { ]), h(CommandEventRail, { key: 'events', - aiTasks: displayAiTasks, + activity: displayActivity, + timeFilter, taskCenter: displayTaskCenter, view: rightRailView, onViewChange: setRightRailView, @@ -3250,15 +2995,6 @@ const CSS = ` overflow-x: auto; } .adtd-root * { box-sizing: border-box; } -.quality-banner { - margin: 8px 0 0; - padding: 8px 12px; - border: 1px solid rgba(255,176,32,.34); - border-radius: 6px; - color: #f1c67d; - background: rgba(139,88,22,.16); - font-size: 11px; -} .adtd-header { position: relative; display: grid; @@ -5519,12 +5255,6 @@ const CSS = ` } .event-update-banner:before { content: "ⓘ"; margin-right: 7px; color: #6ba4fb; } .event-update-banner.warn { border-color: rgba(255,174,52,.42); color: #f1c67d; background: rgba(139,88,22,.2); } -.event-rail-limit-note { - display: block; - margin: 5px 14px 0; - color: rgba(170,222,255,.62); - font-size: 9px; -} .task-center-panel { min-height: 0; padding: 8px 14px 18px; @@ -5951,11 +5681,6 @@ const CSS = ` box-shadow: 0 0 10px #73e9ff; transform: translate(50%, -50%); } -.event-rail-progress.indeterminate .event-rail-progress-track i { - width: 38%; - transform: translateX(-120%); - animation: commandTaskIndeterminate 1.4s ease-in-out infinite; -} .event-rail-progress > small { color: #9a8eff; font-size: 9px; @@ -5969,10 +5694,6 @@ const CSS = ` font-size: 11px; } @keyframes commandFlow { to { stroke-dashoffset: -36; } } -@keyframes commandTaskIndeterminate { - 0% { transform: translateX(-120%); } - 100% { transform: translateX(340%); } -} @keyframes commandSpin { to { transform: rotate(360deg); } } @keyframes commandSpinReverse { to { transform: rotate(-360deg); } } @keyframes commandCorePulse { diff --git a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/severityValues.ts b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/severityValues.ts index 675e639f4..d2fcd6f2a 100644 --- a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/severityValues.ts +++ b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/severityValues.ts @@ -1,6 +1,6 @@ type SeverityItem = { label?: string; - value?: number | null; + value?: number; }; export function severityKey(value: unknown) { @@ -12,19 +12,16 @@ export function severityKey(value: unknown) { return ''; } -export function severityRows( - stats: { severityLevels?: SeverityItem[] }, - metricsAvailable = true, -) { +export function severityRows(stats: { severityLevels?: SeverityItem[] }) { const counts = { critical: 0, high: 0, medium: 0, low: 0 }; for (const item of stats.severityLevels || []) { const key = severityKey(item.label); if (key) counts[key] += Number(item.value || 0); } return [ - { key: 'critical', label: '严重', value: metricsAvailable ? counts.critical : null, tone: 'critical' }, - { key: 'high', label: '高危', value: metricsAvailable ? counts.high : null, tone: 'high' }, - { key: 'medium', label: '中危', value: metricsAvailable ? counts.medium : null, tone: 'medium' }, - { key: 'low', label: '低危', value: metricsAvailable ? counts.low : null, tone: 'low' }, + { key: 'critical', label: '严重', value: counts.critical, tone: 'critical' }, + { key: 'high', label: '高危', value: counts.high, tone: 'high' }, + { key: 'medium', label: '中危', value: counts.medium, tone: 'medium' }, + { key: 'low', label: '低危', value: counts.low, tone: 'low' }, ]; } diff --git a/.flocks/flockshub/plugins/workflows/stream_alert_denoise/workflow.json b/.flocks/flockshub/plugins/workflows/stream_alert_denoise/workflow.json index c2c914faa..ea1b4193e 100644 --- a/.flocks/flockshub/plugins/workflows/stream_alert_denoise/workflow.json +++ b/.flocks/flockshub/plugins/workflows/stream_alert_denoise/workflow.json @@ -27,7 +27,7 @@ "id": "dedup_and_write", "type": "python", "description": "Dedup (terminal): URI normalization + MinHash LSH (128 perms, 5-gram). LSH state persisted to ~/.flocks/workspace/workflows/stream_alert_denoise/ (atomic write, file lock, FIFO LRU eviction). Each output alert = normalized fields + dedup_key + is_duplicate + _lsh_cluster_id. Appends enriched alerts to JSONL files under ~/.flocks/workspace/workflows/stream_alert_denoise//dedup_result_NNN.jsonl. Each new file begins with a header line {_type:file_header, created_at, ...}; max 10,000 alert records per file, auto-increments sequence number on rollover.", - "code": "\nimport os\nimport re\nimport sys\nimport gc as _gc_module\nimport json\nimport pickle\nimport hashlib\nimport datetime\nimport threading\nimport types\nfrom datasketch import MinHash, MinHashLSH\n\nIS_WINDOWS = sys.platform == 'win32'\nif IS_WINDOWS:\n import msvcrt # noqa: F401\nelse:\n import fcntl # noqa: F401\n\nMINHASH_SEED = 2024\nNUM_PERM = 128\nWORKFLOW_NAME = 'stream_alert_denoise'\nLSH_CLUSTER_WARN_THRESHOLD = 100000\n\n# ── Process-level in-memory LSH state cache ──────────────────────────────────\n# Previously the ~649 MB pickle was loaded from disk on EVERY syslog message,\n# causing linear memory growth (Python GC cannot free old objects fast enough\n# under high throughput). We now keep the live MinHashLSH + lsh_cache +\n# dedup_key_cache in sys.modules between exec() invocations. A\n# threading.Lock serialises concurrent workflow threads; the file lock\n# (fcntl/msvcrt) still guards cross-process disk writes.\n_MEM_CACHE_KEY = f'_flocks_lsh_cache_{WORKFLOW_NAME}'\nif _MEM_CACHE_KEY not in sys.modules:\n _m = types.ModuleType(_MEM_CACHE_KEY)\n _m.lsh_index = None\n _m.lsh_cache = {}\n _m.dedup_key_cache = {}\n _m.next_cluster_id = 0\n _m.threshold = None\n _m.state_mtime = 0.0\n _m.append_mtime = 0.0\n _m.initialized = False\n _m.permutations = None\n _m.lock = threading.Lock()\n sys.modules[_MEM_CACHE_KEY] = _m\n_mem = sys.modules[_MEM_CACHE_KEY]\nif not hasattr(_mem, 'append_mtime'):\n _mem.append_mtime = 0.0\nif not hasattr(_mem, 'permutations'):\n _mem.permutations = None\n\ndef normalize_uri(uri):\n uri = str(uri or '')\n uri = re.sub(r'\\d{4}-\\d{2}-\\d{2}', 'DATETIME', uri)\n uri = re.sub(r'[\\da-f]{8}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{12}', 'UUID', uri, flags=re.IGNORECASE)\n uri = re.sub(r'(\\.\\./)+', 'TRAVERSAL', uri)\n uri = re.sub(r'\\bNULL\\b', 'NULL_REPLACED', uri)\n uri = re.sub(r'chr\\$\\d+\\$\\|\\|chr\\$\\d+\\$', 'CHR_SEQUENCE', uri)\n uri = re.sub(r'\\b\\d+={1,2}\\d+\\b', 'NUMBER_COMPARISON', uri)\n uri = re.sub(r'\\b[a-fA-F0-9]{32}\\b', 'HEXADECIMAL CHARACTERS', uri)\n return uri\n\ndef gen_minhash(text, permutations):\n shingles = [text[i:i+5] for i in range(len(text) - 4)]\n m = MinHash(num_perm=NUM_PERM, seed=MINHASH_SEED, permutations=permutations)\n for s in shingles:\n m.update(s.encode('utf-8'))\n return m\n\ndef get_state_paths(threshold):\n from flocks.config import Config\n flocks_root = Config().get_global().data_dir.parent\n state_dir = str(flocks_root / 'workspace' / 'workflows' / WORKFLOW_NAME)\n os.makedirs(state_dir, exist_ok=True)\n base = os.path.join(state_dir, f'lsh_state_np{NUM_PERM}_th{int(threshold * 100)}')\n return base + '.pkl', base + '.lock', base + '.append.log'\n\ndef get_output_dir():\n from flocks.config import Config\n from pathlib import Path\n flocks_root = Config().get_global().data_dir.parent\n date_str = datetime.datetime.now().strftime('%Y-%m-%d')\n out_dir = flocks_root / 'workspace' / 'workflows' / WORKFLOW_NAME / date_str\n out_dir.mkdir(parents=True, exist_ok=True)\n return str(out_dir)\n\ndef acquire_lock(lock_path):\n fh = open(lock_path, 'w+')\n try:\n if IS_WINDOWS:\n fh.write('L'); fh.flush(); fh.seek(0)\n while True:\n try:\n msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK, 1); break\n except OSError:\n continue\n else:\n fcntl.flock(fh.fileno(), fcntl.LOCK_EX)\n except BaseException:\n try:\n fh.close()\n except Exception:\n pass\n raise\n return fh\n\ndef release_lock(fh):\n try:\n if IS_WINDOWS:\n try:\n fh.seek(0); msvcrt.locking(fh.fileno(), msvcrt.LK_UNLCK, 1)\n except OSError:\n pass\n else:\n fcntl.flock(fh.fileno(), fcntl.LOCK_UN)\n finally:\n fh.close()\n\ndef load_state(state_path, append_path, threshold, max_keys=100000):\n lsh_index = None\n lsh_cache = {}\n dedup_key_cache = {}\n next_cid = 0\n if os.path.exists(state_path) and os.path.getsize(state_path) > 0:\n try:\n with open(state_path, 'rb') as f:\n state = pickle.load(f)\n if state.get('num_perm') != NUM_PERM or state.get('threshold') != threshold:\n print('[dedup] state params mismatch, starting fresh')\n else:\n raw_lsh_cache = state['lsh_cache']\n seen_raw = state.get('dedup_key_cache', {})\n raw_dedup_cache = {k: None for k in seen_raw} if isinstance(seen_raw, set) else (dict(seen_raw) if isinstance(seen_raw, dict) else {})\n oversized = len(raw_lsh_cache) > max_keys\n if oversized:\n # Snapshot grew beyond limit (no eviction in old versions).\n # Rebuild LSH index from the NEWEST max_keys clusters only to\n # avoid loading millions of entries into RAM all at once.\n keep_cids = set(list(raw_lsh_cache.keys())[-max_keys:])\n lsh_cache = {cid: mh for cid, mh in raw_lsh_cache.items() if cid in keep_cids}\n lsh_index = MinHashLSH(threshold=threshold, num_perm=NUM_PERM)\n for cid, mh in lsh_cache.items():\n try: lsh_index.insert(cid, mh)\n except Exception: pass\n dedup_key_cache = dict(list(raw_dedup_cache.items())[-max_keys:])\n print(f'[dedup] snapshot truncated {len(raw_lsh_cache)}→{len(lsh_cache)} clusters '\n f'(was over limit {max_keys})')\n del raw_lsh_cache, raw_dedup_cache, state\n _gc_module.collect()\n else:\n lsh_index = state['lsh_index']\n lsh_cache = raw_lsh_cache\n dedup_key_cache = raw_dedup_cache\n next_cid = (max(lsh_cache.keys()) + 1) if lsh_cache else 0\n print(f'[dedup] loaded snapshot: {len(lsh_cache)} clusters, {len(dedup_key_cache)} dedup_keys, next_cid={next_cid}')\n except Exception as e:\n print(f'[dedup] failed to load snapshot ({e}), starting fresh')\n lsh_index = None\n lsh_cache = {}\n dedup_key_cache = {}\n next_cid = 0\n if lsh_index is None:\n lsh_index = MinHashLSH(threshold=threshold, num_perm=NUM_PERM)\n lsh_cache = {}\n dedup_key_cache = {}\n next_cid = 0\n # Replay incremental append-log: only first-seen clusters/keys are stored there.\n if os.path.exists(append_path) and os.path.getsize(append_path) > 0:\n replayed = 0\n try:\n with open(append_path, 'rb') as f:\n while True:\n try:\n rec = pickle.load(f)\n except EOFError:\n break\n except Exception as _re:\n print(f'[dedup] append-log truncated at record {replayed} ({_re}), stopping replay')\n break\n if rec[0] == 'c':\n _, cid, mh = rec\n if cid not in lsh_cache:\n try:\n lsh_index.insert(cid, mh)\n except Exception:\n pass\n lsh_cache[cid] = mh\n if cid + 1 > next_cid:\n next_cid = cid + 1\n elif rec[0] == 'k':\n dedup_key_cache[rec[1]] = None\n replayed += 1\n except Exception as _e:\n print(f'[dedup] failed to replay append-log ({_e})')\n if replayed:\n print(f'[dedup] replayed {replayed} append-log records ({len(lsh_cache)} clusters, {len(dedup_key_cache)} keys)')\n return lsh_index, lsh_cache, dedup_key_cache, next_cid\n\ndef evict_oldest(lsh_index, lsh_cache, dedup_key_cache, max_keys):\n evicted_keys = evicted_clusters = 0\n excess = len(dedup_key_cache) - max_keys\n if excess > 0:\n for k in list(dedup_key_cache.keys())[:excess]:\n del dedup_key_cache[k]\n evicted_keys = excess\n excess = len(lsh_cache) - max_keys\n if excess > 0:\n for cid in list(lsh_cache.keys())[:excess]:\n try: lsh_index.remove(cid)\n except (KeyError, ValueError): pass\n del lsh_cache[cid]\n evicted_clusters = excess\n return evicted_keys, evicted_clusters\n\ndef dump_state_atomic(state_path, lsh_index, lsh_cache, dedup_key_cache, threshold, next_cluster_id):\n tmp = state_path + '.tmp'\n try:\n state = {\n 'lsh_index': lsh_index, 'lsh_cache': lsh_cache,\n 'dedup_key_cache': dedup_key_cache, 'next_cluster_id': next_cluster_id,\n 'num_perm': NUM_PERM, 'threshold': threshold,\n }\n with open(tmp, 'wb') as f:\n pickle.dump(state, f); f.flush(); os.fsync(f.fileno())\n os.replace(tmp, state_path)\n print(f'[dedup] state saved: {len(lsh_cache)} clusters, {len(dedup_key_cache)} dedup_keys')\n except Exception as e:\n print(f'[dedup] failed to save state: {e}')\n if os.path.exists(tmp):\n try: os.remove(tmp)\n except Exception: pass\n\ndef append_deltas(append_path, new_clusters, new_keys):\n # Persist ONLY this run's first-seen clusters + dedup_keys (append-only, O(new)).\n if not new_clusters and not new_keys:\n return\n try:\n with open(append_path, 'ab') as f:\n for _cid, _mh in new_clusters:\n pickle.dump(('c', _cid, _mh), f)\n for _dk in new_keys:\n pickle.dump(('k', _dk), f)\n f.flush(); os.fsync(f.fileno())\n except Exception as e:\n print(f'[dedup] failed to append deltas: {e}')\n\ndef compact_state(state_path, append_path, lsh_index, lsh_cache, dedup_key_cache, threshold, next_cluster_id):\n # Fold the append-log back into a fresh full snapshot, then drop the log.\n dump_state_atomic(state_path, lsh_index, lsh_cache, dedup_key_cache, threshold, next_cluster_id)\n try:\n if os.path.exists(append_path):\n os.remove(append_path)\n except Exception as e:\n print(f'[dedup] failed to truncate append-log: {e}')\n\n# ── Main ──────────────────────────────────────────────────────────────────────\n\nfiltered_alerts = inputs.get('filtered_alerts', [])\ninput_mode = inputs.get('input_mode', 'unknown')\ndedup_enabled = inputs.get('dedup_enabled', True)\nthreshold = float(inputs.get('dedup_threshold', 0.7))\nstrict_fields = inputs.get('strict_fields', ['sip', 'dip'])\nlsh_fields = inputs.get('lsh_fields', ['req_http_url', 'req_body', 'rsp_body'])\nmax_len = int(inputs.get('max_field_len', 500))\nmax_dedup_keys = int(inputs.get('max_dedup_keys', 100000))\nif max_dedup_keys < 1:\n max_dedup_keys = 100000\nstats = dict(inputs.get('stats', {}))\n\nif _mem.permutations is None:\n _mem.permutations = MinHash(num_perm=NUM_PERM, seed=MINHASH_SEED).permutations\n_permutations = _mem.permutations\nstate_path, lock_path, append_path = get_state_paths(threshold)\nevicted_keys = evicted_clusters = 0\n\nwith _mem.lock:\n # ── Load or reuse in-memory LSH state ────────────────────────────────────\n if dedup_enabled:\n disk_state_mtime = os.path.getmtime(state_path) if os.path.exists(state_path) else 0.0\n disk_append_mtime = os.path.getmtime(append_path) if os.path.exists(append_path) else 0.0\n cache_stale = (\n not _mem.initialized\n or _mem.threshold != threshold\n or disk_state_mtime > _mem.state_mtime + 0.5\n or disk_append_mtime > _mem.append_mtime + 0.5\n )\n if cache_stale:\n print(f'[dedup] cache miss (initialized={_mem.initialized}, '\n f'stale_by={disk_state_mtime - _mem.state_mtime:.1f}s), loading from disk')\n # Drop old references before load so GC can immediately reclaim\n # the ~649 MB state rather than waiting for the next cycle.\n old_lsh = _mem.lsh_index\n old_cache = _mem.lsh_cache\n _mem.lsh_index = None\n _mem.lsh_cache = {}\n _mem.dedup_key_cache = {}\n del old_lsh, old_cache\n _gc_module.collect()\n lsh_index, lsh_cache, dedup_key_cache, next_cluster_id = load_state(state_path, append_path, threshold, max_dedup_keys)\n if lsh_index is None:\n lsh_index = MinHashLSH(threshold=threshold, num_perm=NUM_PERM)\n lsh_cache = {}\n dedup_key_cache = {}\n next_cluster_id = 0\n _mem.lsh_index = lsh_index\n _mem.lsh_cache = lsh_cache\n _mem.dedup_key_cache = dedup_key_cache\n _mem.next_cluster_id = next_cluster_id\n _mem.threshold = threshold\n _mem.state_mtime = disk_state_mtime\n _mem.append_mtime = disk_append_mtime\n _mem.initialized = True\n else:\n print(f'[dedup] cache hit: {len(_mem.lsh_cache)} clusters, {len(_mem.dedup_key_cache)} keys')\n lsh_index = _mem.lsh_index\n lsh_cache = _mem.lsh_cache\n dedup_key_cache = _mem.dedup_key_cache\n next_cluster_id = _mem.next_cluster_id\n else:\n lsh_index, lsh_cache, dedup_key_cache, next_cluster_id = None, {}, {}, 0\n\n _cid_box = [next_cluster_id]\n _new_clusters = []\n def query_most_similar(minhash):\n sim_keys = lsh_index.query(minhash)\n if sim_keys:\n candidates = sim_keys[:100]\n sims = [minhash.jaccard(lsh_cache[k]) for k in candidates]\n return candidates[sims.index(max(sims))]\n cluster_id = _cid_box[0]\n _cid_box[0] += 1\n lsh_index.insert(cluster_id, minhash)\n lsh_cache[cluster_id] = minhash\n _new_clusters.append((cluster_id, minhash))\n return cluster_id\n\n enriched = []\n _new_keys = []\n for alert in filtered_alerts:\n alert = dict(alert)\n text_strict = '. '.join(str(alert.get(f, ''))[:max_len] for f in strict_fields)\n text_lsh = normalize_uri('. '.join(str(alert.get(f, ''))[:max_len] for f in lsh_fields))\n\n if not dedup_enabled:\n dk = hashlib.md5(f'{text_strict}. {text_lsh}'.encode('utf-8')).hexdigest()\n alert['_lsh_cluster_id'] = None\n alert['dedup_key'] = dk\n alert['is_duplicate'] = dk in dedup_key_cache\n dedup_key_cache[dk] = None\n enriched.append(alert)\n continue\n\n mh = gen_minhash(text_lsh.lower(), _permutations)\n cluster_id = query_most_similar(mh)\n alert['_lsh_cluster_id'] = cluster_id\n\n dk = hashlib.md5(f'{text_strict}. {cluster_id}'.encode('utf-8')).hexdigest()\n already = dk in dedup_key_cache\n if already:\n del dedup_key_cache[dk]\n else:\n _new_keys.append(dk)\n dedup_key_cache[dk] = None\n alert['dedup_key'] = dk\n alert['is_duplicate'] = already\n enriched.append(alert)\n\n if dedup_enabled:\n evicted_keys, evicted_clusters = evict_oldest(lsh_index, lsh_cache, dedup_key_cache, max_dedup_keys)\n if evicted_keys or evicted_clusters:\n print(f'[dedup] LRU eviction: dropped {evicted_keys} keys, {evicted_clusters} clusters')\n if len(lsh_cache) > LSH_CLUSTER_WARN_THRESHOLD or len(dedup_key_cache) > LSH_CLUSTER_WARN_THRESHOLD:\n print(f'[dedup] WARNING: persisted state holds {len(lsh_cache)} clusters '\n f'and {len(dedup_key_cache)} dedup_keys (warn={LSH_CLUSTER_WARN_THRESHOLD})')\n # Write to disk under file lock to protect against concurrent processes\n _lock_fh = acquire_lock(lock_path)\n try:\n # Incremental persistence: append ONLY this run's newly-created clusters\n # and first-seen dedup_keys (O(new) per message instead of O(N) full pickle).\n append_deltas(append_path, _new_clusters, _new_keys)\n # Periodic compaction: when the append-log grows comparable to the snapshot,\n # fold it into a fresh full snapshot and truncate the log (bounds disk + replay).\n _snap_size = os.path.getsize(state_path) if os.path.exists(state_path) else 0\n _app_size = os.path.getsize(append_path) if os.path.exists(append_path) else 0\n if (_snap_size == 0 and lsh_cache) or _app_size > max(_snap_size, 4 * 1024 * 1024):\n compact_state(state_path, append_path, lsh_index, lsh_cache, dedup_key_cache, threshold, _cid_box[0])\n print(f'[dedup] compacted append-log into snapshot ({len(lsh_cache)} clusters, {len(dedup_key_cache)} keys)')\n finally:\n release_lock(_lock_fh)\n _mem.next_cluster_id = _cid_box[0]\n _mem.state_mtime = os.path.getmtime(state_path) if os.path.exists(state_path) else _mem.state_mtime\n _mem.append_mtime = os.path.getmtime(append_path) if os.path.exists(append_path) else 0.0\n\n# ── Unique alerts (first seen per dedup_key) ──────────────────────────────────\nseen_keys = {}\nunique_alerts = []\nfor a in enriched:\n k = a['dedup_key']\n if k not in seen_keys:\n seen_keys[k] = a\n unique_alerts.append(a)\n\nfirst_seen_count = sum(1 for alert in enriched if not alert.get('is_duplicate'))\ndup_count = len(enriched) - first_seen_count\nprint(f'[dedup] input={len(filtered_alerts)}, enriched={len(enriched)}, unique={len(unique_alerts)}, duplicates={dup_count}')\n\nstats['metric_schema_version'] = 2\nstats['after_dedup_count'] = first_seen_count\nstats['unique_key_count'] = first_seen_count\nstats['dedup_removed_count'] = dup_count\nstats['dedup_ratio'] = round(dup_count / len(enriched), 4) if enriched else 0.0\nstats['dedup_state_persisted'] = bool(dedup_enabled)\nif dedup_enabled:\n stats['lsh_total_clusters'] = len(lsh_cache)\n stats['lsh_total_dedup_keys'] = len(dedup_key_cache)\n stats['lsh_max_dedup_keys'] = max_dedup_keys\n stats['lsh_evicted_keys'] = evicted_keys\n stats['lsh_evicted_clusters'] = evicted_clusters\n\nif dedup_enabled:\n summary = (\n f'stream_alert_denoise done: raw={stats.get(\"raw_count\", 0)}'\n f' -> normalized={stats.get(\"normalized_count\", 0)}'\n f' -> filtered={stats.get(\"after_filter_count\", 0)}'\n f' -> enriched={len(enriched)}, unique={len(unique_alerts)} (compression {stats[\"dedup_ratio\"]:.1%})'\n f' | clusters={len(lsh_cache)}, keys={len(dedup_key_cache)}, max={max_dedup_keys}'\n )\nelse:\n summary = (\n f'stream_alert_denoise done (dedup_enabled=False): '\n f'raw={stats.get(\"raw_count\", 0)}'\n f' -> filtered={stats.get(\"after_filter_count\", 0)}'\n f' -> enriched={len(enriched)}'\n )\nprint(f'[dedup] {summary}')\n\n# ── Write enriched alerts to JSONL (counter sidecar replaces O(N) scan) ───────\n# dedup_result_001.jsonl, 002.jsonl ... each starts with a file_header line.\n# A lightweight sidecar (.dedup_counter.json) tracks the active file seq/count\n# so we never scan the whole file on every execution.\nMAX_RECORDS_PER_FILE = 10000\n_JSONL_PREFIX = 'dedup_result'\n_COUNTER_FILE = '.dedup_counter.json'\n\ndef _get_counter(out_dir):\n path = os.path.join(out_dir, _COUNTER_FILE)\n try:\n with open(path, 'r', encoding='utf-8') as _f:\n d = json.load(_f)\n return int(d.get('seq', 0)), int(d.get('count', 0))\n except Exception:\n return 0, 0\n\ndef _set_counter(out_dir, seq, count):\n path = os.path.join(out_dir, _COUNTER_FILE)\n tmp = path + '.tmp'\n try:\n with open(tmp, 'w', encoding='utf-8') as _f:\n json.dump({'seq': seq, 'count': count}, _f)\n os.replace(tmp, path)\n except Exception:\n pass\n\ndef _find_active_file(out_dir):\n seq, count = _get_counter(out_dir)\n if seq > 0:\n path = os.path.join(out_dir, f'{_JSONL_PREFIX}_{seq:03d}.jsonl')\n if os.path.exists(path):\n return path, count, seq\n # Sidecar missing/stale: one-time recovery scan\n import glob as _glob\n existing = sorted(_glob.glob(os.path.join(out_dir, _JSONL_PREFIX + '_*.jsonl')))\n if not existing:\n return None, 0, 0\n latest = existing[-1]\n try:\n seq = int(os.path.basename(latest).replace(_JSONL_PREFIX + '_', '').replace('.jsonl', ''))\n except ValueError:\n seq = len(existing)\n count = 0\n try:\n with open(latest, 'r', encoding='utf-8') as _f:\n for _line in _f:\n if _line.strip() and '\"_type\"' not in _line:\n count += 1\n except Exception:\n pass\n _set_counter(out_dir, seq, count)\n return latest, count, seq\n\ndef _write_jsonl(out_dir, alerts, now):\n written = []\n active_path, active_count, seq = _find_active_file(out_dir)\n remaining = list(alerts)\n while remaining:\n available = MAX_RECORDS_PER_FILE - active_count\n if available <= 0 or active_path is None:\n seq += 1\n active_path = os.path.join(out_dir, f'{_JSONL_PREFIX}_{seq:03d}.jsonl')\n active_count = 0\n available = MAX_RECORDS_PER_FILE\n header = {\n '_type': 'file_header',\n 'created_at': now.isoformat(),\n 'date': now.strftime('%Y-%m-%d'),\n 'workflow': WORKFLOW_NAME,\n 'seq': seq,\n }\n with open(active_path, 'w', encoding='utf-8') as _hf:\n _hf.write(json.dumps(header, ensure_ascii=False) + '\\n')\n batch = remaining[:available]\n remaining = remaining[available:]\n with open(active_path, 'a', encoding='utf-8') as _af:\n for _alert in batch:\n _af.write(json.dumps(_alert, ensure_ascii=False) + '\\n')\n active_count += len(batch)\n if active_path not in written:\n written.append(active_path)\n if remaining:\n active_path = None\n active_count = 0\n if written:\n _set_counter(out_dir, seq, active_count)\n return written\n\n# Persist ONLY genuinely first-seen alerts (cross-batch is_duplicate=False).\n# NOTE: unique_alerts is only batch-local dedup; in single-alert syslog streaming\n# it is always length 1, so filtering by is_duplicate is what actually drops repeats.\n_persisted_alerts = [a for a in enriched if not a.get('is_duplicate')]\n_now = datetime.datetime.now()\ntry:\n _out_dir = get_output_dir()\n _written_paths = _write_jsonl(_out_dir, _persisted_alerts, _now) if _persisted_alerts else []\n _out_path = _written_paths[-1] if _written_paths else ''\n print(f'[dedup] wrote {len(_persisted_alerts)} first-seen records (skipped {len(enriched)-len(_persisted_alerts)} duplicates) -> {_written_paths}')\n stats['output_path'] = _out_path\n stats['output_paths'] = _written_paths\n outputs['output_path'] = _out_path\n outputs['output_paths'] = _written_paths\nexcept Exception as _we:\n import traceback\n print(f'[dedup] WARNING: failed to write JSONL: {_we}\\n{traceback.format_exc()}')\n outputs['output_path'] = ''\n outputs['output_paths'] = []\n\n# ── Outputs ───────────────────────────────────────────────────────────────────\n# Strip large body/header fields from in-memory run result to reduce flocks\n# run-history memory footprint. Full data is already persisted to JSONL.\n_HEAVY_OUTPUT_FIELDS = {\n 'net_http_reqs_header', 'net_http_resp_header',\n 'net_http_resp_body', 'net_http_reqs_body',\n 'net_http_resp_line', 'net_http_reqs_line',\n 'net_http_reqs_cookie',\n}\ndef _slim_alert(a):\n return {k: v for k, v in a.items() if k not in _HEAVY_OUTPUT_FIELDS}\noutputs['enriched_alerts'] = [_slim_alert(a) for a in enriched]\noutputs['unique_alerts'] = [_slim_alert(a) for a in unique_alerts]\noutputs['stats'] = stats\noutputs['dedup_summary'] = summary\noutputs['input_mode'] = input_mode\n\nif enriched:\n outputs['dedup_key'] = enriched[0].get('dedup_key', '')\n outputs['is_duplicate'] = enriched[0].get('is_duplicate', False)\nelse:\n outputs['dedup_key'] = ''\n outputs['is_duplicate'] = False\n" + "code": "\nimport os\nimport re\nimport sys\nimport gc as _gc_module\nimport json\nimport pickle\nimport hashlib\nimport datetime\nimport threading\nimport types\nfrom datasketch import MinHash, MinHashLSH\n\nIS_WINDOWS = sys.platform == 'win32'\nif IS_WINDOWS:\n import msvcrt # noqa: F401\nelse:\n import fcntl # noqa: F401\n\nMINHASH_SEED = 2024\nNUM_PERM = 128\nWORKFLOW_NAME = 'stream_alert_denoise'\nLSH_CLUSTER_WARN_THRESHOLD = 100000\n\n# ── Process-level in-memory LSH state cache ──────────────────────────────────\n# Previously the ~649 MB pickle was loaded from disk on EVERY syslog message,\n# causing linear memory growth (Python GC cannot free old objects fast enough\n# under high throughput). We now keep the live MinHashLSH + lsh_cache +\n# dedup_key_cache in sys.modules between exec() invocations. A\n# threading.Lock serialises concurrent workflow threads; the file lock\n# (fcntl/msvcrt) still guards cross-process disk writes.\n_MEM_CACHE_KEY = f'_flocks_lsh_cache_{WORKFLOW_NAME}'\nif _MEM_CACHE_KEY not in sys.modules:\n _m = types.ModuleType(_MEM_CACHE_KEY)\n _m.lsh_index = None\n _m.lsh_cache = {}\n _m.dedup_key_cache = {}\n _m.next_cluster_id = 0\n _m.threshold = None\n _m.state_mtime = 0.0\n _m.append_mtime = 0.0\n _m.initialized = False\n _m.permutations = None\n _m.lock = threading.Lock()\n sys.modules[_MEM_CACHE_KEY] = _m\n_mem = sys.modules[_MEM_CACHE_KEY]\nif not hasattr(_mem, 'append_mtime'):\n _mem.append_mtime = 0.0\nif not hasattr(_mem, 'permutations'):\n _mem.permutations = None\n\ndef normalize_uri(uri):\n uri = str(uri or '')\n uri = re.sub(r'\\d{4}-\\d{2}-\\d{2}', 'DATETIME', uri)\n uri = re.sub(r'[\\da-f]{8}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{4}-[\\da-f]{12}', 'UUID', uri, flags=re.IGNORECASE)\n uri = re.sub(r'(\\.\\./)+', 'TRAVERSAL', uri)\n uri = re.sub(r'\\bNULL\\b', 'NULL_REPLACED', uri)\n uri = re.sub(r'chr\\$\\d+\\$\\|\\|chr\\$\\d+\\$', 'CHR_SEQUENCE', uri)\n uri = re.sub(r'\\b\\d+={1,2}\\d+\\b', 'NUMBER_COMPARISON', uri)\n uri = re.sub(r'\\b[a-fA-F0-9]{32}\\b', 'HEXADECIMAL CHARACTERS', uri)\n return uri\n\ndef gen_minhash(text, permutations):\n shingles = [text[i:i+5] for i in range(len(text) - 4)]\n m = MinHash(num_perm=NUM_PERM, seed=MINHASH_SEED, permutations=permutations)\n for s in shingles:\n m.update(s.encode('utf-8'))\n return m\n\ndef get_state_paths(threshold):\n from flocks.config import Config\n flocks_root = Config().get_global().data_dir.parent\n state_dir = str(flocks_root / 'workspace' / 'workflows' / WORKFLOW_NAME)\n os.makedirs(state_dir, exist_ok=True)\n base = os.path.join(state_dir, f'lsh_state_np{NUM_PERM}_th{int(threshold * 100)}')\n return base + '.pkl', base + '.lock', base + '.append.log'\n\ndef get_output_dir():\n from flocks.config import Config\n from pathlib import Path\n flocks_root = Config().get_global().data_dir.parent\n date_str = datetime.datetime.now().strftime('%Y-%m-%d')\n out_dir = flocks_root / 'workspace' / 'workflows' / WORKFLOW_NAME / date_str\n out_dir.mkdir(parents=True, exist_ok=True)\n return str(out_dir)\n\ndef acquire_lock(lock_path):\n fh = open(lock_path, 'w+')\n try:\n if IS_WINDOWS:\n fh.write('L'); fh.flush(); fh.seek(0)\n while True:\n try:\n msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK, 1); break\n except OSError:\n continue\n else:\n fcntl.flock(fh.fileno(), fcntl.LOCK_EX)\n except BaseException:\n try:\n fh.close()\n except Exception:\n pass\n raise\n return fh\n\ndef release_lock(fh):\n try:\n if IS_WINDOWS:\n try:\n fh.seek(0); msvcrt.locking(fh.fileno(), msvcrt.LK_UNLCK, 1)\n except OSError:\n pass\n else:\n fcntl.flock(fh.fileno(), fcntl.LOCK_UN)\n finally:\n fh.close()\n\ndef load_state(state_path, append_path, threshold, max_keys=100000):\n lsh_index = None\n lsh_cache = {}\n dedup_key_cache = {}\n next_cid = 0\n if os.path.exists(state_path) and os.path.getsize(state_path) > 0:\n try:\n with open(state_path, 'rb') as f:\n state = pickle.load(f)\n if state.get('num_perm') != NUM_PERM or state.get('threshold') != threshold:\n print('[dedup] state params mismatch, starting fresh')\n else:\n raw_lsh_cache = state['lsh_cache']\n seen_raw = state.get('dedup_key_cache', {})\n raw_dedup_cache = {k: None for k in seen_raw} if isinstance(seen_raw, set) else (dict(seen_raw) if isinstance(seen_raw, dict) else {})\n oversized = len(raw_lsh_cache) > max_keys\n if oversized:\n # Snapshot grew beyond limit (no eviction in old versions).\n # Rebuild LSH index from the NEWEST max_keys clusters only to\n # avoid loading millions of entries into RAM all at once.\n keep_cids = set(list(raw_lsh_cache.keys())[-max_keys:])\n lsh_cache = {cid: mh for cid, mh in raw_lsh_cache.items() if cid in keep_cids}\n lsh_index = MinHashLSH(threshold=threshold, num_perm=NUM_PERM)\n for cid, mh in lsh_cache.items():\n try: lsh_index.insert(cid, mh)\n except Exception: pass\n dedup_key_cache = dict(list(raw_dedup_cache.items())[-max_keys:])\n print(f'[dedup] snapshot truncated {len(raw_lsh_cache)}→{len(lsh_cache)} clusters '\n f'(was over limit {max_keys})')\n del raw_lsh_cache, raw_dedup_cache, state\n _gc_module.collect()\n else:\n lsh_index = state['lsh_index']\n lsh_cache = raw_lsh_cache\n dedup_key_cache = raw_dedup_cache\n next_cid = (max(lsh_cache.keys()) + 1) if lsh_cache else 0\n print(f'[dedup] loaded snapshot: {len(lsh_cache)} clusters, {len(dedup_key_cache)} dedup_keys, next_cid={next_cid}')\n except Exception as e:\n print(f'[dedup] failed to load snapshot ({e}), starting fresh')\n lsh_index = None\n lsh_cache = {}\n dedup_key_cache = {}\n next_cid = 0\n if lsh_index is None:\n lsh_index = MinHashLSH(threshold=threshold, num_perm=NUM_PERM)\n lsh_cache = {}\n dedup_key_cache = {}\n next_cid = 0\n # Replay incremental append-log: only first-seen clusters/keys are stored there.\n if os.path.exists(append_path) and os.path.getsize(append_path) > 0:\n replayed = 0\n try:\n with open(append_path, 'rb') as f:\n while True:\n try:\n rec = pickle.load(f)\n except EOFError:\n break\n except Exception as _re:\n print(f'[dedup] append-log truncated at record {replayed} ({_re}), stopping replay')\n break\n if rec[0] == 'c':\n _, cid, mh = rec\n if cid not in lsh_cache:\n try:\n lsh_index.insert(cid, mh)\n except Exception:\n pass\n lsh_cache[cid] = mh\n if cid + 1 > next_cid:\n next_cid = cid + 1\n elif rec[0] == 'k':\n dedup_key_cache[rec[1]] = None\n replayed += 1\n except Exception as _e:\n print(f'[dedup] failed to replay append-log ({_e})')\n if replayed:\n print(f'[dedup] replayed {replayed} append-log records ({len(lsh_cache)} clusters, {len(dedup_key_cache)} keys)')\n return lsh_index, lsh_cache, dedup_key_cache, next_cid\n\ndef evict_oldest(lsh_index, lsh_cache, dedup_key_cache, max_keys):\n evicted_keys = evicted_clusters = 0\n excess = len(dedup_key_cache) - max_keys\n if excess > 0:\n for k in list(dedup_key_cache.keys())[:excess]:\n del dedup_key_cache[k]\n evicted_keys = excess\n excess = len(lsh_cache) - max_keys\n if excess > 0:\n for cid in list(lsh_cache.keys())[:excess]:\n try: lsh_index.remove(cid)\n except (KeyError, ValueError): pass\n del lsh_cache[cid]\n evicted_clusters = excess\n return evicted_keys, evicted_clusters\n\ndef dump_state_atomic(state_path, lsh_index, lsh_cache, dedup_key_cache, threshold, next_cluster_id):\n tmp = state_path + '.tmp'\n try:\n state = {\n 'lsh_index': lsh_index, 'lsh_cache': lsh_cache,\n 'dedup_key_cache': dedup_key_cache, 'next_cluster_id': next_cluster_id,\n 'num_perm': NUM_PERM, 'threshold': threshold,\n }\n with open(tmp, 'wb') as f:\n pickle.dump(state, f); f.flush(); os.fsync(f.fileno())\n os.replace(tmp, state_path)\n print(f'[dedup] state saved: {len(lsh_cache)} clusters, {len(dedup_key_cache)} dedup_keys')\n except Exception as e:\n print(f'[dedup] failed to save state: {e}')\n if os.path.exists(tmp):\n try: os.remove(tmp)\n except Exception: pass\n\ndef append_deltas(append_path, new_clusters, new_keys):\n # Persist ONLY this run's first-seen clusters + dedup_keys (append-only, O(new)).\n if not new_clusters and not new_keys:\n return\n try:\n with open(append_path, 'ab') as f:\n for _cid, _mh in new_clusters:\n pickle.dump(('c', _cid, _mh), f)\n for _dk in new_keys:\n pickle.dump(('k', _dk), f)\n f.flush(); os.fsync(f.fileno())\n except Exception as e:\n print(f'[dedup] failed to append deltas: {e}')\n\ndef compact_state(state_path, append_path, lsh_index, lsh_cache, dedup_key_cache, threshold, next_cluster_id):\n # Fold the append-log back into a fresh full snapshot, then drop the log.\n dump_state_atomic(state_path, lsh_index, lsh_cache, dedup_key_cache, threshold, next_cluster_id)\n try:\n if os.path.exists(append_path):\n os.remove(append_path)\n except Exception as e:\n print(f'[dedup] failed to truncate append-log: {e}')\n\n# ── Main ──────────────────────────────────────────────────────────────────────\n\nfiltered_alerts = inputs.get('filtered_alerts', [])\ninput_mode = inputs.get('input_mode', 'unknown')\ndedup_enabled = inputs.get('dedup_enabled', True)\nthreshold = float(inputs.get('dedup_threshold', 0.7))\nstrict_fields = inputs.get('strict_fields', ['sip', 'dip'])\nlsh_fields = inputs.get('lsh_fields', ['req_http_url', 'req_body', 'rsp_body'])\nmax_len = int(inputs.get('max_field_len', 500))\nmax_dedup_keys = int(inputs.get('max_dedup_keys', 100000))\nif max_dedup_keys < 1:\n max_dedup_keys = 100000\nstats = dict(inputs.get('stats', {}))\n\nif _mem.permutations is None:\n _mem.permutations = MinHash(num_perm=NUM_PERM, seed=MINHASH_SEED).permutations\n_permutations = _mem.permutations\nstate_path, lock_path, append_path = get_state_paths(threshold)\nevicted_keys = evicted_clusters = 0\n\nwith _mem.lock:\n # ── Load or reuse in-memory LSH state ────────────────────────────────────\n if dedup_enabled:\n disk_state_mtime = os.path.getmtime(state_path) if os.path.exists(state_path) else 0.0\n disk_append_mtime = os.path.getmtime(append_path) if os.path.exists(append_path) else 0.0\n cache_stale = (\n not _mem.initialized\n or _mem.threshold != threshold\n or disk_state_mtime > _mem.state_mtime + 0.5\n or disk_append_mtime > _mem.append_mtime + 0.5\n )\n if cache_stale:\n print(f'[dedup] cache miss (initialized={_mem.initialized}, '\n f'stale_by={disk_state_mtime - _mem.state_mtime:.1f}s), loading from disk')\n # Drop old references before load so GC can immediately reclaim\n # the ~649 MB state rather than waiting for the next cycle.\n old_lsh = _mem.lsh_index\n old_cache = _mem.lsh_cache\n _mem.lsh_index = None\n _mem.lsh_cache = {}\n _mem.dedup_key_cache = {}\n del old_lsh, old_cache\n _gc_module.collect()\n lsh_index, lsh_cache, dedup_key_cache, next_cluster_id = load_state(state_path, append_path, threshold, max_dedup_keys)\n if lsh_index is None:\n lsh_index = MinHashLSH(threshold=threshold, num_perm=NUM_PERM)\n lsh_cache = {}\n dedup_key_cache = {}\n next_cluster_id = 0\n _mem.lsh_index = lsh_index\n _mem.lsh_cache = lsh_cache\n _mem.dedup_key_cache = dedup_key_cache\n _mem.next_cluster_id = next_cluster_id\n _mem.threshold = threshold\n _mem.state_mtime = disk_state_mtime\n _mem.append_mtime = disk_append_mtime\n _mem.initialized = True\n else:\n print(f'[dedup] cache hit: {len(_mem.lsh_cache)} clusters, {len(_mem.dedup_key_cache)} keys')\n lsh_index = _mem.lsh_index\n lsh_cache = _mem.lsh_cache\n dedup_key_cache = _mem.dedup_key_cache\n next_cluster_id = _mem.next_cluster_id\n else:\n lsh_index, lsh_cache, dedup_key_cache, next_cluster_id = None, {}, {}, 0\n\n _cid_box = [next_cluster_id]\n _new_clusters = []\n def query_most_similar(minhash):\n sim_keys = lsh_index.query(minhash)\n if sim_keys:\n candidates = sim_keys[:100]\n sims = [minhash.jaccard(lsh_cache[k]) for k in candidates]\n return candidates[sims.index(max(sims))]\n cluster_id = _cid_box[0]\n _cid_box[0] += 1\n lsh_index.insert(cluster_id, minhash)\n lsh_cache[cluster_id] = minhash\n _new_clusters.append((cluster_id, minhash))\n return cluster_id\n\n enriched = []\n _new_keys = []\n for alert in filtered_alerts:\n alert = dict(alert)\n text_strict = '. '.join(str(alert.get(f, ''))[:max_len] for f in strict_fields)\n text_lsh = normalize_uri('. '.join(str(alert.get(f, ''))[:max_len] for f in lsh_fields))\n\n if not dedup_enabled:\n dk = hashlib.md5(f'{text_strict}. {text_lsh}'.encode('utf-8')).hexdigest()\n alert['_lsh_cluster_id'] = None\n alert['dedup_key'] = dk\n alert['is_duplicate'] = dk in dedup_key_cache\n dedup_key_cache[dk] = None\n enriched.append(alert)\n continue\n\n mh = gen_minhash(text_lsh.lower(), _permutations)\n cluster_id = query_most_similar(mh)\n alert['_lsh_cluster_id'] = cluster_id\n\n dk = hashlib.md5(f'{text_strict}. {cluster_id}'.encode('utf-8')).hexdigest()\n already = dk in dedup_key_cache\n if already:\n del dedup_key_cache[dk]\n else:\n _new_keys.append(dk)\n dedup_key_cache[dk] = None\n alert['dedup_key'] = dk\n alert['is_duplicate'] = already\n enriched.append(alert)\n\n if dedup_enabled:\n evicted_keys, evicted_clusters = evict_oldest(lsh_index, lsh_cache, dedup_key_cache, max_dedup_keys)\n if evicted_keys or evicted_clusters:\n print(f'[dedup] LRU eviction: dropped {evicted_keys} keys, {evicted_clusters} clusters')\n if len(lsh_cache) > LSH_CLUSTER_WARN_THRESHOLD or len(dedup_key_cache) > LSH_CLUSTER_WARN_THRESHOLD:\n print(f'[dedup] WARNING: persisted state holds {len(lsh_cache)} clusters '\n f'and {len(dedup_key_cache)} dedup_keys (warn={LSH_CLUSTER_WARN_THRESHOLD})')\n # Write to disk under file lock to protect against concurrent processes\n _lock_fh = acquire_lock(lock_path)\n try:\n # Incremental persistence: append ONLY this run's newly-created clusters\n # and first-seen dedup_keys (O(new) per message instead of O(N) full pickle).\n append_deltas(append_path, _new_clusters, _new_keys)\n # Periodic compaction: when the append-log grows comparable to the snapshot,\n # fold it into a fresh full snapshot and truncate the log (bounds disk + replay).\n _snap_size = os.path.getsize(state_path) if os.path.exists(state_path) else 0\n _app_size = os.path.getsize(append_path) if os.path.exists(append_path) else 0\n if (_snap_size == 0 and lsh_cache) or _app_size > max(_snap_size, 4 * 1024 * 1024):\n compact_state(state_path, append_path, lsh_index, lsh_cache, dedup_key_cache, threshold, _cid_box[0])\n print(f'[dedup] compacted append-log into snapshot ({len(lsh_cache)} clusters, {len(dedup_key_cache)} keys)')\n finally:\n release_lock(_lock_fh)\n _mem.next_cluster_id = _cid_box[0]\n _mem.state_mtime = os.path.getmtime(state_path) if os.path.exists(state_path) else _mem.state_mtime\n _mem.append_mtime = os.path.getmtime(append_path) if os.path.exists(append_path) else 0.0\n\n# ── Unique alerts (first seen per dedup_key) ──────────────────────────────────\nseen_keys = {}\nunique_alerts = []\nfor a in enriched:\n k = a['dedup_key']\n if k not in seen_keys:\n seen_keys[k] = a\n unique_alerts.append(a)\n\ndup_count = len(enriched) - len(unique_alerts)\nprint(f'[dedup] input={len(filtered_alerts)}, enriched={len(enriched)}, unique={len(unique_alerts)}, duplicates={dup_count}')\n\nstats['after_dedup_count'] = len(enriched)\nstats['unique_key_count'] = len(unique_alerts)\nstats['dedup_removed_count'] = dup_count\nstats['dedup_ratio'] = round(dup_count / len(enriched), 4) if enriched else 0.0\nstats['dedup_state_persisted'] = bool(dedup_enabled)\nif dedup_enabled:\n stats['lsh_total_clusters'] = len(lsh_cache)\n stats['lsh_total_dedup_keys'] = len(dedup_key_cache)\n stats['lsh_max_dedup_keys'] = max_dedup_keys\n stats['lsh_evicted_keys'] = evicted_keys\n stats['lsh_evicted_clusters'] = evicted_clusters\n\nif dedup_enabled:\n summary = (\n f'stream_alert_denoise done: raw={stats.get(\"raw_count\", 0)}'\n f' -> normalized={stats.get(\"normalized_count\", 0)}'\n f' -> filtered={stats.get(\"after_filter_count\", 0)}'\n f' -> enriched={len(enriched)}, unique={len(unique_alerts)} (compression {stats[\"dedup_ratio\"]:.1%})'\n f' | clusters={len(lsh_cache)}, keys={len(dedup_key_cache)}, max={max_dedup_keys}'\n )\nelse:\n summary = (\n f'stream_alert_denoise done (dedup_enabled=False): '\n f'raw={stats.get(\"raw_count\", 0)}'\n f' -> filtered={stats.get(\"after_filter_count\", 0)}'\n f' -> enriched={len(enriched)}'\n )\nprint(f'[dedup] {summary}')\n\n# ── Write enriched alerts to JSONL (counter sidecar replaces O(N) scan) ───────\n# dedup_result_001.jsonl, 002.jsonl ... each starts with a file_header line.\n# A lightweight sidecar (.dedup_counter.json) tracks the active file seq/count\n# so we never scan the whole file on every execution.\nMAX_RECORDS_PER_FILE = 10000\n_JSONL_PREFIX = 'dedup_result'\n_COUNTER_FILE = '.dedup_counter.json'\n\ndef _get_counter(out_dir):\n path = os.path.join(out_dir, _COUNTER_FILE)\n try:\n with open(path, 'r', encoding='utf-8') as _f:\n d = json.load(_f)\n return int(d.get('seq', 0)), int(d.get('count', 0))\n except Exception:\n return 0, 0\n\ndef _set_counter(out_dir, seq, count):\n path = os.path.join(out_dir, _COUNTER_FILE)\n tmp = path + '.tmp'\n try:\n with open(tmp, 'w', encoding='utf-8') as _f:\n json.dump({'seq': seq, 'count': count}, _f)\n os.replace(tmp, path)\n except Exception:\n pass\n\ndef _find_active_file(out_dir):\n seq, count = _get_counter(out_dir)\n if seq > 0:\n path = os.path.join(out_dir, f'{_JSONL_PREFIX}_{seq:03d}.jsonl')\n if os.path.exists(path):\n return path, count, seq\n # Sidecar missing/stale: one-time recovery scan\n import glob as _glob\n existing = sorted(_glob.glob(os.path.join(out_dir, _JSONL_PREFIX + '_*.jsonl')))\n if not existing:\n return None, 0, 0\n latest = existing[-1]\n try:\n seq = int(os.path.basename(latest).replace(_JSONL_PREFIX + '_', '').replace('.jsonl', ''))\n except ValueError:\n seq = len(existing)\n count = 0\n try:\n with open(latest, 'r', encoding='utf-8') as _f:\n for _line in _f:\n if _line.strip() and '\"_type\"' not in _line:\n count += 1\n except Exception:\n pass\n _set_counter(out_dir, seq, count)\n return latest, count, seq\n\ndef _write_jsonl(out_dir, alerts, now):\n written = []\n active_path, active_count, seq = _find_active_file(out_dir)\n remaining = list(alerts)\n while remaining:\n available = MAX_RECORDS_PER_FILE - active_count\n if available <= 0 or active_path is None:\n seq += 1\n active_path = os.path.join(out_dir, f'{_JSONL_PREFIX}_{seq:03d}.jsonl')\n active_count = 0\n available = MAX_RECORDS_PER_FILE\n header = {\n '_type': 'file_header',\n 'created_at': now.isoformat(),\n 'date': now.strftime('%Y-%m-%d'),\n 'workflow': WORKFLOW_NAME,\n 'seq': seq,\n }\n with open(active_path, 'w', encoding='utf-8') as _hf:\n _hf.write(json.dumps(header, ensure_ascii=False) + '\\n')\n batch = remaining[:available]\n remaining = remaining[available:]\n with open(active_path, 'a', encoding='utf-8') as _af:\n for _alert in batch:\n _af.write(json.dumps(_alert, ensure_ascii=False) + '\\n')\n active_count += len(batch)\n if active_path not in written:\n written.append(active_path)\n if remaining:\n active_path = None\n active_count = 0\n if written:\n _set_counter(out_dir, seq, active_count)\n return written\n\n# Persist ONLY genuinely first-seen alerts (cross-batch is_duplicate=False).\n# NOTE: unique_alerts is only batch-local dedup; in single-alert syslog streaming\n# it is always length 1, so filtering by is_duplicate is what actually drops repeats.\n_persisted_alerts = [a for a in enriched if not a.get('is_duplicate')]\n_now = datetime.datetime.now()\ntry:\n _out_dir = get_output_dir()\n _written_paths = _write_jsonl(_out_dir, _persisted_alerts, _now) if _persisted_alerts else []\n _out_path = _written_paths[-1] if _written_paths else ''\n print(f'[dedup] wrote {len(_persisted_alerts)} first-seen records (skipped {len(enriched)-len(_persisted_alerts)} duplicates) -> {_written_paths}')\n stats['output_path'] = _out_path\n stats['output_paths'] = _written_paths\n outputs['output_path'] = _out_path\n outputs['output_paths'] = _written_paths\nexcept Exception as _we:\n import traceback\n print(f'[dedup] WARNING: failed to write JSONL: {_we}\\n{traceback.format_exc()}')\n outputs['output_path'] = ''\n outputs['output_paths'] = []\n\n# ── Outputs ───────────────────────────────────────────────────────────────────\n# Strip large body/header fields from in-memory run result to reduce flocks\n# run-history memory footprint. Full data is already persisted to JSONL.\n_HEAVY_OUTPUT_FIELDS = {\n 'net_http_reqs_header', 'net_http_resp_header',\n 'net_http_resp_body', 'net_http_reqs_body',\n 'net_http_resp_line', 'net_http_reqs_line',\n 'net_http_reqs_cookie',\n}\ndef _slim_alert(a):\n return {k: v for k, v in a.items() if k not in _HEAVY_OUTPUT_FIELDS}\noutputs['enriched_alerts'] = [_slim_alert(a) for a in enriched]\noutputs['unique_alerts'] = [_slim_alert(a) for a in unique_alerts]\noutputs['stats'] = stats\noutputs['dedup_summary'] = summary\noutputs['input_mode'] = input_mode\n\nif enriched:\n outputs['dedup_key'] = enriched[0].get('dedup_key', '')\n outputs['is_duplicate'] = enriched[0].get('is_duplicate', False)\nelse:\n outputs['dedup_key'] = ''\n outputs['is_duplicate'] = False\n" } ], "edges": [ diff --git a/flocks/workflow/store.py b/flocks/workflow/store.py index 746655258..129d62131 100644 --- a/flocks/workflow/store.py +++ b/flocks/workflow/store.py @@ -37,16 +37,6 @@ "workflow_syslog_config/", ) _WORKFLOW_PREFIXES = _WORKFLOW_KV_PREFIXES + _WORKFLOW_TABLE_PREFIXES -_SOC_DENOISE_WORKFLOW_ID = "stream_alert_denoise" -# Version 3 means the persisted contribution was validated against the -# independently counted workflow input. Older v2 rollups may contain false -# zero ingress values and must not be treated as authoritative by the UI. -_SOC_METRIC_ROLLUP_SCHEMA_VERSION = 3 -_METRIC_RETENTION_MS = 35 * 24 * 60 * 60 * 1000 -# Idempotency keys only need to cover realistic completion retries. Keeping -# this table bounded avoids growth proportional to high-volume syslog traffic. -_METRIC_CONTRIBUTION_KEEP = 100_000 -_METRIC_PRUNE_INTERVAL_MS = 5 * 60 * 1000 _EXECUTION_UPSERT_SQL = """ INSERT OR REPLACE INTO workflow_executions (id, workflow_id, status, current_phase, current_node_id, current_node_type, @@ -65,7 +55,6 @@ class WorkflowStore: _init_pid: Optional[int] = None _db_path: Optional[Path] = None _completion_lock: Optional[asyncio.Lock] = None - _last_metric_prune_at: int = 0 @classmethod def get_db_path(cls) -> Path: @@ -99,7 +88,6 @@ async def init(cls) -> None: cls._initialized = False cls._init_pid = None cls._completion_lock = None - cls._last_metric_prune_at = 0 await Storage._ensure_init() db_path.parent.mkdir(parents=True, exist_ok=True) @@ -163,7 +151,6 @@ async def close(cls) -> None: cls._init_pid = None cls._db_path = None cls._completion_lock = None - cls._last_metric_prune_at = 0 @classmethod async def _db(cls) -> aiosqlite.Connection: @@ -367,275 +354,6 @@ def _execution_row( ), ) - @classmethod - def _pipeline_metric_contribution(cls, exec_data: Dict[str, Any]) -> Optional[Dict[str, Any]]: - workflow_id = str(exec_data.get("workflowId") or "") - if workflow_id != _SOC_DENOISE_WORKFLOW_ID: - return None - output = exec_data.get("outputResults") - output = output if isinstance(output, dict) else {} - stats = output.get("stats") if isinstance(output.get("stats"), dict) else {} - status = str(exec_data.get("status") or "").lower() - success = status in {"success", "completed"} - error_count = 0 if success else 1 - invalid_count = 0 - - def sequence_count(value: Any) -> Optional[int]: - if isinstance(value, str): - try: - value = json.loads(value) - except (TypeError, ValueError, json.JSONDecodeError): - return None - if isinstance(value, list): - return len(value) - if isinstance(value, dict): - if value.get("_type") in {"list", "tuple", "set"}: - count = cls._as_int(value.get("count")) - return count if count is not None and count >= 0 else None - if "data" in value: - value = value.get("data") - return len(value) if isinstance(value, list) else (1 if value else 0) - return 1 if value else 0 - - def input_alert_count() -> Optional[int]: - inputs = exec_data.get("inputParams") - if not isinstance(inputs, dict): - return None - - # Match the workflow's actual input priority. A syslog message is - # an alert only when its JSON payload can be decoded by the receive - # node; malformed/non-empty text must not be counted as accepted. - syslog_message = inputs.get("syslog_message") or inputs.get("syslog") - if isinstance(syslog_message, dict) and syslog_message.get("message"): - try: - parsed_syslog = json.loads(str(syslog_message["message"])) - except (TypeError, ValueError, json.JSONDecodeError): - pass - else: - if isinstance(parsed_syslog, dict): - return 1 - - # `alerts` shadows `alert_list` even when empty, matching the - # receive node's inputs.get('alerts', inputs.get('alert_list', [])). - for key in ("alerts", "alert_list"): - marker_key = f"_{key}_count" - if key not in inputs and marker_key not in inputs: - continue - materialized_count = sequence_count(inputs.get(key)) if key in inputs else None - if materialized_count is not None: - return materialized_count - marker = cls._as_int(inputs.get(marker_key)) - if marker is not None and marker >= 0: - return marker - return None - - # Compatibility fallback for pre-v3/compacted rows which stored - # the accepted batch under raw_alerts rather than the API field. - marker = cls._as_int(inputs.get("_raw_alerts_count")) - if marker is not None and marker >= 0: - return marker - if "raw_alerts" in inputs: - return sequence_count(inputs.get("raw_alerts")) - - # File inputs are intentionally unknown here: reading a user file - # while committing execution state would introduce I/O and TOCTOU - # races. Successful output metrics remain authoritative for them. - if inputs.get("alert_file"): - return None - return None - - input_count = input_alert_count() - input_params = exec_data.get("inputParams") - has_unverified_file_input = ( - isinstance(input_params, dict) and bool(input_params.get("alert_file")) - ) - - def metric_value(key: str) -> Optional[int]: - value = stats.get(key) - if isinstance(value, bool): - return None - try: - parsed = int(value) - except (TypeError, ValueError, OverflowError): - return None - return parsed if parsed >= 0 else None - - raw_count = metric_value("raw_count") - normalized_count = metric_value("normalized_count") - after_filter_count = metric_value("after_filter_count") - unique_count = metric_value("after_dedup_count") - reported_schema_version = metric_value("metric_schema_version") or 0 - required = (raw_count, normalized_count, after_filter_count, unique_count) - valid = success and all(value is not None for value in required) - if valid and not ( - raw_count >= normalized_count >= after_filter_count >= unique_count >= 0 - ): - valid = False - if valid and input_count is not None and raw_count != input_count: - valid = False - if valid and raw_count == 0 and input_count is None and has_unverified_file_input: - # A configured file cannot be safely re-read during persistence. - # Treat a zero output as unverifiable instead of claiming the file - # contained no alerts (it may have failed to load or changed). - valid = False - if valid and reported_schema_version < 2: - if raw_count == 1 and output.get("is_duplicate") is True: - unique_count = 0 - elif raw_count > 1: - valid = False - if success and not valid: - invalid_count = 1 - if not valid: - # Preserve independently verifiable ingress volume even when a - # workflow fails or emits malformed/inconsistent stage metrics. - # Downstream stages remain zero because they were not verified. - raw_count = input_count or 0 - normalized_count = after_filter_count = unique_count = 0 - - filter_removed_count = max(normalized_count - after_filter_count, 0) - duplicate_count = max(after_filter_count - unique_count, 0) - source_counts = {} - raw_source_counts = stats.get("normalize_type_counts") - if valid and isinstance(raw_source_counts, dict) and raw_source_counts.get("_type") != "dict": - for key, value in raw_source_counts.items(): - parsed = cls._as_int(value) - if parsed is not None and parsed > 0: - source_counts[str(key).strip().lower() or "unknown"] = parsed - source_covered_count = ( - normalized_count - if source_counts and sum(source_counts.values()) == normalized_count - else 0 - ) - started_at = cls._as_int(exec_data.get("startedAt")) or cls._now_ms() - bucket_start = started_at - (started_at % 60000) - return { - "execution_id": str(exec_data.get("id") or ""), - "workflow_id": workflow_id, - "bucket_start": bucket_start, - "raw_count": raw_count, - "normalized_count": normalized_count, - "after_filter_count": after_filter_count, - "unique_count": unique_count, - "filter_removed_count": filter_removed_count, - "duplicate_count": duplicate_count, - "source_counts": source_counts, - "source_covered_count": source_covered_count, - "success_count": 1 if success else 0, - "error_count": error_count, - "invalid_count": invalid_count, - "schema_version": _SOC_METRIC_ROLLUP_SCHEMA_VERSION, - } - - @classmethod - async def _record_pipeline_metric_contribution( - cls, - db: aiosqlite.Connection, - contribution: Optional[Dict[str, Any]], - ) -> None: - if not contribution or not contribution["execution_id"]: - return - now_ms = cls._now_ms() - cursor = await db.execute( - """ - INSERT OR IGNORE INTO workflow_metric_contributions - (execution_id, workflow_id, bucket_start, recorded_at) - VALUES (?, ?, ?, ?) - """, - ( - contribution["execution_id"], - contribution["workflow_id"], - contribution["bucket_start"], - now_ms, - ), - ) - if cursor.rowcount <= 0: - return - await db.execute( - """ - INSERT OR IGNORE INTO workflow_metric_meta - (workflow_id, coverage_started_at, updated_at) - VALUES (?, ?, ?) - """, - (contribution["workflow_id"], now_ms, now_ms), - ) - # A v2 bucket may already contain pre-reconciliation counts from the - # same minute. Do not let ON CONFLICT upgrade that mixed bucket to v3; - # replace only that obsolete derived bucket before adding verified data. - await db.execute( - "DELETE FROM workflow_metric_rollups " - "WHERE workflow_id = ? AND bucket_start = ? AND schema_version < ?", - ( - contribution["workflow_id"], - contribution["bucket_start"], - _SOC_METRIC_ROLLUP_SCHEMA_VERSION, - ), - ) - existing = await db.execute( - "SELECT source_counts FROM workflow_metric_rollups " - "WHERE workflow_id = ? AND bucket_start = ?", - (contribution["workflow_id"], contribution["bucket_start"]), - ) - existing_row = await existing.fetchone() - merged_sources = cls._json_loads(existing_row["source_counts"], {}) if existing_row else {} - if not isinstance(merged_sources, dict): - merged_sources = {} - for key, value in contribution["source_counts"].items(): - merged_sources[key] = max(cls._as_int(merged_sources.get(key)) or 0, 0) + value - await db.execute( - """ - INSERT INTO workflow_metric_rollups - (workflow_id, bucket_start, raw_count, normalized_count, - after_filter_count, unique_count, filter_removed_count, - duplicate_count, source_counts, source_covered_count, - success_count, error_count, invalid_count, schema_version, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(workflow_id, bucket_start) DO UPDATE SET - raw_count = raw_count + excluded.raw_count, - normalized_count = normalized_count + excluded.normalized_count, - after_filter_count = after_filter_count + excluded.after_filter_count, - unique_count = unique_count + excluded.unique_count, - filter_removed_count = filter_removed_count + excluded.filter_removed_count, - duplicate_count = duplicate_count + excluded.duplicate_count, - source_counts = excluded.source_counts, - source_covered_count = source_covered_count + excluded.source_covered_count, - success_count = success_count + excluded.success_count, - error_count = error_count + excluded.error_count, - invalid_count = invalid_count + excluded.invalid_count, - schema_version = MAX(schema_version, excluded.schema_version), - updated_at = excluded.updated_at - """, - ( - contribution["workflow_id"], - contribution["bucket_start"], - contribution["raw_count"], - contribution["normalized_count"], - contribution["after_filter_count"], - contribution["unique_count"], - contribution["filter_removed_count"], - contribution["duplicate_count"], - cls._json_dumps(merged_sources), - contribution["source_covered_count"], - contribution["success_count"], - contribution["error_count"], - contribution["invalid_count"], - contribution["schema_version"], - now_ms, - ), - ) - if now_ms - cls._last_metric_prune_at >= _METRIC_PRUNE_INTERVAL_MS: - cutoff = now_ms - _METRIC_RETENTION_MS - await db.execute( - "DELETE FROM workflow_metric_contributions WHERE execution_id IN (" - "SELECT execution_id FROM workflow_metric_contributions " - "ORDER BY recorded_at DESC LIMIT -1 OFFSET ?)", - (_METRIC_CONTRIBUTION_KEEP,), - ) - await db.execute( - "DELETE FROM workflow_metric_rollups WHERE bucket_start < ?", - (cutoff - (cutoff % 60000),), - ) - cls._last_metric_prune_at = now_ms - @classmethod async def upsert_execution(cls, exec_data: Dict[str, Any]) -> None: db = await cls._db() @@ -716,13 +434,11 @@ async def delete_executions_for_workflow(cls, workflow_id: str) -> int: @classmethod async def trim_executions(cls, workflow_id: str, *, keep: int) -> List[str]: - """Trim terminal history without deleting queued or running executions.""" db = await cls._db() async with db.execute( """ SELECT id FROM workflow_executions WHERE workflow_id = ? - AND status NOT IN ('running', 'queued', 'pending') ORDER BY started_at DESC, rowid DESC LIMIT -1 OFFSET ? """, @@ -793,7 +509,6 @@ async def complete_execution( """Atomically persist one final execution summary and its step batch.""" db = await cls._completion_db() exec_id, workflow_id, execution_row = cls._execution_row(exec_data) - metric_contribution = cls._pipeline_metric_contribution(exec_data) step_rows = cls._step_rows(exec_id, steps) lock = cls._completion_lock if lock is None: @@ -813,7 +528,6 @@ async def complete_execution( step_rows, ) await db.execute(_EXECUTION_UPSERT_SQL, execution_row) - await cls._record_pipeline_metric_contribution(db, metric_contribution) await db.commit() except BaseException: try: @@ -1147,38 +861,6 @@ async def kv_clear(cls, prefix: str) -> int: updated_at INTEGER ); -CREATE TABLE IF NOT EXISTS workflow_metric_rollups ( - workflow_id TEXT NOT NULL, - bucket_start INTEGER NOT NULL, - raw_count INTEGER NOT NULL DEFAULT 0, - normalized_count INTEGER NOT NULL DEFAULT 0, - after_filter_count INTEGER NOT NULL DEFAULT 0, - unique_count INTEGER NOT NULL DEFAULT 0, - filter_removed_count INTEGER NOT NULL DEFAULT 0, - duplicate_count INTEGER NOT NULL DEFAULT 0, - source_counts TEXT NOT NULL DEFAULT '{}', - source_covered_count INTEGER NOT NULL DEFAULT 0, - success_count INTEGER NOT NULL DEFAULT 0, - error_count INTEGER NOT NULL DEFAULT 0, - invalid_count INTEGER NOT NULL DEFAULT 0, - schema_version INTEGER NOT NULL DEFAULT 0, - updated_at INTEGER NOT NULL, - PRIMARY KEY (workflow_id, bucket_start) -); - -CREATE TABLE IF NOT EXISTS workflow_metric_contributions ( - execution_id TEXT PRIMARY KEY, - workflow_id TEXT NOT NULL, - bucket_start INTEGER NOT NULL, - recorded_at INTEGER NOT NULL -); - -CREATE TABLE IF NOT EXISTS workflow_metric_meta ( - workflow_id TEXT PRIMARY KEY, - coverage_started_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL -); - CREATE TABLE IF NOT EXISTS workflow_configs ( workflow_id TEXT NOT NULL, kind TEXT NOT NULL, @@ -1202,6 +884,4 @@ async def kv_clear(cls, prefix: str) -> int: "CREATE INDEX IF NOT EXISTS idx_workflow_executions_workflow_status ON workflow_executions(workflow_id, status)", "CREATE INDEX IF NOT EXISTS idx_workflow_executions_trigger ON workflow_executions(workflow_id, trigger_type, trigger_id)", "CREATE INDEX IF NOT EXISTS idx_workflow_execution_steps_exec_step ON workflow_execution_steps(exec_id, step_index)", - "CREATE INDEX IF NOT EXISTS idx_workflow_metric_rollups_workflow_bucket ON workflow_metric_rollups(workflow_id, bucket_start)", - "CREATE INDEX IF NOT EXISTS idx_workflow_metric_contributions_recorded ON workflow_metric_contributions(recorded_at)", ] diff --git a/tests/hub/test_soc_dashboard_schema.py b/tests/hub/test_soc_dashboard_schema.py index 53883a3c5..6983b811b 100644 --- a/tests/hub/test_soc_dashboard_schema.py +++ b/tests/hub/test_soc_dashboard_schema.py @@ -708,704 +708,6 @@ def test_soc_dashboard_activity_tolerates_empty_soc_db_with_workflow_events(tmp_ assert payload["workflowEvents"][0]["sessionId"] == "session-1" -def test_soc_dashboard_ai_tasks_use_authoritative_active_status(tmp_path: Path): - workflow_db = tmp_path / "workflow.db" - now_ms = int(datetime.now().timestamp() * 1000) - with sqlite3.connect(workflow_db) as conn: - conn.execute( - """ - CREATE TABLE workflow_executions ( - id TEXT PRIMARY KEY, - workflow_id TEXT NOT NULL, - status TEXT NOT NULL, - current_phase TEXT, - current_step_index INTEGER, - step_count INTEGER, - input_params TEXT NOT NULL DEFAULT '{}', - output_results TEXT NOT NULL DEFAULT '{}', - error_message TEXT, - started_at INTEGER NOT NULL, - finished_at INTEGER, - updated_at INTEGER, - payload TEXT NOT NULL DEFAULT '{}' - ) - """ - ) - rows = [ - ( - "triage-running", "stream_alert_triage", "running", "analysis", 2, 3, - "{}", json.dumps({"triage_results": [{"alert_name": "SSRF盲打探测"}]}), "", - now_ms - 2_000, None, now_ms, "{}", - ), - ( - "denoise-queued", "stream_alert_denoise", "queued", "queued", 0, 7, - json.dumps({"raw_alerts": [{"id": "a"}, {"id": "b"}]}), "{}", "", - now_ms - 1_000, None, 0, "{}", - ), - ( - "denoise-empty-completed", "stream_alert_denoise", "success", "completed", 7, 7, - "{}", - json.dumps({"stats": {"raw_count": 0, "normalized_count": 0, "after_filter_count": 0, "after_dedup_count": 0}}), - "", now_ms - 3_000, now_ms - 2_500, now_ms - 2_500, "{}", - ), - ( - "denoise-stale", "stream_alert_denoise", "running", "dedup", 3, 7, - "{}", "{}", "", now_ms - 3 * 60 * 60 * 1000, None, - now_ms - 3 * 60 * 60 * 1000, "{}", - ), - ] - conn.executemany( - "INSERT INTO workflow_executions VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - rows, - ) - conn.commit() - - handlers = _load_dashboard_handlers() - handlers.WORKFLOW_DB = workflow_db - - payload = handlers._get_ai_tasks() - - assert payload["connection"] == "online" - assert payload["summary"] == { - "active": 2, - "running": 1, - "waiting": 1, - "stale": 1, - "disabled": 0, - "returned": 2, - "truncated": False, - } - assert [task["executionId"] for task in payload["tasks"]] == [ - "triage-running", - "denoise-queued", - ] - assert payload["tasks"][0]["progress"]["label"] == "第 2/4 步" - assert payload["tasks"][1]["dataQuality"] == "pending" - assert payload["tasks"][1]["counts"]["raw"] == 2 - assert payload["tasks"][1]["rawCountSource"] == "workflow_input" - assert all(task["executionId"] != "denoise-empty-completed" for task in payload["tasks"]) - - -def test_soc_dashboard_ai_tasks_report_missing_workflow_database(tmp_path: Path): - handlers = _load_dashboard_handlers() - handlers.WORKFLOW_DB = tmp_path / "missing.db" - - payload = handlers._get_ai_tasks() - - assert payload["connection"] == "unavailable" - assert payload["reason"] == "workflow_db_missing" - assert payload["tasks"] == [] - - -def test_soc_dashboard_ai_task_input_count_supports_compacted_batches_and_empty_input(): - handlers = _load_dashboard_handlers() - - assert handlers._workflow_task_input_count( - {"alerts": {"_type": "list", "count": 2500}} - ) == 2500 - assert handlers._workflow_task_input_count({"_raw_alerts_count": 4000}) == 4000 - assert handlers._workflow_task_input_count({"alerts": []}) == 0 - assert handlers._workflow_task_input_count( - {"syslog_message": {"message": "not-json"}} - ) is None - assert handlers._workflow_task_input_count( - {"syslog_message": {"message": "[]"}} - ) is None - assert handlers._workflow_task_input_count( - { - "syslog_message": {"message": json.dumps({"id": "syslog-alert"})}, - "alerts": [{"id": index} for index in range(5)], - } - ) == 1 - assert handlers._workflow_task_input_count( - {"_alerts_count": 0, "alert_list": [{"id": "shadowed"}]} - ) == 0 - assert handlers._workflow_task_input_count( - {"_alerts_count": 0, "alerts": [{"id": "materialized"}]} - ) == 1 - - -def test_soc_dashboard_ai_task_reconciles_zero_initialized_output_with_input(): - handlers = _load_dashboard_handlers() - row = { - "id": "running-zero-output", - "output_results": json.dumps( - { - "stats": { - "raw_count": 0, - "normalized_count": 0, - "after_filter_count": 0, - "after_dedup_count": 0, - } - } - ), - "input_params": json.dumps({"alerts": [{"id": "real-alert"}]}), - "started_at": 1, - "updated_at": 1, - "finished_at": 0, - "current_phase": "receive", - "current_step_index": 1, - "step_count": 5, - "payload": "{}", - "error_message": "", - } - - task = handlers._workflow_task_row( - row, - "stream_alert_denoise", - "running", - ) - - assert task["counts"]["raw"] == 1 - assert task["rawCountSource"] == "workflow_input" - assert task["dataQuality"] == "pending" - assert task["emptyBatch"] is False - assert task["emptyInput"] is False - - empty_row = { - **row, - "id": "running-empty-input", - "input_params": json.dumps({"alerts": []}), - } - empty_task = handlers._workflow_task_row( - empty_row, - "stream_alert_denoise", - "running", - ) - assert empty_task["counts"]["raw"] == 0 - assert empty_task["rawCountSource"] == "workflow_input" - assert empty_task["dataQuality"] == "empty-input" - assert empty_task["emptyBatch"] is True - assert empty_task["emptyInput"] is True - - -def test_soc_dashboard_ai_task_rejects_impossible_stage_counts(): - handlers = _load_dashboard_handlers() - output = json.dumps( - { - "stats": { - "raw_count": 1, - "normalized_count": 2, - "after_filter_count": 3, - "after_dedup_count": 4, - } - } - ) - - _, running_quality = handlers._workflow_task_metrics(output, "running") - _, completed_quality = handlers._workflow_task_metrics(output, "completed") - - assert running_quality == "pending" - assert completed_quality == "invalid" - - -def test_soc_dashboard_marks_missing_workflow_metrics_as_unavailable(tmp_path: Path): - handlers = _load_dashboard_handlers() - handlers.WORKFLOW_DB = tmp_path / "missing-workflow.db" - - stats = handlers._get_workflow_denoise_stats( - "stream_alert_denoise", - 1_800_000, - 1_803_600, - force=True, - ) - - assert stats["dataAvailable"] is False - assert stats["metricsAvailable"] is False - assert stats["dataQuality"] == "unavailable" - assert stats["unavailableReason"] == "workflow_db_missing" - assert stats["rawCount"] is None - assert stats["callCount"] is None - - -def test_soc_dashboard_does_not_serve_fresh_cache_after_workflow_db_disappears( - tmp_path: Path, -): - workflow_db = tmp_path / "workflow.db" - start_time = 1_800_000 - start_ms = start_time * 1000 - with sqlite3.connect(workflow_db) as conn: - conn.execute( - "CREATE TABLE workflow_metric_meta " - "(workflow_id TEXT PRIMARY KEY, coverage_started_at INTEGER, updated_at INTEGER)" - ) - conn.execute( - """ - CREATE TABLE workflow_metric_rollups ( - workflow_id TEXT, bucket_start INTEGER, raw_count INTEGER, - normalized_count INTEGER, after_filter_count INTEGER, unique_count INTEGER, - filter_removed_count INTEGER, duplicate_count INTEGER, source_counts TEXT, - source_covered_count INTEGER, success_count INTEGER, error_count INTEGER, - invalid_count INTEGER, schema_version INTEGER, updated_at INTEGER, - PRIMARY KEY (workflow_id, bucket_start) - ) - """ - ) - conn.execute( - "INSERT INTO workflow_metric_meta VALUES (?, ?, ?)", - ("stream_alert_denoise", start_ms, start_ms), - ) - conn.execute( - "INSERT INTO workflow_metric_rollups VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - ( - "stream_alert_denoise", start_ms, 1, 1, 1, 1, 0, 0, - '{"tdp": 1}', 1, 1, 0, 0, 3, start_ms, - ), - ) - conn.commit() - - handlers = _load_dashboard_handlers() - handlers.WORKFLOW_DB = workflow_db - handlers._workflow_stats_cache.clear() - assert handlers._get_workflow_denoise_stats( - "stream_alert_denoise", start_time, start_time + 60, force=True - )["dataQuality"] == "complete" - - workflow_db.unlink() - stats = handlers._get_workflow_denoise_stats( - "stream_alert_denoise", start_time, start_time + 60 - ) - - assert stats["dataAvailable"] is False - assert stats["dataQuality"] == "unavailable" - assert stats["rawCount"] is None - - -def test_soc_dashboard_unbounded_rollup_requires_proven_history_start(tmp_path: Path): - workflow_db = tmp_path / "workflow.db" - bucket_ms = 2_000_000_000_000 - with sqlite3.connect(workflow_db) as conn: - conn.execute( - "CREATE TABLE workflow_metric_meta " - "(workflow_id TEXT PRIMARY KEY, coverage_started_at INTEGER, updated_at INTEGER)" - ) - conn.execute( - """ - CREATE TABLE workflow_metric_rollups ( - workflow_id TEXT, bucket_start INTEGER, raw_count INTEGER, - normalized_count INTEGER, after_filter_count INTEGER, unique_count INTEGER, - filter_removed_count INTEGER, duplicate_count INTEGER, source_counts TEXT, - source_covered_count INTEGER, success_count INTEGER, error_count INTEGER, - invalid_count INTEGER, schema_version INTEGER, updated_at INTEGER, - PRIMARY KEY (workflow_id, bucket_start) - ) - """ - ) - conn.execute( - "INSERT INTO workflow_metric_meta VALUES (?, ?, ?)", - ("stream_alert_denoise", bucket_ms, bucket_ms), - ) - conn.execute( - "INSERT INTO workflow_metric_rollups VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - ( - "stream_alert_denoise", bucket_ms, 7, 7, 6, 5, 1, 1, - '{"tdp": 7}', 7, 1, 0, 0, 3, bucket_ms, - ), - ) - conn.commit() - - handlers = _load_dashboard_handlers() - handlers.WORKFLOW_DB = workflow_db - - stats = handlers._get_workflow_metric_rollups("stream_alert_denoise", 0, 0) - - assert stats["coverageComplete"] is False - assert stats["dataQuality"] == "partial" - - -def test_soc_dashboard_hides_triage_metrics_when_soc_database_is_missing(tmp_path: Path): - workflow_db = tmp_path / "workflow.db" - start_time = 1_788_739_200 - start_ms = start_time * 1000 - with sqlite3.connect(workflow_db) as conn: - conn.execute( - "CREATE TABLE workflow_metric_meta " - "(workflow_id TEXT PRIMARY KEY, coverage_started_at INTEGER, updated_at INTEGER)" - ) - conn.execute( - """ - CREATE TABLE workflow_metric_rollups ( - workflow_id TEXT, bucket_start INTEGER, raw_count INTEGER, - normalized_count INTEGER, after_filter_count INTEGER, unique_count INTEGER, - filter_removed_count INTEGER, duplicate_count INTEGER, source_counts TEXT, - source_covered_count INTEGER, success_count INTEGER, error_count INTEGER, - invalid_count INTEGER, schema_version INTEGER, updated_at INTEGER, - PRIMARY KEY (workflow_id, bucket_start) - ) - """ - ) - conn.execute( - "INSERT INTO workflow_metric_meta VALUES (?, ?, ?)", - ("stream_alert_denoise", start_ms - 60_000, start_ms), - ) - conn.execute( - "INSERT INTO workflow_metric_rollups VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - ( - "stream_alert_denoise", start_ms, 10, 10, 8, 4, 2, 4, - '{"tdp": 10}', 10, 1, 0, 0, 3, start_ms, - ), - ) - conn.commit() - - handlers = _load_dashboard_handlers() - handlers.WORKFLOW_DB = workflow_db - handlers.DEFAULT_SQLITE_DB = tmp_path / "missing-soc.db" - handlers.USAGE_DB = tmp_path / "missing-usage.db" - handlers._schema_ready.clear() - handlers._stats_response_cache.clear() - handlers._workflow_stats_cache.clear() - - stats = handlers._get_stats( - { - "startTime": str(start_time), - "endTime": str(start_time + 3600), - "force": "true", - } - ) - - assert stats["sourceStatus"]["metricQuality"]["status"] == "complete" - assert stats["denoise"]["totalRaw"] == 10 - assert stats["sourceStatus"]["triageQuality"] == { - "status": "unavailable", - "dataAvailable": False, - "metricsAvailable": False, - "unavailableReason": "soc_db_missing", - "recordCount": None, - "dataSource": "unavailable", - } - assert stats["triage"]["totalRecords"] is None - assert stats["triage"]["attackTotal"] is None - assert stats["closedLoop"]["resolutionRate"] is None - assert stats["pipeline"]["attackRate"] is None - assert stats["timeline"]["triageTotal"] == [] - assert stats["sourceStatus"]["missing"][0]["reason"] == "soc_db_missing" - - -def test_soc_dashboard_keeps_real_zero_for_healthy_empty_soc_database(tmp_path: Path): - soc_db = tmp_path / "soc.db" - with sqlite3.connect(soc_db) as conn: - conn.execute( - """ - CREATE TABLE alert_records ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - record_json TEXT NOT NULL, - asset_date TEXT NOT NULL, - event_time INTEGER NOT NULL - ) - """ - ) - conn.commit() - handlers = _load_dashboard_handlers() - handlers.DEFAULT_SQLITE_DB = soc_db - handlers.WORKFLOW_DB = tmp_path / "missing-workflow.db" - handlers.USAGE_DB = tmp_path / "missing-usage.db" - handlers._schema_ready.clear() - handlers._stats_response_cache.clear() - handlers._workflow_stats_cache.clear() - - stats = handlers._get_stats( - {"startDate": "2026-09-07", "endDate": "2026-09-07", "force": "true"} - ) - - assert stats["sourceStatus"]["triageQuality"]["status"] == "complete" - assert stats["sourceStatus"]["triageQuality"]["metricsAvailable"] is True - assert stats["triage"]["totalRecords"] == 0 - assert stats["triage"]["attackTotal"] == 0 - assert stats["closedLoop"]["resolutionRate"] == 0 - - soc_db.unlink() - refreshed = handlers._get_stats( - {"startDate": "2026-09-07", "endDate": "2026-09-07"} - ) - assert refreshed["cacheHit"] is False - assert refreshed["sourceStatus"]["triageQuality"]["status"] == "unavailable" - assert refreshed["triage"]["totalRecords"] is None - - -def test_soc_dashboard_rejects_incomplete_soc_fact_schema(tmp_path: Path): - soc_db = tmp_path / "soc.db" - with sqlite3.connect(soc_db) as conn: - conn.execute( - "CREATE TABLE alert_records " - "(asset_date TEXT, event_time INTEGER, record_json TEXT)" - ) - conn.execute( - "CREATE TABLE soc_dashboard_alert_facts " - "(asset_date TEXT, event_time INTEGER)" - ) - conn.commit() - - handlers = _load_dashboard_handlers() - handlers.DEFAULT_SQLITE_DB = soc_db - - sources, quality = handlers._find_sqlite_sources_with_quality( - "2026-09-01", - "2026-09-07", - ) - - assert sources == [] - assert quality["metricsAvailable"] is False - assert quality["unavailableReason"] == "soc_dashboard_schema_unavailable" - - -def test_soc_dashboard_reads_persisted_denoise_metric_rollups(tmp_path: Path): - workflow_db = tmp_path / "workflow.db" - start_time = 1_800_000 - end_time = start_time + 3600 - start_ms = start_time * 1000 - with sqlite3.connect(workflow_db) as conn: - conn.execute( - "CREATE TABLE workflow_metric_meta " - "(workflow_id TEXT PRIMARY KEY, coverage_started_at INTEGER, updated_at INTEGER)" - ) - conn.execute( - """ - CREATE TABLE workflow_metric_rollups ( - workflow_id TEXT, - bucket_start INTEGER, - raw_count INTEGER, - normalized_count INTEGER, - after_filter_count INTEGER, - unique_count INTEGER, - filter_removed_count INTEGER, - duplicate_count INTEGER, - source_counts TEXT, - source_covered_count INTEGER, - success_count INTEGER, - error_count INTEGER, - invalid_count INTEGER, - schema_version INTEGER, - updated_at INTEGER, - PRIMARY KEY (workflow_id, bucket_start) - ) - """ - ) - conn.execute( - "INSERT INTO workflow_metric_meta VALUES (?, ?, ?)", - ("stream_alert_denoise", start_ms - 60_000, start_ms + 120_000), - ) - conn.executemany( - "INSERT INTO workflow_metric_rollups VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - [ - # Pre-v3 rows were not reconciled with input volume and must - # not contaminate authoritative dashboard totals. - ("stream_alert_denoise", start_ms + 120_000, 999, 999, 999, 999, 0, 0, '{"tdp": 999}', 999, 1, 0, 0, 2, start_ms + 120_000), - ("stream_alert_denoise", start_ms, 10, 10, 8, 3, 2, 5, '{"tdp": 10}', 10, 1, 0, 0, 3, start_ms), - ("stream_alert_denoise", start_ms + 60_000, 4, 4, 4, 1, 0, 3, '{"skyeye": 4}', 4, 1, 0, 0, 3, start_ms + 60_000), - ], - ) - conn.commit() - - handlers = _load_dashboard_handlers() - handlers.WORKFLOW_DB = workflow_db - - stats = handlers._get_workflow_denoise_stats( - "stream_alert_denoise", - start_time, - end_time, - force=True, - ) - - assert stats["metricsAvailable"] is True - assert stats["sourceMetricsAvailable"] is True - assert stats["dataQuality"] == "complete" - assert stats["rawCount"] == 14 - assert stats["normalizedCount"] == 14 - assert stats["afterFilterCount"] == 12 - assert stats["uniqueCount"] == 4 - assert stats["filterRemovedCount"] == 2 - assert stats["duplicateCount"] == 8 - assert stats["sourceCounts"] == {"tdp": 10, "skyeye": 4} - assert stats["sourceCoverageRate"] == 1 - assert sum(stats["seriesRaw"]) == 14 - assert sum(stats["seriesUnique"]) == 4 - - -def test_soc_dashboard_separates_core_metric_and_source_coverage_quality(tmp_path: Path): - workflow_db = tmp_path / "workflow.db" - start_time = 1_800_000 - end_time = start_time + 3600 - start_ms = start_time * 1000 - with sqlite3.connect(workflow_db) as conn: - conn.execute( - "CREATE TABLE workflow_metric_meta " - "(workflow_id TEXT PRIMARY KEY, coverage_started_at INTEGER, updated_at INTEGER)" - ) - conn.execute( - """ - CREATE TABLE workflow_metric_rollups ( - workflow_id TEXT, bucket_start INTEGER, raw_count INTEGER, - normalized_count INTEGER, after_filter_count INTEGER, unique_count INTEGER, - filter_removed_count INTEGER, duplicate_count INTEGER, source_counts TEXT, - source_covered_count INTEGER, success_count INTEGER, error_count INTEGER, - invalid_count INTEGER, schema_version INTEGER, updated_at INTEGER, - PRIMARY KEY (workflow_id, bucket_start) - ) - """ - ) - conn.execute( - "INSERT INTO workflow_metric_meta VALUES (?, ?, ?)", - ("stream_alert_denoise", start_ms - 60_000, start_ms), - ) - conn.execute( - "INSERT INTO workflow_metric_rollups VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - ( - "stream_alert_denoise", start_ms, 10, 6, 5, 4, 1, 1, - '{"tdp": 6}', 6, 1, 0, 0, 3, start_ms, - ), - ) - conn.commit() - - handlers = _load_dashboard_handlers() - handlers.WORKFLOW_DB = workflow_db - - stats = handlers._get_workflow_denoise_stats( - "stream_alert_denoise", start_time, end_time, force=True - ) - - assert stats["metricsAvailable"] is True - assert stats["sourceMetricsAvailable"] is False - assert stats["dataQuality"] == "partial" - assert stats["sourceCoverageRate"] == 0.6 - assert stats["unprocessedInputCount"] == 4 - - with sqlite3.connect(workflow_db) as conn: - conn.execute( - "UPDATE workflow_metric_rollups SET error_count = 1 " - "WHERE workflow_id = ?", - ("stream_alert_denoise",), - ) - conn.commit() - - failed_stats = handlers._get_workflow_denoise_stats( - "stream_alert_denoise", start_time, end_time, force=True - ) - - assert failed_stats["metricsAvailable"] is False - assert failed_stats["dataQuality"] == "partial" - - -def test_soc_dashboard_activity_cursor_tracks_actual_poll_window(): - handlers = _load_dashboard_handlers() - - encoded = handlers._encode_activity_cursor(12, 34, 5_000) - decoded = handlers._decode_activity_cursor(encoded) - - assert decoded == {"lastRowId": 12, "lastActivityId": 34, "polledAt": 5_000} - - -def test_soc_dashboard_activity_rate_uses_actual_poll_window(): - handlers = _load_dashboard_handlers() - settings = { - "table": "alerts", - "activity_table": "activity", - "record_column": "record_json", - "event_time_column": "event_time", - } - with sqlite3.connect(":memory:") as conn: - conn.row_factory = sqlite3.Row - conn.execute("CREATE TABLE alerts (record_json TEXT, event_time INTEGER)") - conn.execute( - "CREATE TABLE activity " - "(activity_id INTEGER PRIMARY KEY, alert_row_id INTEGER, event_time INTEGER, record_json TEXT)" - ) - conn.executemany( - "INSERT INTO alerts VALUES (?, ?)", - [(json.dumps({"id": index}), 100 + index) for index in range(4)], - ) - - _, _, batch = handlers._activity_rows( - conn, - settings, - last_row_id=0, - last_activity_id=0, - latest_row_id=4, - latest_activity_id=0, - limit=10, - window_ms=8_000, - ) - - assert batch["receivedCount"] == 4 - assert batch["windowMs"] == 8_000 - assert batch["ratePerSecond"] == 0.5 - - -def test_soc_dashboard_does_not_replace_full_window_with_partial_rollup(tmp_path: Path): - workflow_db = tmp_path / "workflow.db" - start_time = 1_800_000 - end_time = start_time + 3600 - start_ms = start_time * 1000 - coverage_ms = start_ms + 180_000 - with sqlite3.connect(workflow_db) as conn: - conn.execute( - "CREATE TABLE workflow_metric_meta " - "(workflow_id TEXT PRIMARY KEY, coverage_started_at INTEGER, updated_at INTEGER)" - ) - conn.execute( - """ - CREATE TABLE workflow_metric_rollups ( - workflow_id TEXT, - bucket_start INTEGER, - raw_count INTEGER, - normalized_count INTEGER, - after_filter_count INTEGER, - unique_count INTEGER, - filter_removed_count INTEGER, - duplicate_count INTEGER, - source_counts TEXT, - source_covered_count INTEGER, - success_count INTEGER, - error_count INTEGER, - invalid_count INTEGER, - schema_version INTEGER, - updated_at INTEGER, - PRIMARY KEY (workflow_id, bucket_start) - ) - """ - ) - conn.execute( - "CREATE TABLE workflow_executions " - "(workflow_id TEXT, status TEXT, started_at INTEGER)" - ) - conn.execute( - "INSERT INTO workflow_metric_meta VALUES (?, ?, ?)", - ("stream_alert_denoise", coverage_ms, coverage_ms), - ) - conn.execute( - "INSERT INTO workflow_metric_rollups VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - ( - "stream_alert_denoise", coverage_ms, 2, 2, 2, 1, 0, 1, - '{"tdp": 2}', 2, 1, 0, 0, 3, coverage_ms, - ), - ) - conn.executemany( - "INSERT INTO workflow_executions VALUES (?, ?, ?)", - [ - ("stream_alert_denoise", "success", start_ms + offset) - for offset in (60_000, 120_000, 180_000) - ], - ) - conn.commit() - - handlers = _load_dashboard_handlers() - handlers.WORKFLOW_DB = workflow_db - - stats = handlers._get_workflow_denoise_stats( - "stream_alert_denoise", - start_time, - end_time, - force=True, - ) - - assert stats["metricsAvailable"] is False - assert stats["dataQuality"] == "legacy-partial" - assert stats["coverageComplete"] is False - assert stats["coverageStartedAt"] == coverage_ms - assert stats["callCount"] == 3 - assert stats["rawCount"] == 3 - assert stats["shadowMetricsAvailable"] is True - - def test_soc_dashboard_task_center_summarizes_tasks_and_workflows(tmp_path: Path, monkeypatch): tasks_db = tmp_path / "tasks.db" today_at_1100 = datetime.now().astimezone().replace( diff --git a/tests/workflow/test_workflow_store.py b/tests/workflow/test_workflow_store.py index 2fc1965b4..b33234648 100644 --- a/tests/workflow/test_workflow_store.py +++ b/tests/workflow/test_workflow_store.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -import json import os from pathlib import Path from unittest.mock import AsyncMock @@ -25,7 +24,6 @@ def _reset_state() -> None: WorkflowStore._init_pid = None WorkflowStore._db_path = None WorkflowStore._completion_lock = None - WorkflowStore._last_metric_prune_at = 0 @pytest.fixture(autouse=True) @@ -100,368 +98,6 @@ async def test_workflow_store_records_execution_steps_config_and_kv() -> None: assert await WorkflowStore.kv_list_keys("workflow_runtime/") == ["workflow_runtime/wf-1"] -@pytest.mark.asyncio -async def test_workflow_store_trim_keeps_active_executions() -> None: - await WorkflowStore.init() - for index, status in enumerate(("running", "queued", "pending", "success", "failed", "success")): - await WorkflowStore.upsert_execution( - { - "id": f"exec-{index}", - "workflowId": "wf-trim", - "status": status, - "startedAt": 100 + index, - } - ) - - trimmed = await WorkflowStore.trim_executions("wf-trim", keep=1) - remaining = await WorkflowStore.list_executions("wf-trim", limit=20) - - assert set(trimmed) == {"exec-3", "exec-4"} - assert {row["id"] for row in remaining} == {"exec-0", "exec-1", "exec-2", "exec-5"} - - -@pytest.mark.asyncio -async def test_workflow_store_rolls_up_denoise_metrics_idempotently() -> None: - await WorkflowStore.init() - now_ms = WorkflowStore._now_ms() - execution = { - "id": "denoise-exec-1", - "workflowId": "stream_alert_denoise", - "status": "success", - "startedAt": now_ms, - "finishedAt": now_ms + 1_000, - "outputResults": { - "stats": { - "metric_schema_version": 2, - "raw_count": 10, - "normalized_count": 10, - "after_filter_count": 6, - "after_dedup_count": 2, - "normalize_type_counts": {"tdp": 8, "skyeye": 2}, - } - }, - } - - await WorkflowStore.complete_execution(execution, []) - await WorkflowStore.complete_execution(execution, []) - - db = await WorkflowStore.raw_db() - async with db.execute( - "SELECT raw_count, normalized_count, after_filter_count, unique_count, " - "filter_removed_count, duplicate_count, source_counts, source_covered_count, " - "success_count, error_count, invalid_count, schema_version " - "FROM workflow_metric_rollups WHERE workflow_id = ?", - ("stream_alert_denoise",), - ) as cursor: - row = await cursor.fetchone() - async with db.execute("SELECT COUNT(*) AS total FROM workflow_metric_contributions") as cursor: - contribution_count = (await cursor.fetchone())["total"] - - assert row is not None - assert tuple(row[:6]) == (10, 10, 6, 2, 4, 4) - assert row["source_counts"] == '{"tdp": 8, "skyeye": 2}' - assert row["source_covered_count"] == 10 - assert tuple(row[8:12]) == (1, 0, 0, 3) - assert contribution_count == 1 - - -@pytest.mark.asyncio -async def test_workflow_store_replaces_legacy_counts_in_the_first_verified_bucket() -> None: - await WorkflowStore.init() - now_ms = WorkflowStore._now_ms() - bucket_start = now_ms - (now_ms % 60_000) - db = await WorkflowStore.raw_db() - await db.execute( - """ - INSERT INTO workflow_metric_rollups - (workflow_id, bucket_start, raw_count, normalized_count, - after_filter_count, unique_count, filter_removed_count, - duplicate_count, source_counts, source_covered_count, - success_count, error_count, invalid_count, schema_version, updated_at) - VALUES (?, ?, 999, 999, 999, 999, 0, 0, ?, 999, 1, 0, 0, 2, ?) - """, - ("stream_alert_denoise", bucket_start, '{"tdp": 999}', now_ms), - ) - await db.commit() - - await WorkflowStore.complete_execution( - { - "id": "denoise-first-v3", - "workflowId": "stream_alert_denoise", - "status": "success", - "startedAt": now_ms, - "inputParams": {"alerts": [{"id": "a"}]}, - "outputResults": { - "stats": { - "metric_schema_version": 2, - "raw_count": 1, - "normalized_count": 1, - "after_filter_count": 1, - "after_dedup_count": 1, - "normalize_type_counts": {"tdp": 1}, - } - }, - }, - [], - ) - - async with db.execute( - "SELECT raw_count, source_counts, schema_version FROM workflow_metric_rollups " - "WHERE workflow_id = ? AND bucket_start = ?", - ("stream_alert_denoise", bucket_start), - ) as cursor: - row = await cursor.fetchone() - - assert row is not None - assert tuple(row) == (1, '{"tdp": 1}', 3) - - -@pytest.mark.asyncio -async def test_workflow_store_preserves_input_volume_when_metrics_are_invalid_or_failed() -> None: - await WorkflowStore.init() - now_ms = WorkflowStore._now_ms() - - await WorkflowStore.complete_execution( - { - "id": "denoise-invalid-output", - "workflowId": "stream_alert_denoise", - "status": "success", - "startedAt": now_ms, - "inputParams": {"alerts": [{"id": "a"}, {"id": "b"}, {"id": "c"}]}, - "outputResults": { - "stats": { - "metric_schema_version": 2, - "raw_count": 0, - "normalized_count": 0, - "after_filter_count": 0, - "after_dedup_count": 0, - } - }, - }, - [], - ) - await WorkflowStore.complete_execution( - { - "id": "denoise-failed-input", - "workflowId": "stream_alert_denoise", - "status": "failed", - "startedAt": now_ms, - "inputParams": {"alerts": {"_type": "list", "count": 1200}}, - "outputResults": {}, - }, - [], - ) - - db = await WorkflowStore.raw_db() - async with db.execute( - "SELECT raw_count, normalized_count, success_count, error_count, invalid_count " - "FROM workflow_metric_rollups WHERE workflow_id = ?", - ("stream_alert_denoise",), - ) as cursor: - row = await cursor.fetchone() - - assert row is not None - assert tuple(row) == (1203, 0, 1, 1, 1) - - -@pytest.mark.asyncio -async def test_workflow_store_does_not_count_malformed_syslog_as_accepted_alert() -> None: - await WorkflowStore.init() - now_ms = WorkflowStore._now_ms() - - await WorkflowStore.complete_execution( - { - "id": "denoise-malformed-syslog", - "workflowId": "stream_alert_denoise", - "status": "success", - "startedAt": now_ms, - "inputParams": {"syslog_message": {"message": "not-json"}}, - "outputResults": { - "stats": { - "metric_schema_version": 2, - "raw_count": 0, - "normalized_count": 0, - "after_filter_count": 0, - "after_dedup_count": 0, - } - }, - }, - [], - ) - - db = await WorkflowStore.raw_db() - async with db.execute( - "SELECT raw_count, invalid_count FROM workflow_metric_rollups WHERE workflow_id = ?", - ("stream_alert_denoise",), - ) as cursor: - row = await cursor.fetchone() - - assert row is not None - assert tuple(row) == (0, 0) - - await WorkflowStore.complete_execution( - { - "id": "denoise-non-object-syslog", - "workflowId": "stream_alert_denoise", - "status": "success", - "startedAt": now_ms, - "inputParams": {"syslog_message": {"message": "[]"}}, - "outputResults": { - "stats": { - "metric_schema_version": 2, - "raw_count": 0, - "normalized_count": 0, - "after_filter_count": 0, - "after_dedup_count": 0, - } - }, - }, - [], - ) - - async with db.execute( - "SELECT raw_count, invalid_count FROM workflow_metric_rollups WHERE workflow_id = ?", - ("stream_alert_denoise",), - ) as cursor: - row = await cursor.fetchone() - - assert row is not None - assert tuple(row) == (0, 0) - - -@pytest.mark.asyncio -async def test_workflow_store_input_reconciliation_matches_receive_node_priority() -> None: - await WorkflowStore.init() - now_ms = WorkflowStore._now_ms() - cases = [ - ( - "syslog-wins", - { - "syslog_message": {"message": json.dumps({"id": "single"})}, - "alerts": [{"id": index} for index in range(5)], - }, - 1, - ), - ( - "alerts-shadow-alert-list", - {"alerts": [], "alert_list": [{"id": "shadowed"}]}, - 0, - ), - ( - "canonical-alerts-win", - {"raw_alerts": [{"id": index} for index in range(5)], "alerts": [{"id": "api"}]}, - 1, - ), - ( - "materialized-alerts-win-over-stale-marker", - {"_alerts_count": 0, "alerts": [{"id": "materialized"}]}, - 1, - ), - ] - for execution_id, input_params, raw_count in cases: - await WorkflowStore.complete_execution( - { - "id": execution_id, - "workflowId": "stream_alert_denoise", - "status": "success", - "startedAt": now_ms, - "inputParams": input_params, - "outputResults": { - "stats": { - "metric_schema_version": 2, - "raw_count": raw_count, - "normalized_count": raw_count, - "after_filter_count": raw_count, - "after_dedup_count": raw_count, - } - }, - }, - [], - ) - - db = await WorkflowStore.raw_db() - async with db.execute( - "SELECT raw_count, success_count, invalid_count FROM workflow_metric_rollups " - "WHERE workflow_id = ?", - ("stream_alert_denoise",), - ) as cursor: - row = await cursor.fetchone() - - assert row is not None - assert tuple(row) == (3, 4, 0) - - -@pytest.mark.asyncio -async def test_workflow_store_does_not_accept_unverified_zero_for_file_input() -> None: - await WorkflowStore.init() - now_ms = WorkflowStore._now_ms() - - await WorkflowStore.complete_execution( - { - "id": "denoise-file-zero", - "workflowId": "stream_alert_denoise", - "status": "success", - "startedAt": now_ms, - "inputParams": {"alert_file": "/tmp/alerts.json"}, - "outputResults": { - "stats": { - "metric_schema_version": 2, - "raw_count": 0, - "normalized_count": 0, - "after_filter_count": 0, - "after_dedup_count": 0, - } - }, - }, - [], - ) - - db = await WorkflowStore.raw_db() - async with db.execute( - "SELECT raw_count, invalid_count FROM workflow_metric_rollups WHERE workflow_id = ?", - ("stream_alert_denoise",), - ) as cursor: - row = await cursor.fetchone() - - assert row is not None - assert tuple(row) == (0, 1) - - -@pytest.mark.asyncio -async def test_workflow_store_marks_legacy_batch_metrics_invalid() -> None: - await WorkflowStore.init() - now_ms = WorkflowStore._now_ms() - - await WorkflowStore.complete_execution( - { - "id": "denoise-legacy-batch", - "workflowId": "stream_alert_denoise", - "status": "success", - "startedAt": now_ms, - "outputResults": { - "stats": { - "raw_count": 2, - "normalized_count": 2, - "after_filter_count": 2, - "after_dedup_count": 2, - } - }, - }, - [], - ) - - db = await WorkflowStore.raw_db() - async with db.execute( - "SELECT raw_count, success_count, invalid_count FROM workflow_metric_rollups " - "WHERE workflow_id = ?", - ("stream_alert_denoise",), - ) as cursor: - row = await cursor.fetchone() - - assert row is not None - assert tuple(row) == (0, 1, 1) - - @pytest.mark.asyncio async def test_workflow_store_increment_stats_is_atomic_for_concurrent_updates() -> None: await WorkflowStore.init() diff --git a/webui/src/utils/socDashboardPageRuntime.test.tsx b/webui/src/utils/socDashboardPageRuntime.test.tsx index 091a35d14..888fa93ff 100644 --- a/webui/src/utils/socDashboardPageRuntime.test.tsx +++ b/webui/src/utils/socDashboardPageRuntime.test.tsx @@ -50,15 +50,6 @@ describe('SOC dashboard contract page runtime', () => { }, }); } - if (path === '/ai-tasks') { - return Promise.resolve({ - data: { - connection: 'online', - summary: { active: 0, running: 0, waiting: 0, stale: 0 }, - tasks: [], - }, - }); - } if (path === '/task-center') { return Promise.resolve({ data: { scheduledTasks: [], workflows: [] } }); } @@ -82,123 +73,12 @@ describe('SOC dashboard contract page runtime', () => { await waitFor(() => { expect(pageGetMock).toHaveBeenCalledWith('/stats', expect.anything()); expect(pageGetMock).toHaveBeenCalledWith('/activity', expect.anything()); - expect(pageGetMock).toHaveBeenCalledWith('/ai-tasks', expect.anything()); expect(pageGetMock).toHaveBeenCalledWith('/task-center', expect.anything()); }); expect(screen.getByText('Flocks AI 智能告警态势中心')).toBeInTheDocument(); }); - it('shows unavailable denoise metrics as dashes instead of false zeros', async () => { - pageGetMock.mockImplementation((path: string) => { - if (path === '/stats') { - return Promise.resolve({ - data: { - generatedAt: new Date().toISOString(), - denoise: { totalRaw: 0, totalUnique: 0, duplicateRate: 0, duplicates: 0 }, - pipeline: { raw: 0, unique: 0 }, - sources: [{ key: 'ndr', label: 'NDR', value: 0, rate: 0, active: false }], - sourceStatus: { - metricQuality: { - status: 'unavailable', - dataAvailable: false, - metricsAvailable: false, - unavailableReason: 'workflow_db_missing', - }, - }, - }, - }); - } - if (path === '/activity') { - return Promise.resolve({ - data: { - cursor: 'cursor', events: [], recentEvents: [], workflowEvents: [], batch: {}, - workflowStats: { callCount: null, latestStartedAt: null }, - }, - }); - } - if (path === '/ai-tasks') { - return Promise.resolve({ data: { connection: 'online', summary: {}, tasks: [] } }); - } - if (path === '/task-center') return Promise.resolve({ data: { scheduledTasks: [], workflows: [] } }); - return Promise.reject(new Error(`unexpected path: ${path}`)); - }); - - const { container } = render(); - - expect(await screen.findByText('降噪统计数据源不可用,相关数字已隐藏;系统正在重试')).toBeInTheDocument(); - const rawMetric = screen.getByText('原始告警量').closest('.command-metric') as HTMLElement; - expect(within(rawMetric).getByText('--')).toHaveAttribute('title', '数据不可用'); - const ndrSource = screen.getByText('NDR').closest('.command-source') as HTMLElement; - expect(within(ndrSource).getByText('--')).toBeInTheDocument(); - expect(container.querySelector('.command-source b')).toHaveAttribute('title', '数据不可用'); - }); - - it('shows unavailable SOC triage metrics as dashes instead of false zeros', async () => { - pageGetMock.mockImplementation((path: string) => { - if (path === '/stats') { - return Promise.resolve({ - data: { - generatedAt: new Date().toISOString(), - denoise: { totalRaw: 10, totalUnique: 4, duplicateRate: 0.6, duplicates: 6 }, - triage: { - totalRecords: 0, - attackTotal: 0, - attackSuccess: 0, - benign: 0, - unknown: 0, - }, - pipeline: { attackRate: 0, successRate: 0 }, - closedLoop: { autoClosed: 0, manualDecision: 0, pending: 0, resolutionRate: 0 }, - severityLevels: [], - sourceStatus: { - metricQuality: { status: 'complete', metricsAvailable: true }, - triageQuality: { - status: 'unavailable', - dataAvailable: false, - metricsAvailable: false, - unavailableReason: 'soc_db_missing', - }, - }, - }, - }); - } - if (path === '/activity') { - return Promise.resolve({ - data: { - cursor: 'cursor', events: [], recentEvents: [], workflowEvents: [], batch: {}, - workflowStats: { callCount: 1, latestStartedAt: Date.now() }, - }, - }); - } - if (path === '/ai-tasks') { - return Promise.resolve({ data: { connection: 'online', summary: {}, tasks: [] } }); - } - if (path === '/task-center') return Promise.resolve({ data: { scheduledTasks: [], workflows: [] } }); - return Promise.reject(new Error(`unexpected path: ${path}`)); - }); - - render(); - - expect(await screen.findByText('SOC 事件数据库不可用,研判、事件与闭环数字已隐藏;系统正在重试')).toBeInTheDocument(); - const eventMetric = screen.getByText('安全事件量').closest('.command-metric') as HTMLElement; - expect(within(eventMetric).getByText('--')).toHaveAttribute('title', '数据不可用'); - const criticalSeverity = screen.getByText('严重').closest('.severity-node') as HTMLElement; - expect(within(criticalSeverity).getByText('--')).toHaveAttribute('title', '数据不可用'); - expect(screen.queryByText('SOC 事件数据库不可用,研判、事件与闭环数字已隐藏;系统正在重试')).toBeInTheDocument(); - }); - - it('renders unknown metrics as dashes on the loading frame', () => { - pageGetMock.mockImplementation(() => new Promise(() => {})); - - render(); - - const rawMetric = screen.getByText('原始告警量').closest('.command-metric') as HTMLElement; - const eventMetric = screen.getByText('安全事件量').closest('.command-metric') as HTMLElement; - expect(within(rawMetric).getByText('--')).toHaveAttribute('title', '数据不可用'); - expect(within(eventMetric).getByText('--')).toHaveAttribute('title', '数据不可用'); - }); - it('pauses task-center polling while the page is hidden', async () => { setDocumentHidden(true); @@ -286,15 +166,6 @@ describe('SOC dashboard contract page runtime', () => { if (path === '/task-center') { return Promise.resolve({ data: { scheduledTasks: [], workflows: [] } }); } - if (path === '/ai-tasks') { - return Promise.resolve({ - data: { - connection: 'online', - summary: { active: 0, running: 0, waiting: 0, stale: 0 }, - tasks: [], - }, - }); - } return Promise.reject(new Error(`unexpected path: ${path}`)); }); @@ -323,15 +194,6 @@ describe('SOC dashboard contract page runtime', () => { }, }); } - if (path === '/ai-tasks') { - return Promise.resolve({ - data: { - connection: 'online', - summary: { active: 0, running: 0, waiting: 0, stale: 0 }, - tasks: [], - }, - }); - } if (path === '/task-center') { return Promise.resolve({ data: { @@ -418,86 +280,6 @@ describe('SOC dashboard contract page runtime', () => { expect(within(workflowStats).getByText('今日调用')).toBeInTheDocument(); }); - it('uses authoritative workflow task status instead of activity playback state', async () => { - const now = Date.now(); - pageGetMock.mockImplementation((path: string) => { - if (path === '/stats') return Promise.resolve({ data: {} }); - if (path === '/activity') { - return Promise.resolve({ - data: { - cursor: 'cursor', - events: [], - recentEvents: [], - workflowEvents: [ - { - eventId: 'workflow-execution:completed-history', - stage: 'denoise', - status: 'completed', - occurredAt: new Date(now).toISOString(), - triggerSource: 'workflow_execution', - workflowId: 'stream_alert_denoise', - alert: { id: 'history', threatName: '不应进入任务栏' }, - result: { isDuplicate: false, rawCount: 0 }, - }, - ], - batch: {}, - workflowStats: { callCount: 0, latestStartedAt: 0 }, - tokenUsage: { totalTokens: 0, todayTokens: 0, todayRequests: 0, dailySeries: [] }, - }, - }); - } - if (path === '/ai-tasks') { - return Promise.resolve({ - data: { - connection: 'online', - summary: { active: 2, running: 1, waiting: 1, stale: 0 }, - tasks: [ - { - taskId: 'workflow-execution:running-1', - workflowId: 'stream_alert_triage', - executionId: 'running-1', - stage: 'triage', - status: 'running', - startedAt: now, - title: 'SSRF盲打探测攻击结果未知', - counts: { raw: null }, - dataQuality: 'pending', - progress: { mode: 'steps', current: 2, total: 3, percent: 0.6667, label: '第 2/3 步' }, - }, - { - taskId: 'workflow-execution:queued-1', - workflowId: 'stream_alert_denoise', - executionId: 'queued-1', - stage: 'denoise', - status: 'queued', - startedAt: now - 1000, - title: '降噪批次', - counts: { raw: 0 }, - dataQuality: 'complete', - rawCountSource: 'workflow_output', - emptyInput: false, - progress: { mode: 'waiting', percent: null, label: '等待调度' }, - }, - ], - }, - }); - } - if (path === '/task-center') return Promise.resolve({ data: { scheduledTasks: [], workflows: [] } }); - return Promise.reject(new Error(`unexpected path: ${path}`)); - }); - - render(); - - expect(await screen.findByText('正在处理 1 个,等待 1 个')).toBeInTheDocument(); - expect(screen.getByText('SSRF盲打探测攻击结果未知')).toBeInTheDocument(); - expect(screen.getByText('降噪批次')).toBeInTheDocument(); - expect(screen.getByText(/原始条数待校验/)).toBeInTheDocument(); - expect(screen.queryByText(/原始 0 条/)).not.toBeInTheDocument(); - expect(screen.getByText('第 2/3 步')).toBeInTheDocument(); - expect(screen.queryByText('不应进入任务栏')).not.toBeInTheDocument(); - expect(screen.queryByText('降噪处理完成')).not.toBeInTheDocument(); - }); - it('uses dashboard mock rows with the same workflow execution field shape as real task-center data', async () => { window.localStorage.setItem('soc-dashboard-mock-v1', '1'); From dc2e18b193a2f212ba00deb2f2d575a5fdefb127 Mon Sep 17 00:00:00 2001 From: John Yin <10972267+john-yin2333@user.noreply.gitee.com> Date: Thu, 10 Sep 2026 23:34:35 +0800 Subject: [PATCH 3/7] fix(workflow): prevent timeout polling traceback memory growth (cherry picked from commit 2968862ca5c29875f43f7fab71b61fd82c03c831) --- flocks/workflow/_async_runtime.py | 29 +++-- tests/workflow/test_async_runtime.py | 178 ++++++++++++++++++++++++++- tests/workflow/test_workflow_llm.py | 31 ++++- 3 files changed, 223 insertions(+), 15 deletions(-) diff --git a/flocks/workflow/_async_runtime.py b/flocks/workflow/_async_runtime.py index 36d7e232f..a71f563e3 100644 --- a/flocks/workflow/_async_runtime.py +++ b/flocks/workflow/_async_runtime.py @@ -140,16 +140,25 @@ def run_sync_cancellable( _assert_not_workflow_loop(coro, loop) future = asyncio.run_coroutine_threadsafe(coro, loop) - while True: - if cancel_checker(): - future.cancel() - raise asyncio.CancelledError() - try: - return future.result(timeout=poll_interval_s) - except concurrent.futures.TimeoutError: - continue - except concurrent.futures.CancelledError as exc: - raise asyncio.CancelledError() from exc + try: + while True: + if cancel_checker(): + future.cancel() + raise asyncio.CancelledError() + try: + return future.result(timeout=poll_interval_s) + except concurrent.futures.TimeoutError: + # A completed coroutine can itself raise TimeoutError (also + # used by asyncio.wait_for). It is indistinguishable by type + # from an unfinished Future's polling timeout. Retrying a + # failed Future spins without waiting and keeps extending the + # same exception's traceback, consuming unbounded memory. + if future.done(): + # Resolve outside the polling try/except. This also handles + # success or cancellation racing the timed-out wait. + return future.result() + except concurrent.futures.CancelledError as exc: + raise asyncio.CancelledError() from exc def _get_loop_for_testing() -> tuple[asyncio.AbstractEventLoop | None, threading.Thread | None]: diff --git a/tests/workflow/test_async_runtime.py b/tests/workflow/test_async_runtime.py index 93cbef363..e9b87026e 100644 --- a/tests/workflow/test_async_runtime.py +++ b/tests/workflow/test_async_runtime.py @@ -3,7 +3,8 @@ from __future__ import annotations import asyncio -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future, ThreadPoolExecutor +import threading import pytest @@ -99,7 +100,8 @@ async def _self_cancel(): ) -def test_run_sync_from_inside_the_loop_thread_raises_instead_of_deadlocking(): +@pytest.mark.parametrize("cancellable", [False, True]) +def test_run_sync_from_inside_the_loop_thread_raises_instead_of_deadlocking(cancellable): """Invoking run_sync from the dedicated loop's own thread would self-deadlock. The guard must surface that as RuntimeError instead of hanging the caller. @@ -107,8 +109,178 @@ def test_run_sync_from_inside_the_loop_thread_raises_instead_of_deadlocking(): loop = _async_runtime._ensure_loop() async def _trigger_from_loop(): - _async_runtime.run_sync(_echo("never")) + if cancellable: + _async_runtime.run_sync_cancellable(_echo("never"), lambda: False) + else: + _async_runtime.run_sync(_echo("never")) future = asyncio.run_coroutine_threadsafe(_trigger_from_loop(), loop) with pytest.raises(RuntimeError, match="self-deadlock"): future.result(timeout=2.0) + + +@pytest.mark.parametrize("use_wait_for", [False, True]) +def test_cancellable_propagates_coroutine_timeout_without_traceback_growth(monkeypatch, use_wait_for): + """A failed Future must not be polled forever as if it were unfinished.""" + submitted = [] + submit = asyncio.run_coroutine_threadsafe + + def capture_submission(coro, loop): + future = submit(coro, loop) + submitted.append(future) + return future + + monkeypatch.setattr(asyncio, "run_coroutine_threadsafe", capture_submission) + polls = 0 + + def never_cancel(): + nonlocal polls + polls += 1 + # Bound failures on old code: do not let a regression OOM the test host. + assert polls < 1000, "completed timeout Future was polled repeatedly" + return False + + async def timed_out(): + if use_wait_for: + await asyncio.wait_for(asyncio.sleep(60), timeout=0.005) + else: + raise TimeoutError("upstream timed out") + + with pytest.raises(TimeoutError): + _async_runtime.run_sync_cancellable(timed_out(), never_cancel, poll_interval_s=0.001) + + traceback = submitted[0].exception().__traceback__ + depth = 0 + while traceback is not None: + depth += 1 + traceback = traceback.tb_next + assert depth < 30 + assert _async_runtime.run_sync(_echo("loop still usable")) == "loop still usable" + + +def test_cancellable_keeps_polling_an_unfinished_future(): + polls = 0 + release = threading.Event() + + def never_cancel(): + nonlocal polls + polls += 1 + assert polls < 1000 + if polls >= 2: + release.set() + return False + + async def delayed_result(): + while not release.is_set(): + await asyncio.sleep(0.001) + return "done" + + assert _async_runtime.run_sync_cancellable( + delayed_result(), never_cancel, poll_interval_s=0.001, + ) == "done" + assert polls > 1 + + +@pytest.mark.parametrize("outcome", ["success", "cancelled", "error"]) +def test_cancellable_resolves_completion_racing_a_poll_timeout(monkeypatch, outcome): + class RacingFuture(Future): + def result(self, timeout=None): + if not self.done(): + if outcome == "success": + self.set_result("done") + elif outcome == "cancelled": + self.cancel() + else: + self.set_exception(ValueError("upstream failure")) + raise TimeoutError("poll expired immediately before completion") + return super().result(timeout=timeout) + + future = RacingFuture() + + def submit(coro, loop): + coro.close() + return future + + monkeypatch.setattr(asyncio, "run_coroutine_threadsafe", submit) + if outcome == "cancelled": + with pytest.raises(asyncio.CancelledError): + _async_runtime.run_sync_cancellable(_echo(None), lambda: False) + elif outcome == "error": + with pytest.raises(ValueError, match="upstream failure"): + _async_runtime.run_sync_cancellable(_echo(None), lambda: False) + else: + assert _async_runtime.run_sync_cancellable(_echo(None), lambda: False) == "done" + + +def test_cancellable_propagates_coroutine_cancellation(): + async def cancelled(): + raise asyncio.CancelledError() + + with pytest.raises(asyncio.CancelledError): + _async_runtime.run_sync_cancellable(cancelled(), lambda: False) + + +def test_cancellable_operator_cancel_releases_the_waiting_coroutine(): + started = threading.Event() + stopped = threading.Event() + + async def waiting(): + started.set() + try: + await asyncio.sleep(60) + finally: + stopped.set() + + with pytest.raises(asyncio.CancelledError): + _async_runtime.run_sync_cancellable(waiting(), started.is_set, poll_interval_s=0.001) + assert stopped.wait(timeout=2), "cancelled coroutine did not run its cleanup" + + +def test_cancellable_parallel_timeouts_do_not_poison_other_calls(): + def worker(index): + polls = 0 + + def never_cancel(): + nonlocal polls + polls += 1 + assert polls < 1000 + return False + + async def call(): + await asyncio.sleep(0) + if index % 2: + raise TimeoutError(f"timeout-{index}") + return index + + if index % 2: + with pytest.raises(TimeoutError, match=f"^timeout-{index}$"): + _async_runtime.run_sync_cancellable(call(), never_cancel, poll_interval_s=0.001) + return index + return _async_runtime.run_sync_cancellable(call(), never_cancel, poll_interval_s=0.001) + + with ThreadPoolExecutor(max_workers=8) as pool: + assert list(pool.map(worker, range(200))) == list(range(200)) + + +@pytest.mark.parametrize("method", ["run", "run_safe"]) +def test_cancellable_tool_adapter_propagates_tool_timeout(monkeypatch, method): + from flocks.tool import ToolRegistry + from flocks.workflow.tools_adapter import FlocksToolAdapter + + async def execute(*args, **kwargs): + await asyncio.wait_for(asyncio.sleep(60), timeout=0.005) + + monkeypatch.setattr(ToolRegistry, "init", lambda: None) + monkeypatch.setattr(ToolRegistry, "get", lambda name: object()) + monkeypatch.setattr(ToolRegistry, "execute", execute) + polls = 0 + + def never_cancel(): + nonlocal polls + polls += 1 + assert polls < 1000, "tool timeout was swallowed by cancellation polling" + return False + + adapter = FlocksToolAdapter().with_cancel_checker(never_cancel) + with pytest.raises(TimeoutError): + getattr(adapter, method)("synthetic-timeout") diff --git a/tests/workflow/test_workflow_llm.py b/tests/workflow/test_workflow_llm.py index 1767fadc6..8d6b5a538 100644 --- a/tests/workflow/test_workflow_llm.py +++ b/tests/workflow/test_workflow_llm.py @@ -255,17 +255,44 @@ def test_llm_does_not_retry_an_oversized_response(monkeypatch): assert provider.calls == 1 -def test_llm_timeout_retries_then_raises(monkeypatch): +@pytest.mark.parametrize("cancellable", [False, True]) +def test_llm_timeout_retries_then_raises(monkeypatch, cancellable): provider = _FakeProvider("demo", "timeout", models=["m"]) _patch_provider(monkeypatch, {"demo": provider}) - client = LLMClient(provider_id="demo", model="m") + polls = 0 + + def never_cancel(): + nonlocal polls + polls += 1 + assert polls < 1000, "LLM timeout was swallowed by the cancellation poll loop" + return False + + client = LLMClient( + provider_id="demo", model="m", cancel_checker=never_cancel if cancellable else None, + ) with pytest.raises(ValueError, match="timed out after 0.01s"): client.ask("hello", timeout_s=0.01, max_retries=2, retry_delay_s=0) assert provider.calls == 3 +def test_cancellable_llm_can_retry_after_timeout_and_succeed(monkeypatch): + provider = _FakeProvider("demo", ["timeout", "ok"], models=["m"]) + _patch_provider(monkeypatch, {"demo": provider}) + polls = 0 + + def never_cancel(): + nonlocal polls + polls += 1 + assert polls < 1000 + return False + + client = LLMClient(provider_id="demo", model="m", cancel_checker=never_cancel) + assert client.ask("hello", timeout_s=0.01, max_retries=1, retry_delay_s=0) == "demo:m" + assert provider.calls == 2 + + def test_llm_ask_honors_cancel_checker(monkeypatch): provider = _FakeProvider("demo", "slow", models=["m"]) _patch_provider(monkeypatch, {"demo": provider}) From 39b0f415bbb46325fcd1f5b91141363a8a3395cd Mon Sep 17 00:00:00 2001 From: John Yin <10972267+john-yin2333@user.noreply.gitee.com> Date: Fri, 11 Sep 2026 10:51:13 +0800 Subject: [PATCH 4/7] fix(soc): stabilize dashboard alert identity and task state (cherry picked from commit 8691040704d3949a6a4995a6b47913c70b7b2dde) --- .flocks/flockshub/index.json | 2 +- .../plugins/webuis/soc_ui/manifest.json | 2 +- .../soc_ui/soc_dashboard/api/handlers.py | 170 ++++++-- .../webuis/soc_ui/soc_dashboard/src/Page.tsx | 408 ++++++++---------- .../plugins/webuis/soc_ui/workspace.json | 2 +- tests/hub/test_hub_catalog.py | 6 +- tests/hub/test_soc_dashboard_schema.py | 145 ++++++- .../utils/socDashboardPageRuntime.test.tsx | 186 +++++++- 8 files changed, 635 insertions(+), 286 deletions(-) diff --git a/.flocks/flockshub/index.json b/.flocks/flockshub/index.json index 4d3560e97..5d5a7e92a 100644 --- a/.flocks/flockshub/index.json +++ b/.flocks/flockshub/index.json @@ -14640,7 +14640,7 @@ "name": "SOC Workspace WebUI", "description": "SOC workspace pages for posture, overview, and alert investigation.", "descriptionCn": "SOC 工作区页面,包含态势、SOC 总览和告警调查。", - "version": "1.1.5", + "version": "1.1.6", "category": "workflow-automation", "tags": [ "siem", diff --git a/.flocks/flockshub/plugins/webuis/soc_ui/manifest.json b/.flocks/flockshub/plugins/webuis/soc_ui/manifest.json index 25c7d5cdd..cfa3f5464 100644 --- a/.flocks/flockshub/plugins/webuis/soc_ui/manifest.json +++ b/.flocks/flockshub/plugins/webuis/soc_ui/manifest.json @@ -5,7 +5,7 @@ "name": "SOC Workspace WebUI", "description": "SOC workspace pages for posture, overview, and alert investigation.", "descriptionCn": "SOC 工作区页面,包含态势、SOC 总览和告警调查。", - "version": "1.1.5", + "version": "1.1.6", "author": "Flocks Team", "license": "MIT", "homepage": "", diff --git a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/handlers.py b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/handlers.py index 240bd4286..10723e969 100644 --- a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/handlers.py +++ b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/handlers.py @@ -6,6 +6,7 @@ import sqlite3 import time from collections import Counter, OrderedDict +from contextlib import closing from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path @@ -1054,45 +1055,125 @@ def _get_workflow_progress( } +def _dashboard_execution_detail(output_text, input_text): + """Read a bounded preview for this execution only; never open result files. + + Kept separate from metric aggregation and task-center/workflow contracts. + A missing count is not zero (nor an assumed single-alert batch). + """ + output = _safe_json_object(output_text) + inputs = _safe_json_object(input_text) + def text(*values): + return _first_text(*(value[:512] for value in values if isinstance(value, str))) + + queue = [(value, 0) for value in ( + output.get("enriched_alerts_with_triage"), output.get("triage_results"), + output.get("unique_alerts"), output.get("enriched_alerts"), + inputs.get("_soc_alert_preview"), inputs.get("alerts"), + inputs.get("alert_list"), inputs.get("syslog_message"), inputs.get("syslog"), + ) if value is not None] + best, best_score = {}, 0 + for _ in range(48): + if not queue: + break + value, depth = queue.pop(0) + if depth > 6: + continue + if isinstance(value, str): + if len(value) > 65536: + continue + # Syslog prefixes can precede a vendor JSON envelope. + start = value.find("{") + parsed = _safe_json_object(value[start:] if start >= 0 else value) + if parsed: + queue.append((parsed, depth + 1)) + elif isinstance(value, list): + queue.extend((item, depth + 1) for item in value[:3]) + elif isinstance(value, dict): + if value.get("_type") == "dict": + continue # Persisted key-only summary, not an alert. + alert = { + "id": text(value.get("id"), value.get("alert_id"), value.get("uuid")), + "threatName": text(value.get("threat_name"), value.get("alert_name"), value.get("attack_name")), + "sourceType": text(value.get("_source_type"), value.get("source_type"), value.get("device_type")), + "srcIp": text(value.get("sip"), value.get("src_ip"), value.get("source_ip"), value.get("net_real_src_ip")), + "dstIp": text(value.get("dip"), value.get("dst_ip"), value.get("destination_ip"), value.get("net_dest_ip")), + } + score = sum(bool(alert[key]) for key in ("threatName", "srcIp", "dstIp")) + if score > best_score: + best, best_score = alert, score + for key in ("preview", "data", "alert", "alerts", "message", "msg", "log", "event", "payload"): + if key in value: + queue.append((value[key], depth + 1)) + stats = output.get("stats") if isinstance(output.get("stats"), dict) else {} + raw = stats.get("raw_count", inputs.get("_soc_alert_count")) + try: + count = int(raw) if raw is not None and not isinstance(raw, bool) else None + if count is not None and (count < 0 or str(raw).strip() != str(count)): + count = None + except (ValueError, TypeError, OverflowError): + count = None + if count is None: + alerts = inputs.get("alerts", inputs.get("alert_list")) + if isinstance(alerts, list): + count = len(alerts) + if best: + best["sourceType"] = best["sourceType"] or text( + inputs.get("source_log_type"), output.get("source_log_type") + ) + return best, count + + def _get_workflow_recent_events( workflow_name: str, start_time: int = 0, end_time: int = 0, limit: int = 10, + snapshot=None, ) -> list: if not WORKFLOW_DB.is_file(): + if snapshot is not None: + snapshot.update(complete=False, available=False) return [] workflow_stage = "triage" if workflow_name in TRIAGE_WORKFLOW_IDS else "denoise" - query_params = [workflow_name] try: - with sqlite3.connect(WORKFLOW_DB) as conn: + with closing(sqlite3.connect(f"{WORKFLOW_DB.resolve().as_uri()}?mode=ro", uri=True, timeout=0.2)) as conn: + deadline = time.monotonic() + 0.2 + conn.set_progress_handler(lambda: int(time.monotonic() > deadline), 1000) + conn.row_factory = sqlite3.Row + conn.execute("BEGIN") execution_columns = { row[1] for row in conn.execute("PRAGMA table_info(workflow_executions)").fetchall() } + # Do not materialize unbounded execution payloads in a 3-second UI poll. + def preview_column(name): + if name not in execution_columns: + return f"'{{}}' AS {name}" + return f"CASE WHEN length({name}) <= 262144 THEN {name} ELSE '{{}}' END AS {name}" + + latest_select = ", ".join([ + "id", "status", "started_at", + _workflow_execution_column_expr(execution_columns, "updated_at", "started_at"), + *(preview_column(name) for name in ("output_results", "input_params", "payload")), + ]) + query = f"SELECT {latest_select} FROM workflow_executions WHERE workflow_id = ?" + query_params = [workflow_name] + if start_time > 0 and end_time > 0: + query += " AND started_at >= ? AND started_at <= ?" + query_params.extend((int(start_time * 1000), int(end_time * 1000))) + row_limit = max(1, min(_safe_int(limit), 10)) + active_rows = conn.execute( + query + " AND status IN ('running', 'queued', 'pending') ORDER BY started_at DESC LIMIT ?", + [*query_params, row_limit], + ).fetchall() + recent_rows = conn.execute(query + " ORDER BY started_at DESC LIMIT ?", [*query_params, row_limit]).fetchall() + rows = list({row["id"]: row for row in [*active_rows, *recent_rows]}.values()) + if snapshot is not None and len(active_rows) == row_limit: + snapshot["complete"] = False except Exception: - return [] - latest_select = ", ".join( - [ - "id", - "status", - "started_at", - _workflow_execution_column_expr(execution_columns, "output_results", "'{}'"), - _workflow_execution_column_expr(execution_columns, "input_params", "'{}'"), - _workflow_execution_column_expr(execution_columns, "payload", "'{}'"), - ] - ) - query = f"SELECT {latest_select} FROM workflow_executions WHERE workflow_id = ?" - if start_time > 0 and end_time > 0: - query += " AND started_at >= ? AND started_at <= ?" - query_params.extend((int(start_time * 1000), int(end_time * 1000))) - query += " ORDER BY started_at DESC LIMIT ?" - query_params.append(max(1, min(_safe_int(limit), 10))) - try: - with sqlite3.connect(WORKFLOW_DB) as conn: - conn.row_factory = sqlite3.Row - rows = conn.execute(query, query_params).fetchall() - except Exception: + if snapshot is not None: + snapshot.update(complete=False, available=False) return [] events = [] @@ -1104,24 +1185,20 @@ def _get_workflow_recent_events( input_text = row["input_params"] payload_text = row["payload"] metrics = _workflow_execution_metrics(output_text, input_text) - preview = metrics["preview"] - raw_count = metrics["rawCount"] + alert, raw_count = _dashboard_execution_detail(output_text, input_text) + if workflow_stage == "denoise" and raw_count == 0 and not alert: + continue # No alert work; keep the execution itself untouched. unique_count = metrics["uniqueCount"] - threat_name = "" - if workflow_stage == "triage": + threat_name = alert.get("threatName", "") + if not threat_name and workflow_stage == "triage": threat_name = _workflow_latest_alert_name(workflow_name, output_text, input_text) if not threat_name: - threat_name = str( - preview.get("threat_name") - or preview.get("_threat_type") - or preview.get("threat_type") - or f"降噪批次 · 原始 {raw_count} 条" - ) + threat_name = f"降噪批次 · 原始 {raw_count} 条" if raw_count and raw_count > 0 else "降噪批次 · 数量未提供" normalized_status = str(status or "").lower() event_status = ( "completed" if normalized_status in {"success", "completed"} - else "running" + else normalized_status if normalized_status in {"running", "queued", "pending"} else "failed" ) @@ -1139,17 +1216,18 @@ def _get_workflow_recent_events( "occurredAt": datetime.fromtimestamp( _safe_int(started_at) / 1000 ).astimezone().isoformat(timespec="seconds"), + "updatedAt": datetime.fromtimestamp( + _safe_int(row["updated_at"] or started_at) / 1000 + ).astimezone().isoformat(timespec="milliseconds"), "triggerSource": "workflow_execution", "workflowId": workflow_name, "sessionId": session_id, "messageId": message_id, "sampleCount": max(unique_count, 1), "alert": { - "id": str(preview.get("id") or execution_id), - "sourceType": metrics["sourceType"], + **alert, + "id": alert.get("id") or execution_id, "threatName": threat_name, - "srcIp": preview.get("sip") or preview.get("src_ip") or preview.get("net_real_src_ip"), - "dstIp": preview.get("dip") or preview.get("dst_ip") or preview.get("net_dest_ip"), }, "result": { "isDuplicate": metrics["isDuplicate"], @@ -1159,6 +1237,7 @@ def _get_workflow_recent_events( for key, value in metrics.items() if key not in {"preview", "sourceCounts", "sourceType"} }, + "rawCount": raw_count, }, } ) @@ -1927,9 +2006,10 @@ def _get_activity(params): start_time, end_time, ) + workflow_snapshot = {"complete": True, "available": True} workflow_events = [ - *_get_workflow_recent_events("stream_alert_denoise", start_time, end_time), - *_get_workflow_recent_events("stream_alert_triage", start_time, end_time), + *_get_workflow_recent_events("stream_alert_denoise", start_time, end_time, snapshot=workflow_snapshot), + *_get_workflow_recent_events("stream_alert_triage", start_time, end_time, snapshot=workflow_snapshot), ] raw_cursor = str(params.get("cursor") or "").strip() bootstrap = str(params.get("bootstrap") or "").strip().lower() == "latest" @@ -1943,6 +2023,7 @@ def _get_activity(params): cursor_reset=bool(raw_cursor), workflow_stats=workflow_stats, workflow_events=workflow_events, + workflow_snapshot=workflow_snapshot, ) cursor = _decode_activity_cursor(raw_cursor) if raw_cursor else None @@ -1963,6 +2044,7 @@ def _get_activity(params): cursor_reset=cursor_reset, workflow_stats=workflow_stats, workflow_events=workflow_events, + workflow_snapshot=workflow_snapshot, ) latest_row_id, latest_activity_id = _activity_latest_cursor(conn, settings) @@ -1982,6 +2064,7 @@ def _get_activity(params): recent_events=recent_events, workflow_stats=workflow_stats, workflow_events=workflow_events, + workflow_snapshot=workflow_snapshot, ) last_row_id = max(_safe_int(cursor.get("lastRowId")), 0) @@ -2007,6 +2090,7 @@ def _get_activity(params): "", workflow_stats=workflow_stats, workflow_events=workflow_events, + workflow_snapshot=workflow_snapshot, ), "error": f"activity query failed: {exc}", } @@ -2029,6 +2113,7 @@ def _get_activity(params): batch=batch, workflow_stats=workflow_stats, workflow_events=workflow_events, + workflow_snapshot=workflow_snapshot, ), "overflowCount": overflow_count, } @@ -2044,6 +2129,7 @@ def _activity_response( batch=None, workflow_stats=None, workflow_events=None, + workflow_snapshot=None, ): return { "cursor": _encode_activity_cursor(last_row_id, last_activity_id), @@ -2055,6 +2141,8 @@ def _activity_response( "cursorReset": cursor_reset, "workflowStats": workflow_stats or {"callCount": None, "latestStartedAt": None}, "workflowEvents": workflow_events or [], + "workflowSnapshotComplete": bool(workflow_snapshot and workflow_snapshot["complete"]), + "workflowSnapshotAvailable": bool(workflow_snapshot and workflow_snapshot.get("available", True)), "tokenUsage": _read_token_usage(), } diff --git a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/Page.tsx b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/Page.tsx index c08e3d499..15126b392 100644 --- a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/Page.tsx +++ b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/Page.tsx @@ -184,6 +184,7 @@ function createActivityState() { denoise: { current: null, queue: [], last: null }, triage: { current: null, queue: [], last: null }, recent: [], + lastIdentified: null, batch: emptyActivityBatch(), batchUpdatedAt: 0, mode: 'normal', @@ -503,6 +504,54 @@ function isRunningWorkflowEvent(event) { && ['running', 'queued', 'pending'].includes(String(event?.status || '').toLowerCase()); } +function workflowActivityPriority(event) { + return isRunningWorkflowEvent(event) ? event.status === 'running' ? 2 : 1 : 0; +} + +function displayAlertText(value) { + if (typeof value !== 'string') return ''; + const text = value.trim(); + return /^(unknown|none|null|--|待识别|待识别资产|未知告警|未知来源)$/i.test(text) ? '' : text; +} + +function hasAlertIdentity(event) { + const alert = event?.alert || {}; + return Boolean(displayAlertText(alert.srcIp) || displayAlertText(alert.dstIp) + || (displayAlertText(alert.threatName) && !/^(降噪批次|研判批次)/.test(alert.threatName))); +} + +function isVisibleActivity(event) { + if (!event?.eventId || !['denoise', 'triage'].includes(event.stage)) return false; + return !(event.stage === 'denoise' && event.triggerSource === 'workflow_execution' + && event.result?.rawCount === 0 && !hasAlertIdentity(event)); +} + +function mergeActivityEvent(previous, incoming) { + if (!previous) return incoming; + const previousTime = Date.parse(previous.updatedAt || previous.occurredAt || '') || 0; + const incomingTime = Date.parse(incoming.updatedAt || incoming.occurredAt || '') || 0; + if (incomingTime < previousTime) return previous; + // A late polling response cannot resurrect a terminal execution. + if (previous.triggerSource === 'workflow_execution' + && !isRunningWorkflowEvent(previous) && isRunningWorkflowEvent(incoming)) return previous; + const incomingIdentity = hasAlertIdentity(incoming); + const differentAlert = incomingIdentity && displayAlertText(incoming.alert?.id) + && displayAlertText(previous.alert?.id) && incoming.alert.id !== previous.alert.id; + const alert = differentAlert ? {} : { ...previous.alert }; + for (const [key, value] of Object.entries(incoming.alert || {})) { + if (displayAlertText(value)) { + if (key === 'id' && !incomingIdentity && hasAlertIdentity(previous)) continue; + if (key === 'threatName' && /^(降噪批次|研判批次)/.test(value) && hasAlertIdentity(previous)) continue; + alert[key] = value; + } + } + return { + ...previous, ...incoming, alert, + result: { ...previous.result, ...incoming.result }, + playbackStartedAt: previous.playbackStartedAt, + }; +} + function normalizeActivityBatch(raw) { const batch = { ...emptyActivityBatch(), ...(raw || {}) }; for (const key of ['windowMs', 'receivedCount', 'duplicateCount', 'uniqueCount', 'clusterCount', 'triageUpdatedCount', 'sampledCount', 'suppressedCount', 'ratePerSecond']) { @@ -521,14 +570,15 @@ function resolveActivityMode(previous, batch) { return { mode, calmPolls }; } -function enqueueActivity(previous, events, generatedAt, recentEvents, rawBatch) { +function enqueueActivity(previous, events, generatedAt, recentEvents, rawBatch, workflowSnapshotComplete = false) { const batch = normalizeActivityBatch(rawBatch); const modeState = resolveActivityMode(previous, batch); const hasBatch = batch.receivedCount > 0 || batch.triageUpdatedCount > 0; - const incomingEvents = (events || []).filter(Boolean); - const incomingRecentEvents = (recentEvents || []).filter(Boolean); + const incomingEvents = (events || []).filter(isVisibleActivity); + const incomingRecentEvents = (recentEvents || []).filter(isVisibleActivity); if ( !hasBatch + && !workflowSnapshotComplete && !incomingEvents.length && !incomingRecentEvents.length && previous.connection === 'online' @@ -547,20 +597,39 @@ function enqueueActivity(previous, events, generatedAt, recentEvents, rawBatch) mode: modeState.mode, calmPolls: modeState.calmPolls, }; + if (workflowSnapshotComplete) { + const ids = new Set(incomingEvents.map((event) => event.eventId)); + const retained = (event) => !isRunningWorkflowEvent(event) || ids.has(event.eventId); + for (const kind of ['denoise', 'triage']) { + const lane = next[kind]; + if (lane.current && !retained(lane.current)) lane.current = null; + lane.queue = lane.queue.filter(retained); + } + next.recent = next.recent.filter(retained); + } if (batch.mode !== 'normal' && batch.receivedCount > 0) next.denoise.queue = []; for (const event of incomingEvents) { if (!event || !['denoise', 'triage'].includes(event.stage) || !event.eventId) continue; + const known = [next[event.stage].current, next[event.stage].last, ...next.recent] + .find((item) => item?.eventId === event.eventId); + const merged = mergeActivityEvent(known, event); const enriched = event.stage === 'denoise' - ? { ...event, playbackMode: batch.mode, batch } - : event; + ? { ...merged, playbackMode: batch.mode, batch } + : merged; const lane = next[enriched.stage]; + if (enriched.triggerSource === 'workflow_execution' && !isRunningWorkflowEvent(enriched)) { + if (lane.current?.eventId === enriched.eventId) lane.current = null; + lane.queue = lane.queue.filter((item) => item.eventId !== enriched.eventId); + if (!lane.last || activityTimestamp(enriched) >= activityTimestamp(lane.last)) lane.last = enriched; + continue; + } if (lane.current?.eventId === enriched.eventId) { - lane.current = { ...lane.current, ...enriched, playbackStartedAt: lane.current.playbackStartedAt }; + lane.current = mergeActivityEvent(lane.current, enriched); continue; } const queuedIndex = lane.queue.findIndex((item) => item?.eventId === enriched.eventId); if (queuedIndex >= 0) { - lane.queue[queuedIndex] = { ...lane.queue[queuedIndex], ...enriched }; + lane.queue[queuedIndex] = mergeActivityEvent(lane.queue[queuedIndex], enriched); continue; } const knownLast = lane.last?.eventId === enriched.eventId; @@ -568,19 +637,30 @@ function enqueueActivity(previous, events, generatedAt, recentEvents, rawBatch) } for (const kind of ['denoise', 'triage']) { const lane = next[kind]; + lane.queue.sort((a, b) => workflowActivityPriority(b) - workflowActivityPriority(a)); + if (workflowActivityPriority(lane.queue[0]) > workflowActivityPriority(lane.current)) { + const previousCurrent = lane.current; + lane.current = lane.queue.shift(); + if (isRunningWorkflowEvent(previousCurrent)) lane.queue.push(previousCurrent); + } if (lane.queue.length > ACTIVITY_QUEUE_LIMIT) { - lane.queue = lane.queue.slice(-ACTIVITY_QUEUE_LIMIT); + lane.queue = lane.queue.slice(0, ACTIVITY_QUEUE_LIMIT); } } - const incomingRecent = incomingRecentEvents.length ? incomingRecentEvents : [...incomingEvents].reverse(); + const incomingRecent = [...incomingRecentEvents, ...incomingEvents]; for (const event of [...incomingRecent].reverse()) { if (!event?.eventId) continue; - const merged = [event, ...next.recent.filter((item) => item.eventId !== event.eventId)]; + const merged = [mergeActivityEvent(next.recent.find((item) => item.eventId === event.eventId), event), + ...next.recent.filter((item) => item.eventId !== event.eventId)]; + const priority = (a, b) => Number(isRunningWorkflowEvent(b)) - Number(isRunningWorkflowEvent(a)) + || activityTimestamp(b) - activityTimestamp(a); next.recent = [ - ...merged.filter((item) => item.stage === 'triage').slice(0, 12), - ...merged.filter((item) => item.stage === 'denoise').slice(0, 12), + ...merged.filter((item) => item.stage === 'triage').sort(priority).slice(0, 12), + ...merged.filter((item) => item.stage === 'denoise').sort(priority).slice(0, 12), ].sort((left, right) => Date.parse(right.occurredAt || '') - Date.parse(left.occurredAt || '')); } + next.lastIdentified = [...next.recent, next.lastIdentified] + .filter(hasAlertIdentity).sort((a, b) => activityTimestamp(b) - activityTimestamp(a))[0] || null; return next; } @@ -739,36 +819,6 @@ function fullNumber(value) { return new Intl.NumberFormat('zh-CN').format(n); } -function workflowDenoiseActivity(callCount, delta, generatedAt, workflowEvent) { - if (workflowEvent) { - return { - ...workflowEvent, - eventId: `workflow-playback:${callCount}:${workflowEvent.eventId}`, - statsDelta: delta, - workflowCallCount: callCount, - }; - } - const occurredAt = generatedAt || new Date().toISOString(); - return { - eventId: `workflow-denoise:${callCount}:${occurredAt}`, - stage: 'denoise', - status: 'completed', - occurredAt, - triggerSource: 'workflow_stats', - statsDelta: delta, - workflowCallCount: callCount, - hiddenFromQueue: true, - sampleCount: delta, - alert: { - sourceType: 'workflow.db', - threatName: '降噪工作流统计更新', - }, - result: { - clusterId: `累计 ${fullNumber(callCount)}`, - isDuplicate: false, - }, - }; -} function compactNumber(value) { const n = Number(value || 0); @@ -1113,12 +1163,13 @@ function ActivityStageCard({ kind, lane, stats }) { } function AiCore({ stats, activity }) { - const denoiseActive = Boolean(activity.denoise.current); - const triageActive = Boolean(activity.triage.current); + const denoiseActive = activity.denoise.current?.status === 'running'; + const triageActive = activity.triage.current?.status === 'running'; const activeCount = Number(denoiseActive) + Number(triageActive); - const activeEvent = activity.denoise.current || activity.triage.current; + const activeEvent = (denoiseActive && activity.denoise.current) || (triageActive && activity.triage.current) || null; const activeKind = activeEvent?.stage || ''; - const queueCount = activity.denoise.queue.length + activity.triage.queue.length; + const queueCount = buildEventQueueTasks(activity) + .filter((task) => task.state === 'waiting').length; const coreLabel = activeCount === 2 ? '双任务处理中' : denoiseActive @@ -1128,72 +1179,25 @@ function AiCore({ stats, activity }) { const workflowDenoiseActive = activeKind === 'denoise' && ['workflow_stats', 'workflow_execution'].includes(activeEvent?.triggerSource); const workflowExecutionActive = workflowDenoiseActive && activeEvent?.triggerSource === 'workflow_execution'; - const workflowMetricsAvailable = workflowExecutionActive && activeEvent?.result?.metricsAvailable; - const alertName = activeEvent?.alert?.threatName || '未知告警'; - const alertSource = activeEvent?.alert?.sourceType || '未知来源'; - const sourceAddress = activeEvent?.alert?.srcIp || '待识别'; - const targetAddress = activeEvent?.alert?.dstIp || '待识别'; - const operations = workflowExecutionActive && !workflowMetricsAvailable - ? [ - `接入告警 ${alertName}`, - `识别来源 ${alertSource}`, - `关联资产 ${sourceAddress} → ${targetAddress}`, - '等待可用降噪结果', - ] - : workflowExecutionActive - ? [ - `接入原始告警 ${fullNumber(activeEvent?.result?.rawCount)}`, - `完成标准化 ${fullNumber(activeEvent?.result?.normalizedCount)}`, - `过滤与收敛 ${fullNumber(activeEvent?.result?.reducedCount)}`, - `留存研判告警 ${fullNumber(activeEvent?.result?.uniqueCount)}`, - ] - : workflowDenoiseActive - ? [ - '检测 workflow.db 统计更新', - `读取降噪调用增量 +${fullNumber(activeEvent?.statsDelta || 1)}`, - '同步降噪处理状态', - `累计调用 ${fullNumber(activeEvent?.workflowCallCount)}`, - ] + // Keep one coherent alert. A historical fallback is labelled, never grafted + // onto a running execution whose input has not been provided. + const evidenceEvent = [activity.denoise.current, activity.triage.current].find(hasAlertIdentity) + || [...(activity.recent || []), activity.lastIdentified, activity.denoise.last, activity.triage.last] + .filter(hasAlertIdentity).sort((a, b) => activityTimestamp(b) - activityTimestamp(a))[0] + || activeEvent; + const evidenceIsCurrent = evidenceEvent && [activity.denoise.current, activity.triage.current] + .some((event) => event?.eventId === evidenceEvent.eventId && event.status === 'running'); + const evidenceItems = [ + { label: '告警名称', value: displayAlertText(evidenceEvent?.alert?.threatName) || '未提供' }, + { label: '来源类型', value: displayAlertText(evidenceEvent?.alert?.sourceType) || '未提供' }, + { label: '源地址', value: displayAlertText(evidenceEvent?.alert?.srcIp) || '未提供' }, + { label: '目标地址', value: displayAlertText(evidenceEvent?.alert?.dstIp) || '未提供' }, + ]; + const operations = workflowExecutionActive + ? ['降噪工作流执行中', '提取告警特征', '等待降噪结果', '以实际执行状态为准'] : activeKind === 'denoise' - ? [ - `接入 ${activeEvent?.alert?.sourceType || '告警数据'}`, - '提取请求与网络特征', - `匹配相似簇 ${activeEvent?.result?.clusterId || '--'}`, - activeEvent?.result?.isDuplicate ? '输出:重复告警收敛' : '输出:保留代表告警', - ] - : [ - '提取攻击证据', - '关联历史情报与资产', - '执行风险推理', - `生成结论:${activeEvent?.result?.verdictLabel || '待确认'}`, - ]; - const evidenceItems = workflowExecutionActive && !workflowMetricsAvailable - ? [ - { label: '告警名称', value: alertName }, - { label: '来源类型', value: alertSource }, - { label: '源地址', value: sourceAddress }, - { label: '目标地址', value: targetAddress }, - ] - : workflowExecutionActive - ? [ - { label: '原始告警', value: fullNumber(activeEvent?.result?.rawCount) }, - { label: '过滤数量', value: fullNumber(activeEvent?.result?.filterRemovedCount) }, - { label: '去重数量', value: fullNumber(activeEvent?.result?.duplicateCount) }, - { label: '降噪率', value: pct(activeEvent?.result?.reductionRate) }, - ] - : workflowDenoiseActive - ? [ - { label: '本次增量', value: `+${fullNumber(activeEvent?.statsDelta || 1)}` }, - { label: '累计处理', value: fullNumber(activeEvent?.workflowCallCount) }, - { label: '处理模式', value: activity.mode === 'surge' ? '洪峰' : activity.mode === 'burst' ? '批量' : '实时' }, - { label: '当前队列', value: fullNumber(queueCount) }, - ] - : activeEvent ? [ - { label: '攻击源', value: activeEvent.alert?.srcIp || activeEvent.alert?.sourceType || '新告警' }, - { label: '目标资产', value: activeEvent.alert?.dstIp || '待识别资产' }, - { label: activeKind === 'denoise' ? '特征' : '攻击路径', value: activeEvent.alert?.requestUri || activeEvent.alert?.threatName || '特征提取中' }, - { label: activeKind === 'denoise' ? '相似聚类' : '风险判断', value: activeKind === 'denoise' ? `簇 ${activeEvent.result?.clusterId || '--'}` : activityResultText(activeEvent) }, - ] : []; + ? ['告警接入记录', '提取请求与网络特征', '相似特征聚类', '降噪结果展示'] + : ['提取攻击证据', '关联历史情报与资产', '执行风险推理', '等待研判结果']; const statusLabel = activity.connection === 'error' ? '活动数据等待重连' : activeCount @@ -1238,14 +1242,16 @@ function AiCore({ stats, activity }) { key: 'operation-idle', }, `AI新增研判 ${compactNumber(stats.triage.newTriaged)}`), ]), - activeEvent ? h('div', { className: 'ai-evidence-field', key: `evidence-${activeEvent.eventId}` }, evidenceItems.map((item, index) => h('div', { + h('div', { className: 'ai-evidence-field', key: 'evidence', 'aria-label': evidenceIsCurrent ? '当前任务告警' : '最近告警记录' }, evidenceItems.map((item, index) => h('div', { className: `ai-evidence-card evidence-${index + 1}`, key: item.label, style: { animationDelay: `${180 + index * 220}ms` }, }, [ h('span', { key: 'label' }, item.label), h('b', { title: item.value, key: 'value' }, item.value), - ]))) : null, + ]))), + h('span', { className: 'core-evidence-source', key: 'evidence-source' }, + evidenceIsCurrent ? '当前任务告警' : evidenceEvent ? '最近告警记录' : '等待告警数据'), h('div', { className: 'core-particle particle-a', key: 'particle-a' }), h('div', { className: 'core-particle particle-b', key: 'particle-b' }), h('div', { className: 'core-particle particle-c', key: 'particle-c' }), @@ -1622,7 +1628,7 @@ function TimeRefreshPopover({ value, refreshValue, open, onToggle, onApply, onCl } function CommandHeader({ title, timeFilter, refreshKey, timeMenuOpen, setTimeMenuOpen, applyTimeRefresh, stats, loading, refresh, activity }) { - const active = activity.denoise.current || activity.triage.current; + const active = [activity.denoise.current, activity.triage.current].some((event) => event?.status === 'running'); const loadActive = activity.mode !== 'normal' && activity.batch?.receivedCount > 0; const status = activity.connection === 'error' ? '活动通道重连中' @@ -1772,7 +1778,7 @@ function triageContextText(stats) { function CommandActivityLane({ kind, lane, peerLane, stats }) { const event = lane.current || lane.last; - const active = Boolean(lane.current); + const active = lane.current?.status === 'running'; const steps = kind === 'denoise' ? ['接入', '特征', '聚类', '降噪'] : ['证据', '情报', '推理', '结论']; const duration = activityDuration(event); const drumDuration = kind === 'denoise' ? (active ? '2.6s' : '6.6s') : (active ? '7.2s' : '9.2s'); @@ -1780,7 +1786,8 @@ function CommandActivityLane({ kind, lane, peerLane, stats }) { const playbackMode = kind === 'denoise' ? event?.playbackMode : 'normal'; const status = active ? playbackMode === 'surge' ? '洪峰处理' : playbackMode === 'burst' ? '批量处理' : '处理中' - : event ? '最近完成' : '待机巡航'; + : isRunningWorkflowEvent(event) ? '等待处理' + : event?.status === 'failed' ? '最近失败' : event ? '最近完成' : '待机巡航'; const sampleCount = Math.max(Number(event?.sampleCount || 1), 1); const eventTitle = event?.alert?.threatName ? `${event.alert.threatName}${sampleCount > 1 ? ` × ${sampleCount}` : ''}` @@ -1832,8 +1839,8 @@ function CommandActivityLane({ kind, lane, peerLane, stats }) { } function CommandGraph({ stats, activity }) { - const denoiseActive = Boolean(activity.denoise.current); - const triageActive = Boolean(activity.triage.current); + const denoiseActive = activity.denoise.current?.status === 'running'; + const triageActive = activity.triage.current?.status === 'running'; const severityToneFor = (event) => { if (!event) return ''; return severityKey(event.result?.threatSeverity); @@ -1941,78 +1948,31 @@ function activityTimestamp(event) { } function activityTaskKey(event) { - const alertId = String(event?.alert?.id || '').trim(); - return alertId || String(event?.eventId || '').trim(); + // Executions are tasks, not alerts. Two executions can preview the same alert. + return String(event?.eventId || '').trim(); } function buildEventQueueTasks(activity, timeFilter) { - const stateByEventId = new Map(); - for (const kind of ['denoise', 'triage']) { - const lane = activity[kind]; - if (lane.current?.eventId) stateByEventId.set(lane.current.eventId, 'processing'); - for (const event of lane.queue) { - if (event?.eventId) stateByEventId.set(event.eventId, 'waiting'); - } - } - + const taskByKey = new Map(); const allEvents = [ ...(activity.recent || []), - activity.denoise.last, - activity.triage.last, - ...activity.denoise.queue, - ...activity.triage.queue, - activity.denoise.current, - activity.triage.current, - ].filter((event) => event?.eventId && !event.hiddenFromQueue && eventMatchesTimeFilter(event, timeFilter)); - const taskByKey = new Map(); - for (const event of allEvents) { - const key = activityTaskKey(event); - if (!key) continue; - const task = taskByKey.get(key) || { key, denoise: null, triage: null, latestAt: 0 }; - task[event.stage] = event; - task.latestAt = Math.max(task.latestAt, activityTimestamp(event)); - taskByKey.set(key, task); + activity.denoise.last, activity.triage.last, + ...activity.denoise.queue, ...activity.triage.queue, + activity.denoise.current, activity.triage.current, + ].filter((event) => isVisibleActivity(event) && !event.hiddenFromQueue + && event.triggerSource === 'workflow_execution' && (!timeFilter || eventMatchesTimeFilter(event, timeFilter))); + for (const incoming of allEvents) { + const key = activityTaskKey(incoming); + const event = mergeActivityEvent(taskByKey.get(key)?.event, incoming); + const status = String(event.status || '').toLowerCase(); + const state = status === 'running' ? 'processing' + : ['queued', 'pending'].includes(status) ? 'waiting' : 'completed'; + taskByKey.set(key, { key, event, state, stage: event.stage, + [event.stage]: event, latestAt: activityTimestamp(event) }); } - - const tasks = [...taskByKey.values()].map((task) => { - const denoiseState = stateByEventId.get(task.denoise?.eventId) || ''; - const triageState = stateByEventId.get(task.triage?.eventId) || ''; - let state = 'completed'; - let stage = task.triage ? 'triage' : 'denoise'; - if (triageState === 'processing') { - state = 'processing'; - stage = 'triage'; - } else if (denoiseState === 'processing') { - state = 'processing'; - stage = 'denoise'; - } else if (triageState === 'waiting') { - state = 'waiting'; - stage = 'triage'; - } else if (denoiseState === 'waiting') { - state = 'waiting'; - stage = 'denoise'; - } else if (!task.triage && task.denoise && task.denoise.status !== 'failed' && !task.denoise.result?.isDuplicate) { - state = 'waiting'; - stage = 'triage'; - } - return { - ...task, - state, - stage, - event: stage === 'triage' && task.triage ? task.triage : task.denoise, - }; - }); - const stateRank = { processing: 0, waiting: 1, completed: 2 }; - tasks.sort((left, right) => { - const stateDelta = stateRank[left.state] - stateRank[right.state]; - if (stateDelta) return stateDelta; - if (left.state === 'waiting') return right.latestAt - left.latestAt; - if (left.state === 'processing' && left.stage !== right.stage) return left.stage === 'triage' ? -1 : 1; - return right.latestAt - left.latestAt; - }); - - return tasks; + return [...taskByKey.values()].sort((a, b) => stateRank[a.state] - stateRank[b.state] + || b.latestAt - a.latestAt || a.key.localeCompare(b.key)); } function useAnimatedTaskWindow(tasks, transitionKey) { @@ -2041,6 +2001,8 @@ function useAnimatedTaskWindow(tasks, transitionKey) { task.event?.eventId || '', task.event?.playbackStartedAt || '', task.latestAt || '', + JSON.stringify(task.event?.alert || {}), + JSON.stringify(task.event?.result || {}), ].join(':')).join('|'); useEffect(() => { @@ -2102,30 +2064,10 @@ function useAnimatedTaskWindow(tasks, transitionKey) { return displayedTasks; } -function EventQueueProgress({ event }) { - const { useEffect, useRef, useState } = getReact(); - const duration = Math.max(activityDuration(event), 1); - const start = useRef({ eventId: '', value: 0 }); - if (start.current.eventId !== event.eventId) { - start.current = { - eventId: event.eventId, - value: Number(event.playbackStartedAt || Date.now()), - }; - } - const [now, setNow] = useState(Date.now()); - useEffect(() => { - const update = () => setNow(Date.now()); - update(); - const id = window.setInterval(update, 500); - return () => window.clearInterval(id); - }, [event.eventId]); - const elapsed = Math.min(Math.max(now - start.current.value, 0), duration); - const progress = elapsed / duration; - const elapsedSeconds = Math.min(Math.floor(elapsed / 1000), Math.round(duration / 1000)); - return h('div', { className: 'event-rail-progress', 'aria-label': 'AI 任务处理进度' }, [ - h('span', { className: 'event-rail-progress-track', style: { '--queue-progress': progress }, key: 'track' }, [h('i', { key: 'fill' })]), - h('small', { key: 'duration' }, `${elapsedSeconds}s / ${Math.round(duration / 1000)}s`), - ]); +function EventQueueProgress() { + // Animation duration is not workflow progress. Avoid a fabricated countdown. + return h('div', { className: 'event-rail-progress', 'aria-label': '工作流运行状态' }, + h('small', null, '执行中 · 等待结果更新')); } function taskCenterPercent(value) { @@ -2422,10 +2364,9 @@ function CommandAiTaskPanel({ activity, timeFilter }) { filterTransitionKey, ); const counts = { - processing: visibleTasks.filter((task) => task.state === 'processing').length, - waiting: visibleTasks.filter((task) => task.state === 'waiting').length, + processing: tasks.filter((task) => task.state === 'processing').length, + waiting: tasks.filter((task) => task.state === 'waiting').length, }; - const queueCount = visibleTasks.length; const banner = activity.connection === 'error' ? '处理任务连接异常,正在重试' : counts.processing @@ -2435,21 +2376,16 @@ function CommandAiTaskPanel({ activity, timeFilter }) { h('div', { className: cx('event-update-banner', activity.connection === 'error' && 'warn'), key: 'banner' }, banner), h('div', { className: 'event-rail-list', key: 'list' }, visibleTasks.length ? visibleTasks.map((task) => { const event = task.event; - const sampleCount = Math.max(Number(task.denoise?.sampleCount || 1), 1); - const title = `${event?.alert?.threatName || '未知告警'}${sampleCount > 1 ? ` × ${sampleCount}` : ''}`; + const title = displayAlertText(event?.alert?.threatName) || (task.stage === 'triage' ? '研判批次' : '降噪批次 · 数量未提供'); const stageLabel = task.stage === 'triage' ? task.state === 'waiting' ? '待研判' : '智能研判' : task.state === 'waiting' ? '待降噪' : '智能降噪'; const stateLabel = task.state === 'processing' ? '处理中' : '等待处理'; - const detail = event?.triggerSource === 'workflow_execution' - ? task.stage === 'triage' - ? task.state === 'processing' ? '研判工作流处理中' : '研判工作流待处理' - : event.result?.isDuplicate ? '重复告警已收敛' : '降噪处理完成' - : task.state === 'processing' - ? task.stage === 'triage' ? '证据关联与结论生成中' : '特征提取与相似聚类中' - : '等待 AI 处理'; + const detail = task.state === 'processing' + ? task.stage === 'triage' ? '研判工作流处理中' : '降噪工作流处理中' + : task.stage === 'triage' ? '研判工作流排队中' : '降噪工作流排队中'; const hasExecution = Boolean(workflowIdFromEvent(event) && executionIdFromWorkflowEvent(event)); const handleOpen = () => { if (hasExecution) openWorkflowExecutionFromEvent(event); @@ -2776,6 +2712,7 @@ export default function Page() { : { cursor: activityCursor.current, limit: 40, ...timeFilterParams(timeFilter) }; const response = await getApi().page.get('/activity', { params }); const payload = response.data || {}; + if (stopped) return; if (payload.error) throw new Error(payload.error); activityCursor.current = payload.cursor || activityCursor.current; if (!stopped) { @@ -2797,7 +2734,6 @@ export default function Page() { const hasWorkflowCount = rawCallCount !== null && rawCallCount !== undefined && Number.isFinite(Number(rawCallCount)); - let workflowDelta = 0; let workflowChanged = false; if (hasWorkflowCount) { const callCount = Math.max(Math.trunc(Number(rawCallCount)), 0); @@ -2807,15 +2743,6 @@ export default function Page() { callCount > previousProgress.callCount || latestStartedAt > previousProgress.latestStartedAt ); - if (workflowChanged) { - workflowDelta = Math.max(callCount - previousProgress.callCount, 1); - incomingEvents.push(workflowDenoiseActivity( - callCount, - workflowDelta, - payload.generatedAt, - workflowEvents[0], - )); - } workflowProgressByFilter.current.set(workflowFilterKey, { callCount, latestStartedAt }); } const incomingRecentEvents = bootstrap @@ -2823,13 +2750,11 @@ export default function Page() { : workflowChanged ? [...rawIncomingEvents, ...workflowEvents] : rawIncomingEvents; - setActivity((previous) => enqueueActivity( - previous, - incomingEvents, - payload.generatedAt, - incomingRecentEvents, - payload.batch, - )); + setActivity((previous) => { + const next = enqueueActivity(previous, incomingEvents, payload.generatedAt, + incomingRecentEvents, payload.batch, payload.workflowSnapshotComplete === true); + return payload.workflowSnapshotAvailable === false ? { ...next, connection: 'error' } : next; + }); const batch = normalizeActivityBatch(payload.batch); const hasStatsChange = workflowChanged || batch.receivedCount > 0 @@ -6109,12 +6034,23 @@ const CSS = ` inset: 0; pointer-events: none; } +.core-evidence-source { + position: absolute; + z-index: 8; + bottom: 2%; + left: 50%; + transform: translateX(-50%); + color: #9bd5e9; + font-size: 10px; + white-space: nowrap; +} .ai-evidence-card { position: absolute; display: flex; flex-direction: column; width: 104px; min-height: 42px; + pointer-events: auto; padding: 6px 8px; opacity: 0; border: 1px solid color-mix(in srgb, var(--core-task-accent) 48%, transparent); diff --git a/.flocks/flockshub/plugins/webuis/soc_ui/workspace.json b/.flocks/flockshub/plugins/webuis/soc_ui/workspace.json index 3bf045c38..730c229bd 100644 --- a/.flocks/flockshub/plugins/webuis/soc_ui/workspace.json +++ b/.flocks/flockshub/plugins/webuis/soc_ui/workspace.json @@ -1,6 +1,6 @@ { "id": "soc_ui", - "version": "1.1.5", + "version": "1.1.6", "title": "SOC 工作区", "titleEn": "SOC Workspace", "icon": "ShieldCheck", diff --git a/tests/hub/test_hub_catalog.py b/tests/hub/test_hub_catalog.py index 60315efa6..831e7c15e 100644 --- a/tests/hub/test_hub_catalog.py +++ b/tests/hub/test_hub_catalog.py @@ -296,17 +296,17 @@ def test_catalog_uses_webui_workspace_version_for_inferred_installs( entry = {item.id: item for item in list_catalog(plugin_type="webui")}["soc_ui"] - assert entry.version == "1.1.5" + assert entry.version == "1.1.6" assert entry.state == "updateAvailable" assert entry.installedVersion == "1.0.0" - workspace["version"] = "1.1.5" + workspace["version"] = "1.1.6" workspace_path.write_text(json.dumps(workspace), encoding="utf-8") refreshed = {item.id: item for item in list_catalog(plugin_type="webui")}["soc_ui"] assert refreshed.state == "installed" - assert refreshed.installedVersion == "1.1.5" + assert refreshed.installedVersion == "1.1.6" def test_pentest_agents_are_listed_in_agent_catalog(): diff --git a/tests/hub/test_soc_dashboard_schema.py b/tests/hub/test_soc_dashboard_schema.py index 6983b811b..b04835bf7 100644 --- a/tests/hub/test_soc_dashboard_schema.py +++ b/tests/hub/test_soc_dashboard_schema.py @@ -1,10 +1,151 @@ import importlib.util +import gc import json import sqlite3 import sys +import time +import tracemalloc from datetime import datetime, timedelta from pathlib import Path +import pytest + + +def _activity_workflow_db(tmp_path): + path = tmp_path / "workflow.db" + with sqlite3.connect(path) as conn: + conn.execute("""CREATE TABLE workflow_executions ( + id TEXT PRIMARY KEY, workflow_id TEXT, status TEXT, + started_at INTEGER, updated_at INTEGER, + input_params TEXT, output_results TEXT, payload TEXT)""") + conn.execute("CREATE INDEX idx_started ON workflow_executions(workflow_id, started_at DESC)") + conn.execute("CREATE INDEX idx_status ON workflow_executions(workflow_id, status)") + return path + + +def _activity_execution(path, key, *, status="running", started=1000, inputs=None, output=None, payload=None): + with sqlite3.connect(path) as conn: + conn.execute("INSERT INTO workflow_executions VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ( + key, "stream_alert_denoise", status, started, started + 100, + json.dumps(inputs or {}), json.dumps(output or {}), json.dumps(payload or {}), + )) + + +@pytest.mark.parametrize("wrapper", ["direct", "preview", "syslog", "nested"]) +def test_activity_preview_handles_bounded_execution_shapes(wrapper): + handlers = _load_dashboard_handlers() + record = {"id": "a", "threat_name": "scan", "_source_type": "ndr", + "net_real_src_ip": "192.0.2.1", "net_dest_ip": "198.51.100.2"} + output, inputs = {}, {} + if wrapper == "direct": + inputs = {"alerts": [record]} + elif wrapper == "preview": + output = {"unique_alerts": {"_type": "list", "preview": [record]}} + elif wrapper == "syslog": + inputs = {"syslog_message": {"message": "Sep 11 device vendor: " + json.dumps(record)}} + else: + inputs = {"syslog_message": {"message": json.dumps({"data": {"message": json.dumps(record)}})}} + alert, count = handlers._dashboard_execution_detail(json.dumps(output), json.dumps(inputs)) + assert alert == {"id": "a", "threatName": "scan", "sourceType": "ndr", + "srcIp": "192.0.2.1", "dstIp": "198.51.100.2"} + assert count == (1 if wrapper == "direct" else None) + + +def test_activity_does_not_combine_unrelated_alerts_or_guess_source(): + handlers = _load_dashboard_handlers() + output = {"unique_alerts": [ + {"id": "a", "threat_name": "first", "sip": "192.0.2.1"}, + {"id": "b", "dip": "198.51.100.2", "_source_type": "ndr"}, + ]} + alert, _ = handlers._dashboard_execution_detail(json.dumps(output), "{}") + assert alert["id"] == "a" + assert alert["dstIp"] == alert["sourceType"] == "" + + +def test_activity_distinguishes_empty_unknown_and_real_batches(tmp_path): + handlers = _load_dashboard_handlers() + handlers.WORKFLOW_DB = _activity_workflow_db(tmp_path) + _activity_execution(handlers.WORKFLOW_DB, "empty", output={"stats": {"raw_count": 0}}) + _activity_execution(handlers.WORKFLOW_DB, "unknown", inputs={"source_log_type": "ndr"}) + _activity_execution(handlers.WORKFLOW_DB, "nonempty", inputs={"_soc_alert_count": 5}) + _activity_execution(handlers.WORKFLOW_DB, "queued", status="queued", inputs={"_soc_alert_count": 2}) + events = {e["eventId"]: e for e in handlers._get_workflow_recent_events("stream_alert_denoise")} + assert "workflow-execution:empty" not in events + assert events["workflow-execution:unknown"]["result"]["rawCount"] is None + assert events["workflow-execution:unknown"]["alert"]["threatName"] == "降噪批次 · 数量未提供" + assert events["workflow-execution:nonempty"]["result"]["rawCount"] == 5 + assert events["workflow-execution:queued"]["status"] == "queued" + with sqlite3.connect(handlers.WORKFLOW_DB) as conn: + assert conn.execute("SELECT count(*) FROM workflow_executions").fetchone()[0] == 4 + + +def test_activity_keeps_long_running_execution_and_bounds_history(tmp_path): + handlers = _load_dashboard_handlers() + handlers.WORKFLOW_DB = _activity_workflow_db(tmp_path) + _activity_execution(handlers.WORKFLOW_DB, "long-running", started=1000) + for index in range(30): + _activity_execution(handlers.WORKFLOW_DB, f"done-{index}", status="success", started=2000 + index) + snapshot = {"complete": True} + events = handlers._get_workflow_recent_events("stream_alert_denoise", snapshot=snapshot) + assert len(events) == 11 + assert events[0]["eventId"] == "workflow-execution:long-running" + assert events[0]["updatedAt"] != events[0]["occurredAt"] + assert snapshot["complete"] is True + assert all(e["eventId"] != "workflow-execution:long-running" for e in + handlers._get_workflow_recent_events("stream_alert_denoise", 2, 3)) + + +def test_activity_large_payload_and_database_lock_degrade_without_writes(tmp_path): + handlers = _load_dashboard_handlers() + handlers.WORKFLOW_DB = _activity_workflow_db(tmp_path) + _activity_execution(handlers.WORKFLOW_DB, "large", payload={"large": "x" * 1_000_000}, + inputs={"large": "x" * 1_000_000}) + events = handlers._get_workflow_recent_events("stream_alert_denoise") + assert len(json.dumps(events)) < 4096 + with sqlite3.connect(handlers.WORKFLOW_DB) as conn: + conn.execute("BEGIN EXCLUSIVE") + snapshot = {"complete": True} + started = time.monotonic() + assert handlers._get_workflow_recent_events("stream_alert_denoise", snapshot=snapshot) == [] + assert time.monotonic() - started < 1.0 + assert snapshot["complete"] is False + + +def test_activity_truncated_active_snapshot_is_not_authoritative(tmp_path): + handlers = _load_dashboard_handlers() + handlers.WORKFLOW_DB = _activity_workflow_db(tmp_path) + for index in range(12): + _activity_execution(handlers.WORKFLOW_DB, str(index)) + snapshot = {"complete": True} + assert len(handlers._get_workflow_recent_events("stream_alert_denoise", snapshot=snapshot)) <= 20 + assert snapshot["complete"] is False + + +def test_activity_repeated_projection_does_not_retain_payloads(tmp_path): + handlers = _load_dashboard_handlers() + handlers.WORKFLOW_DB = _activity_workflow_db(tmp_path) + for index in range(12): + _activity_execution(handlers.WORKFLOW_DB, str(index), inputs={ + "_soc_alert_preview": {"threat_name": "scan", "sip": "192.0.2.1"}, + "padding": "x" * 20000, + }) + tracemalloc.start(1) + try: + for _ in range(10): + handlers._get_workflow_recent_events("stream_alert_denoise") + gc.collect() + before, _ = tracemalloc.get_traced_memory() + for _ in range(200): + events = handlers._get_workflow_recent_events("stream_alert_denoise") + assert len(events) <= 20 + assert len(json.dumps(events)) < 20000 + gc.collect() + after, peak = tracemalloc.get_traced_memory() + assert after - before < 1_000_000 + assert peak < 8_000_000 + finally: + tracemalloc.stop() + def _load_dashboard_handlers(): handler_path = ( @@ -1888,7 +2029,7 @@ def insert_execution( } assert [event["alert"]["threatName"] for event in activity["workflowEvents"]] == [ "Syslog duplicate", - "降噪批次 · 原始 1 条", + "降噪批次 · 数量未提供", ] events = handlers._get_workflow_recent_events( @@ -1898,7 +2039,7 @@ def insert_execution( ) assert [event["alert"]["threatName"] for event in events] == [ "Syslog duplicate", - "降噪批次 · 原始 1 条", + "降噪批次 · 数量未提供", ] assert events[0]["result"]["rawCount"] == 1 assert events[0]["result"]["uniqueCount"] == 0 diff --git a/webui/src/utils/socDashboardPageRuntime.test.tsx b/webui/src/utils/socDashboardPageRuntime.test.tsx index 888fa93ff..e3240b556 100644 --- a/webui/src/utils/socDashboardPageRuntime.test.tsx +++ b/webui/src/utils/socDashboardPageRuntime.test.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { render, screen, waitFor, within } from '@testing-library/react'; +import { act, render, screen, waitFor, within } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -26,6 +26,29 @@ function setDocumentHidden(value: boolean) { }); } +function workflowEvent(id: string, status = 'running', extra: Record = {}) { + return { + eventId: `workflow-execution:${id}`, triggerSource: 'workflow_execution', + workflowId: 'stream_alert_denoise', stage: 'denoise', status, + occurredAt: new Date().toISOString(), updatedAt: new Date().toISOString(), + alert: { id: 'same-alert', threatName: '边界扫描', sourceType: 'ndr', + srcIp: '192.0.2.1', dstIp: '198.51.100.2' }, + result: { rawCount: 5, metricsAvailable: true }, ...extra, + }; +} + +function mockActivity(read: () => any) { + const fallback = pageGetMock.getMockImplementation()!; + pageGetMock.mockImplementation((path: string, ...args: any[]) => path === '/activity' + ? Promise.resolve().then(() => ({ data: { cursor: 'test-cursor', events: [], recentEvents: [], + workflowSnapshotComplete: true, ...read() } })) + : fallback(path, ...args)); +} + +async function pollActivity() { + await act(async () => { await vi.advanceTimersByTimeAsync(3000); }); +} + describe('SOC dashboard contract page runtime', () => { beforeEach(() => { installContractSdk(); @@ -58,6 +81,7 @@ describe('SOC dashboard contract page runtime', () => { }); afterEach(() => { + vi.useRealTimers(); delete (globalThis as any).__FLOCKS_WEBUI_CONTRACT_SDK__; if (originalDocumentHidden) { Object.defineProperty(document, 'hidden', originalDocumentHidden); @@ -295,6 +319,166 @@ describe('SOC dashboard contract page runtime', () => { expect(screen.queryByText('查看对话')).not.toBeInTheDocument(); }); + it('shows only actual running/queued tasks, not completed playback or empty batches', async () => { + mockActivity(() => ({ workflowEvents: [ + workflowEvent('done', 'completed'), workflowEvent('failed', 'failed'), + workflowEvent('empty', 'running', { alert: { threatName: '降噪批次 · 原始 0 条' }, result: { rawCount: 0 } }), + workflowEvent('active'), workflowEvent('queued', 'queued'), + ] })); + const { container } = render(); + await waitFor(() => expect(container.querySelectorAll('.event-rail-item')).toHaveLength(2)); + const rail = container.querySelector('.event-rail-list') as HTMLElement; + expect(within(rail).getByText('处理中')).toBeInTheDocument(); + expect(within(rail).getByText('等待处理')).toBeInTheDocument(); + expect(within(rail).queryByText(/原始 0 条/)).not.toBeInTheDocument(); + expect(within(rail).queryByText(/降噪处理完成/)).not.toBeInTheDocument(); + expect(within(rail).queryByText(/s \/ .*s/)).not.toBeInTheDocument(); + }); + + it('keeps four identity cards stable when metrics and partial fields arrive', async () => { + vi.useFakeTimers(); + let event = workflowEvent('active'); + mockActivity(() => ({ workflowEvents: [event] })); + const { container } = render(); + await act(async () => {}); + const cards = container.querySelector('.ai-evidence-field') as HTMLElement; + expect(within(cards).getByText('告警名称')).toBeInTheDocument(); + expect(within(cards).getByText('来源类型')).toBeInTheDocument(); + const original = cards.textContent; + event = { ...event, alert: { id: 'same-alert', threatName: '降噪批次', sourceType: 'unknown', srcIp: '', dstIp: '' }, + result: { rawCount: 5, metricsAvailable: false } }; + await pollActivity(); + expect(cards.textContent).toBe(original); + expect(cards.querySelectorAll('.ai-evidence-card')).toHaveLength(4); + expect(within(cards).queryByText('原始告警')).not.toBeInTheDocument(); + }); + + it('refreshes right-hand task metadata even when ID and status are unchanged', async () => { + vi.useFakeTimers(); + let event = workflowEvent('active', 'running', { alert: { threatName: '降噪批次 · 数量未提供' } }); + mockActivity(() => ({ workflowEvents: [event] })); + const { container } = render(); + await act(async () => {}); + event = workflowEvent('active'); + await pollActivity(); + const rail = container.querySelector('.event-rail-list') as HTMLElement; + expect(within(rail).getByText('边界扫描')).toBeInTheDocument(); + expect(within(rail).getByText('192.0.2.1 → 198.51.100.2')).toBeInTheDocument(); + }); + + it('does not replay a terminal workflow or synthesize tasks from counter increases', async () => { + vi.useFakeTimers(); + const running = workflowEvent('active'); + let event = running; + let callCount = 1; + mockActivity(() => ({ workflowEvents: [event], workflowStats: { callCount } })); + const { container } = render(); + await act(async () => {}); + event = { ...running, status: 'completed' }; + callCount += 20; + await pollActivity(); + await act(async () => { await vi.advanceTimersByTimeAsync(500); }); + expect(container.querySelectorAll('.event-rail-item')).toHaveLength(0); + expect(screen.getByText('最近告警记录')).toBeInTheDocument(); + event = running; // out-of-order running response must not resurrect it + await pollActivity(); + expect(container.querySelectorAll('.event-rail-item')).toHaveLength(0); + expect(container.querySelectorAll('.ai-core.core-processing')).toHaveLength(0); + }); + + it('preserves data on failed/incomplete polls and removes absent tasks only with a complete snapshot', async () => { + vi.useFakeTimers(); + let data: any = { workflowEvents: [workflowEvent('active')] }; + mockActivity(() => data); + const { container } = render(); + await act(async () => {}); + data = { workflowEvents: [], workflowSnapshotComplete: false }; + await pollActivity(); + expect(container.querySelectorAll('.event-rail-item')).toHaveLength(1); + data = { error: 'offline' }; + await pollActivity(); + expect(container.querySelectorAll('.event-rail-item')).toHaveLength(1); + expect(screen.getByText('处理任务连接异常,正在重试')).toBeInTheDocument(); + data = { workflowEvents: [], workflowSnapshotComplete: true }; + await act(async () => { await vi.advanceTimersByTimeAsync(6500); }); + await act(async () => { await vi.advanceTimersByTimeAsync(500); }); + expect(container.querySelectorAll('.event-rail-item')).toHaveLength(0); + }); + + it('labels a historical alert fallback without assigning it to an unrelated active batch', async () => { + mockActivity(() => ({ workflowEvents: [workflowEvent('batch', 'running', { + alert: { threatName: '降噪批次 · 数量未提供' }, result: { rawCount: null }, + })], recentEvents: [{ ...workflowEvent('history', 'completed'), triggerSource: 'soc_record' }] })); + const { container } = render(); + await waitFor(() => expect(screen.getByText('最近告警记录')).toBeInTheDocument()); + const rail = container.querySelector('.event-rail-list') as HTMLElement; + expect(within(rail).getByText('降噪批次 · 数量未提供')).toBeInTheDocument(); + expect(within(rail).queryByText('边界扫描')).not.toBeInTheDocument(); + const cards = container.querySelector('.ai-evidence-field') as HTMLElement; + expect(within(cards).getByText('边界扫描')).toBeInTheDocument(); + }); + + it('keeps the visible queue bounded under repeated polling and clears poll timers on unmount', async () => { + vi.useFakeTimers(); + let batch = 0; + mockActivity(() => ({ workflowEvents: Array.from({ length: 20 }, (_, index) => + workflowEvent(`${batch}-${index}`)) })); + const { container, unmount } = render(); + await act(async () => {}); + for (batch = 1; batch <= 20; batch += 1) await pollActivity(); + await act(async () => { await vi.advanceTimersByTimeAsync(500); }); + expect(container.querySelectorAll('.event-rail-item').length).toBeLessThanOrEqual(10); + unmount(); + const calls = pageGetMock.mock.calls.length; + await act(async () => { await vi.advanceTimersByTimeAsync(30000); }); + expect(pageGetMock.mock.calls.length).toBe(calls); + }); + + it('does not mix two different alerts from the same batch execution', async () => { + vi.useFakeTimers(); + let event = workflowEvent('batch'); + mockActivity(() => ({ workflowEvents: [event] })); + const { container } = render(); + await act(async () => {}); + event = workflowEvent('batch', 'running', { alert: { + id: 'different-alert', threatName: '另一条告警', srcIp: '203.0.113.9', + } }); + await pollActivity(); + const cards = container.querySelector('.ai-evidence-field') as HTMLElement; + expect(within(cards).getByText('另一条告警')).toBeInTheDocument(); + expect(within(cards).queryByText('198.51.100.2')).not.toBeInTheDocument(); + expect(within(cards).getAllByText('未提供')).toHaveLength(2); + }); + + it('promotes real running work ahead of an earlier queued task', async () => { + vi.useFakeTimers(); + let events = [workflowEvent('queued', 'queued', { alert: { threatName: '等待中的批次' } })]; + mockActivity(() => ({ workflowEvents: events })); + const { container } = render(); + await act(async () => {}); + events = [...events, workflowEvent('running')]; + await pollActivity(); + const cards = container.querySelector('.ai-evidence-field') as HTMLElement; + expect(within(cards).getByText('边界扫描')).toBeInTheDocument(); + expect(container.querySelectorAll('.event-rail-item')).toHaveLength(2); + expect(container.querySelector('.ai-core')).toHaveClass('core-processing'); + }); + + it('ignores an in-flight activity response after unmount', async () => { + vi.useFakeTimers(); + const fallback = pageGetMock.getMockImplementation()!; + let resolveActivity: (value: any) => void = () => {}; + pageGetMock.mockImplementation((path: string, ...args: any[]) => path === '/activity' + ? new Promise((resolve) => { resolveActivity = resolve; }) : fallback(path, ...args)); + const { unmount } = render(); + await act(async () => {}); + unmount(); + const callCount = pageGetMock.mock.calls.length; + await act(async () => { resolveActivity({ data: { cursor: 'late', workflowEvents: [workflowEvent('late')] } }); }); + await act(async () => { await vi.advanceTimersByTimeAsync(30000); }); + expect(pageGetMock.mock.calls.length).toBe(callCount); + }); + it('reacts to the shared SOC dashboard title change event', async () => { render(); From f1fe9eb1d5de0d9e2546079c4f596565870f38b4 Mon Sep 17 00:00:00 2001 From: John Yin <10972267+john-yin2333@user.noreply.gitee.com> Date: Fri, 11 Sep 2026 11:44:33 +0800 Subject: [PATCH 5/7] fix(soc): bound activity reads and expire unconfirmed tasks (cherry picked from commit c713d529b2eaffb3417141bfee199b74a6656140) --- .flocks/flockshub/index.json | 2 +- .../plugins/webuis/soc_ui/manifest.json | 2 +- .../soc_ui/soc_dashboard/api/handlers.py | 57 ++++++-- .../webuis/soc_ui/soc_dashboard/src/Page.tsx | 73 ++++++++-- .../plugins/webuis/soc_ui/workspace.json | 2 +- tests/hub/test_hub_catalog.py | 6 +- tests/hub/test_soc_dashboard_schema.py | 137 ++++++++++++++++++ .../utils/socDashboardPageRuntime.test.tsx | 81 +++++++++++ 8 files changed, 331 insertions(+), 29 deletions(-) diff --git a/.flocks/flockshub/index.json b/.flocks/flockshub/index.json index 5d5a7e92a..1a8176561 100644 --- a/.flocks/flockshub/index.json +++ b/.flocks/flockshub/index.json @@ -14640,7 +14640,7 @@ "name": "SOC Workspace WebUI", "description": "SOC workspace pages for posture, overview, and alert investigation.", "descriptionCn": "SOC 工作区页面,包含态势、SOC 总览和告警调查。", - "version": "1.1.6", + "version": "1.1.7", "category": "workflow-automation", "tags": [ "siem", diff --git a/.flocks/flockshub/plugins/webuis/soc_ui/manifest.json b/.flocks/flockshub/plugins/webuis/soc_ui/manifest.json index cfa3f5464..bb3542920 100644 --- a/.flocks/flockshub/plugins/webuis/soc_ui/manifest.json +++ b/.flocks/flockshub/plugins/webuis/soc_ui/manifest.json @@ -5,7 +5,7 @@ "name": "SOC Workspace WebUI", "description": "SOC workspace pages for posture, overview, and alert investigation.", "descriptionCn": "SOC 工作区页面,包含态势、SOC 总览和告警调查。", - "version": "1.1.6", + "version": "1.1.7", "author": "Flocks Team", "license": "MIT", "homepage": "", diff --git a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/handlers.py b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/handlers.py index 10723e969..5d78f3d65 100644 --- a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/handlers.py +++ b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/api/handlers.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path -from threading import RLock +from threading import Lock, RLock DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") @@ -77,6 +77,7 @@ _token_usage_cache = {"updatedAt": 0.0, "mtimeNs": 0, "value": None} _TOKEN_USAGE_CACHE_TTL = 30.0 _cache_lock = RLock() +_workflow_activity_read_lock = Lock() _schema_lock = RLock() _schema_ready: set = set() _activity_pruned_at: float = 0 @@ -1131,6 +1132,20 @@ def _get_workflow_recent_events( limit: int = 10, snapshot=None, ) -> list: + # Polling must not multiply native SQLite allocations across dashboard tabs. + # The worker owns this lock until its reads/parsing actually finish, even if + # the HTTP request awaiting asyncio.to_thread is cancelled. + if not _workflow_activity_read_lock.acquire(blocking=False): + if snapshot is not None: + snapshot.update(complete=False, available=False) + return [] + try: + return _read_workflow_recent_events(workflow_name, start_time, end_time, limit, snapshot) + finally: + _workflow_activity_read_lock.release() + + +def _read_workflow_recent_events(workflow_name, start_time, end_time, limit, snapshot): if not WORKFLOW_DB.is_file(): if snapshot is not None: snapshot.update(complete=False, available=False) @@ -1139,25 +1154,32 @@ def _get_workflow_recent_events( try: with closing(sqlite3.connect(f"{WORKFLOW_DB.resolve().as_uri()}?mode=ro", uri=True, timeout=0.2)) as conn: deadline = time.monotonic() + 0.2 - conn.set_progress_handler(lambda: int(time.monotonic() > deadline), 1000) + conn.set_progress_handler(lambda: int(time.monotonic() > deadline), 100) conn.row_factory = sqlite3.Row conn.execute("BEGIN") execution_columns = { row[1] for row in conn.execute("PRAGMA table_info(workflow_executions)").fetchall() } - # Do not materialize unbounded execution payloads in a 3-second UI poll. + # octet_length(column) reads the record's byte-length metadata, not + # the entire TEXT. length(TEXT), including a cast-based fallback, + # is NOT a native-memory bound. Older SQLite must omit previews. + try: + conn.execute("SELECT octet_length('')").fetchone() + has_octet_length = True + except sqlite3.OperationalError: + has_octet_length = False + def preview_column(name): - if name not in execution_columns: + if name not in execution_columns or not has_octet_length: return f"'{{}}' AS {name}" - return f"CASE WHEN length({name}) <= 262144 THEN {name} ELSE '{{}}' END AS {name}" + return f"CASE WHEN octet_length({name}) <= 262144 THEN {name} ELSE '{{}}' END AS {name}" - latest_select = ", ".join([ + metadata_select = ", ".join([ "id", "status", "started_at", _workflow_execution_column_expr(execution_columns, "updated_at", "started_at"), - *(preview_column(name) for name in ("output_results", "input_params", "payload")), ]) - query = f"SELECT {latest_select} FROM workflow_executions WHERE workflow_id = ?" + query = f"SELECT {metadata_select} FROM workflow_executions WHERE workflow_id = ?" query_params = [workflow_name] if start_time > 0 and end_time > 0: query += " AND started_at >= ? AND started_at <= ?" @@ -1168,7 +1190,22 @@ def preview_column(name): [*query_params, row_limit], ).fetchall() recent_rows = conn.execute(query + " ORDER BY started_at DESC LIMIT ?", [*query_params, row_limit]).fetchall() - rows = list({row["id"]: row for row in [*active_rows, *recent_rows]}.values()) + ids = list(dict.fromkeys(row["id"] for row in [*active_rows, *recent_rows])) + if time.monotonic() > deadline: + raise sqlite3.OperationalError("activity read budget exceeded") + # Select payloads only AFTER bounding IDs; a sorting query must not + # materialize previews for every matching execution before LIMIT. + rows = [] + if ids: + previews = ", ".join(preview_column(name) for name in ("output_results", "input_params", "payload")) + placeholders = ",".join("?" for _ in ids) + by_id = {row["id"]: row for row in conn.execute( + f"SELECT {metadata_select}, {previews} FROM workflow_executions WHERE id IN ({placeholders})", + ids, + ).fetchall()} + rows = [by_id[key] for key in ids] + if time.monotonic() > deadline: + raise sqlite3.OperationalError("activity read budget exceeded") if snapshot is not None and len(active_rows) == row_limit: snapshot["complete"] = False except Exception: @@ -1226,7 +1263,7 @@ def preview_column(name): "sampleCount": max(unique_count, 1), "alert": { **alert, - "id": alert.get("id") or execution_id, + "id": alert.get("id") or "", "threatName": threat_name, }, "result": { diff --git a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/Page.tsx b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/Page.tsx index 15126b392..12ff14012 100644 --- a/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/Page.tsx +++ b/.flocks/flockshub/plugins/webuis/soc_ui/soc_dashboard/src/Page.tsx @@ -86,6 +86,7 @@ const EMPTY_STATS = { const ACTIVITY_QUEUE_LIMIT = 8; const EVENT_RAIL_TASK_LIMIT = 10; const ACTIVITY_POLL_MS = 3000; +const WORKFLOW_CONFIRMATION_TTL_MS = 30000; const ACTIVITY_REPLAY_WINDOW_MS = 10 * 60 * 1000; const ACTIVITY_SEEN_KEY = 'soc-dashboard-seen-activity-v1'; const EVENT_RAIL_DEFAULT_WIDTH = 330; @@ -533,11 +534,18 @@ function mergeActivityEvent(previous, incoming) { if (incomingTime < previousTime) return previous; // A late polling response cannot resurrect a terminal execution. if (previous.triggerSource === 'workflow_execution' - && !isRunningWorkflowEvent(previous) && isRunningWorkflowEvent(incoming)) return previous; + && ['completed', 'success', 'failed', 'cancelled'].includes(previous.status) + && isRunningWorkflowEvent(incoming)) return previous; + if (incoming.status === 'unconfirmed' && previous.lastConfirmedAt > incoming.lastConfirmedAt) return previous; const incomingIdentity = hasAlertIdentity(incoming); - const differentAlert = incomingIdentity && displayAlertText(incoming.alert?.id) - && displayAlertText(previous.alert?.id) && incoming.alert.id !== previous.alert.id; - const alert = differentAlert ? {} : { ...previous.alert }; + const incomingId = displayAlertText(incoming.alert?.id); + const previousId = displayAlertText(previous.alert?.id); + // An execution ID is not an alert ID (including responses from older builds). + // Without a shared real identity, use a whole preview, never graft endpoints. + const sameAlert = incomingId && incomingId === previousId + && incomingId !== executionIdFromWorkflowEvent(incoming) + && previousId !== executionIdFromWorkflowEvent(previous); + const alert = incomingIdentity && !sameAlert ? {} : { ...previous.alert }; for (const [key, value] of Object.entries(incoming.alert || {})) { if (displayAlertText(value)) { if (key === 'id' && !incomingIdentity && hasAlertIdentity(previous)) continue; @@ -549,9 +557,33 @@ function mergeActivityEvent(previous, incoming) { ...previous, ...incoming, alert, result: { ...previous.result, ...incoming.result }, playbackStartedAt: previous.playbackStartedAt, + lastConfirmedAt: Math.max(previous.lastConfirmedAt || 0, incoming.lastConfirmedAt || 0), }; } +function expireWorkflowActivity(previous, now = Date.now()) { + let changed = false; + const expire = (event) => { + if (!isRunningWorkflowEvent(event) || !event.lastConfirmedAt + || now - event.lastConfirmedAt < WORKFLOW_CONFIRMATION_TTL_MS) return event; + changed = true; + // A missing/truncated response is not evidence of completion or failure. + return { ...event, status: 'unconfirmed' }; + }; + const next = { ...previous, recent: previous.recent.map(expire), lastIdentified: expire(previous.lastIdentified) }; + for (const kind of ['denoise', 'triage']) { + const lane = previous[kind]; + const current = expire(lane.current); + next[kind] = { + ...lane, + current: current?.status === 'unconfirmed' ? null : current, + last: current?.status === 'unconfirmed' ? current : expire(lane.last), + queue: lane.queue.map(expire).filter((event) => event.status !== 'unconfirmed'), + }; + } + return changed ? next : previous; +} + function normalizeActivityBatch(raw) { const batch = { ...emptyActivityBatch(), ...(raw || {}) }; for (const key of ['windowMs', 'receivedCount', 'duplicateCount', 'uniqueCount', 'clusterCount', 'triageUpdatedCount', 'sampledCount', 'suppressedCount', 'ratePerSecond']) { @@ -571,11 +603,14 @@ function resolveActivityMode(previous, batch) { } function enqueueActivity(previous, events, generatedAt, recentEvents, rawBatch, workflowSnapshotComplete = false) { + previous = expireWorkflowActivity(previous); const batch = normalizeActivityBatch(rawBatch); const modeState = resolveActivityMode(previous, batch); const hasBatch = batch.receivedCount > 0 || batch.triageUpdatedCount > 0; - const incomingEvents = (events || []).filter(isVisibleActivity); - const incomingRecentEvents = (recentEvents || []).filter(isVisibleActivity); + const confirmed = (event) => event.triggerSource === 'workflow_execution' + ? { ...event, lastConfirmedAt: Date.now() } : event; + const incomingEvents = (events || []).filter(isVisibleActivity).map(confirmed); + const incomingRecentEvents = (recentEvents || []).filter(isVisibleActivity).map(confirmed); if ( !hasBatch && !workflowSnapshotComplete @@ -599,10 +634,11 @@ function enqueueActivity(previous, events, generatedAt, recentEvents, rawBatch, }; if (workflowSnapshotComplete) { const ids = new Set(incomingEvents.map((event) => event.eventId)); - const retained = (event) => !isRunningWorkflowEvent(event) || ids.has(event.eventId); + const retained = (event) => !(isRunningWorkflowEvent(event) || event?.status === 'unconfirmed') || ids.has(event.eventId); for (const kind of ['denoise', 'triage']) { const lane = next[kind]; if (lane.current && !retained(lane.current)) lane.current = null; + if (lane.last?.status === 'unconfirmed' && !retained(lane.last)) lane.last = null; lane.queue = lane.queue.filter(retained); } next.recent = next.recent.filter(retained); @@ -1786,7 +1822,7 @@ function CommandActivityLane({ kind, lane, peerLane, stats }) { const playbackMode = kind === 'denoise' ? event?.playbackMode : 'normal'; const status = active ? playbackMode === 'surge' ? '洪峰处理' : playbackMode === 'burst' ? '批量处理' : '处理中' - : isRunningWorkflowEvent(event) ? '等待处理' + : event?.status === 'unconfirmed' ? '状态待确认' : isRunningWorkflowEvent(event) ? '等待处理' : event?.status === 'failed' ? '最近失败' : event ? '最近完成' : '待机巡航'; const sampleCount = Math.max(Number(event?.sampleCount || 1), 1); const eventTitle = event?.alert?.threatName @@ -1965,12 +2001,12 @@ function buildEventQueueTasks(activity, timeFilter) { const key = activityTaskKey(incoming); const event = mergeActivityEvent(taskByKey.get(key)?.event, incoming); const status = String(event.status || '').toLowerCase(); - const state = status === 'running' ? 'processing' + const state = status === 'unconfirmed' ? 'unconfirmed' : status === 'running' ? 'processing' : ['queued', 'pending'].includes(status) ? 'waiting' : 'completed'; taskByKey.set(key, { key, event, state, stage: event.stage, [event.stage]: event, latestAt: activityTimestamp(event) }); } - const stateRank = { processing: 0, waiting: 1, completed: 2 }; + const stateRank = { processing: 0, waiting: 1, unconfirmed: 2, completed: 3 }; return [...taskByKey.values()].sort((a, b) => stateRank[a.state] - stateRank[b.state] || b.latestAt - a.latestAt || a.key.localeCompare(b.key)); } @@ -2366,12 +2402,14 @@ function CommandAiTaskPanel({ activity, timeFilter }) { const counts = { processing: tasks.filter((task) => task.state === 'processing').length, waiting: tasks.filter((task) => task.state === 'waiting').length, + unconfirmed: tasks.filter((task) => task.state === 'unconfirmed').length, }; const banner = activity.connection === 'error' ? '处理任务连接异常,正在重试' : counts.processing ? `AI 正在并行处理 ${counts.processing} 个任务` - : counts.waiting ? '最新 10 条待处理任务' : '等待新的降噪或研判任务'; + : counts.waiting ? '最新 10 条待处理任务' + : counts.unconfirmed ? '任务状态待确认,等待数据更新' : '等待新的降噪或研判任务'; return [ h('div', { className: cx('event-update-banner', activity.connection === 'error' && 'warn'), key: 'banner' }, banner), h('div', { className: 'event-rail-list', key: 'list' }, visibleTasks.length ? visibleTasks.map((task) => { @@ -2382,8 +2420,10 @@ function CommandAiTaskPanel({ activity, timeFilter }) { : task.state === 'waiting' ? '待降噪' : '智能降噪'; const stateLabel = task.state === 'processing' ? '处理中' - : '等待处理'; - const detail = task.state === 'processing' + : task.state === 'unconfirmed' ? '状态待确认' : '等待处理'; + const detail = task.state === 'unconfirmed' + ? '近期未收到该任务的状态,尚不能确认是否结束' + : task.state === 'processing' ? task.stage === 'triage' ? '研判工作流处理中' : '降噪工作流处理中' : task.stage === 'triage' ? '研判工作流排队中' : '降噪工作流排队中'; const hasExecution = Boolean(workflowIdFromEvent(event) && executionIdFromWorkflowEvent(event)); @@ -2596,6 +2636,12 @@ export default function Page() { void loadStats(timeFilter); }, [loadStats, timeFilter]); + useEffect(() => { + // Also expires stale state while requests are failing or remain in flight. + const timer = window.setInterval(() => setActivity(expireWorkflowActivity), ACTIVITY_POLL_MS); + return () => window.clearInterval(timer); + }, []); + useEffect(() => { const intervalMs = REFRESH_INTERVAL_MS[refreshKey]; if (!intervalMs) return undefined; @@ -5557,6 +5603,7 @@ const CSS = ` .event-stage { color: #83aef1; background: rgba(52,86,143,.5); } .event-rail-item.state-processing .event-stage { color: #57e1b5; background: rgba(23,111,83,.48); } .event-rail-item.state-waiting .event-stage { color: #d6a95e; background: rgba(120,81,23,.38); } +.event-rail-item.state-unconfirmed .event-stage { color: #a9c0d0; background: rgba(75,98,118,.32); } .event-rail-item > strong, .event-rail-item > span, .event-rail-item > small { diff --git a/.flocks/flockshub/plugins/webuis/soc_ui/workspace.json b/.flocks/flockshub/plugins/webuis/soc_ui/workspace.json index 730c229bd..72f4ffa42 100644 --- a/.flocks/flockshub/plugins/webuis/soc_ui/workspace.json +++ b/.flocks/flockshub/plugins/webuis/soc_ui/workspace.json @@ -1,6 +1,6 @@ { "id": "soc_ui", - "version": "1.1.6", + "version": "1.1.7", "title": "SOC 工作区", "titleEn": "SOC Workspace", "icon": "ShieldCheck", diff --git a/tests/hub/test_hub_catalog.py b/tests/hub/test_hub_catalog.py index 831e7c15e..5bea2aa6d 100644 --- a/tests/hub/test_hub_catalog.py +++ b/tests/hub/test_hub_catalog.py @@ -296,17 +296,17 @@ def test_catalog_uses_webui_workspace_version_for_inferred_installs( entry = {item.id: item for item in list_catalog(plugin_type="webui")}["soc_ui"] - assert entry.version == "1.1.6" + assert entry.version == "1.1.7" assert entry.state == "updateAvailable" assert entry.installedVersion == "1.0.0" - workspace["version"] = "1.1.6" + workspace["version"] = "1.1.7" workspace_path.write_text(json.dumps(workspace), encoding="utf-8") refreshed = {item.id: item for item in list_catalog(plugin_type="webui")}["soc_ui"] assert refreshed.state == "installed" - assert refreshed.installedVersion == "1.1.6" + assert refreshed.installedVersion == "1.1.7" def test_pentest_agents_are_listed_in_agent_catalog(): diff --git a/tests/hub/test_soc_dashboard_schema.py b/tests/hub/test_soc_dashboard_schema.py index b04835bf7..127171eda 100644 --- a/tests/hub/test_soc_dashboard_schema.py +++ b/tests/hub/test_soc_dashboard_schema.py @@ -1,12 +1,16 @@ import importlib.util +import asyncio import gc import json import sqlite3 +import subprocess import sys import time import tracemalloc from datetime import datetime, timedelta from pathlib import Path +from threading import Event +from types import SimpleNamespace import pytest @@ -121,6 +125,139 @@ def test_activity_truncated_active_snapshot_is_not_authoritative(tmp_path): assert snapshot["complete"] is False +def test_activity_does_not_substitute_execution_id_for_missing_alert_id(tmp_path): + handlers = _load_dashboard_handlers() + handlers.WORKFLOW_DB = _activity_workflow_db(tmp_path) + _activity_execution(handlers.WORKFLOW_DB, "batch", inputs={ + "alerts": [{"threat_name": "scan", "sip": "192.0.2.1"}], + }) + event = handlers._get_workflow_recent_events("stream_alert_denoise")[0] + assert event["eventId"] == "workflow-execution:batch" + assert event["alert"]["id"] == "" + + +def test_activity_old_sqlite_omits_payloads_instead_of_using_unsafe_length(tmp_path, monkeypatch): + handlers = _load_dashboard_handlers() + handlers.WORKFLOW_DB = _activity_workflow_db(tmp_path) + _activity_execution(handlers.WORKFLOW_DB, "old-sqlite", inputs={ + "alerts": [{"threat_name": "must not read this"}], + }) + original_connect = sqlite3.connect + queries = [] + + class OldSQLiteConnection(sqlite3.Connection): + def execute(self, sql, *args): + queries.append(sql) + if sql == "SELECT octet_length('')": + raise sqlite3.OperationalError("no such function: octet_length") + return super().execute(sql, *args) + + monkeypatch.setattr(handlers.sqlite3, "connect", lambda *a, **kw: original_connect(*a, **kw, factory=OldSQLiteConnection)) + events = handlers._get_workflow_recent_events("stream_alert_denoise") + assert events[0]["status"] == "running" + assert "must not read this" not in json.dumps(events) + assert not any("CASE WHEN" in query or "length(payload)" in query for query in queries) + + +def test_activity_reader_does_not_wait_or_open_another_db_when_busy(tmp_path, monkeypatch): + handlers = _load_dashboard_handlers() + handlers.WORKFLOW_DB = _activity_workflow_db(tmp_path) + _activity_execution(handlers.WORKFLOW_DB, "active") + with handlers._workflow_activity_read_lock: + with monkeypatch.context() as patch: + def unexpected_connect(*args, **kwargs): + raise AssertionError("contending reader must not open SQLite") + patch.setattr(handlers.sqlite3, "connect", unexpected_connect) + snapshot = {"complete": True, "available": True} + assert handlers._get_workflow_recent_events("stream_alert_denoise", snapshot=snapshot) == [] + assert snapshot == {"complete": False, "available": False} + assert handlers._get_workflow_recent_events("stream_alert_denoise") + + +@pytest.mark.asyncio +async def test_cancelled_request_keeps_activity_read_slot_until_worker_exits(monkeypatch): + handlers = _load_dashboard_handlers() + entered, release = Event(), Event() + + def blocked_read(*args): + entered.set() + assert release.wait(3), "test must release its worker" + return [] + + monkeypatch.setattr(handlers, "_read_workflow_recent_events", blocked_read) + request = asyncio.create_task(asyncio.to_thread(handlers._get_workflow_recent_events, "stream_alert_denoise")) + try: + assert await asyncio.to_thread(entered.wait, 3) + request.cancel() + with pytest.raises(asyncio.CancelledError): + await request + assert handlers._workflow_activity_read_lock.locked() + snapshot = {"complete": True, "available": True} + assert handlers._get_workflow_recent_events("stream_alert_denoise", snapshot=snapshot) == [] + assert snapshot == {"complete": False, "available": False} + finally: + release.set() + assert await asyncio.to_thread(handlers._workflow_activity_read_lock.acquire, True, 3) + handlers._workflow_activity_read_lock.release() + + +def test_activity_preview_limit_counts_utf8_bytes_not_characters(tmp_path): + handlers = _load_dashboard_handlers() + handlers.WORKFLOW_DB = _activity_workflow_db(tmp_path) + _activity_execution(handlers.WORKFLOW_DB, "unicode") + inputs = json.dumps({"padding": "中" * 100_000, "alerts": [{"threat_name": "oversized-preview"}]}, ensure_ascii=False) + assert len(inputs) < 262144 < len(inputs.encode("utf-8")) + with sqlite3.connect(handlers.WORKFLOW_DB) as conn: + conn.execute("UPDATE workflow_executions SET input_params = ?", (inputs,)) + events = handlers._get_workflow_recent_events("stream_alert_denoise") + assert events[0]["status"] == "running" + assert "oversized-preview" not in json.dumps(events) + + +def test_activity_deadline_is_checked_even_without_sqlite_progress_callback(tmp_path, monkeypatch): + handlers = _load_dashboard_handlers() + handlers.WORKFLOW_DB = _activity_workflow_db(tmp_path) + _activity_execution(handlers.WORKFLOW_DB, "active") + ticks = iter([100.0, 101.0]) + monkeypatch.setattr(handlers, "time", SimpleNamespace(monotonic=lambda: next(ticks, 101.0))) + snapshot = {"complete": True, "available": True} + assert handlers._get_workflow_recent_events("stream_alert_denoise", snapshot=snapshot) == [] + assert snapshot == {"complete": False, "available": False} + # Errors must release the worker-owned lock as well. + assert handlers._workflow_activity_read_lock.acquire(blocking=False) + handlers._workflow_activity_read_lock.release() + + +@pytest.mark.skipif(sys.platform not in {"darwin", "linux"}, reason="native RSS via resource is Unix-only") +def test_activity_large_text_is_not_materialized_in_native_sqlite_memory(tmp_path): + handlers = _load_dashboard_handlers() + db = _activity_workflow_db(tmp_path) + _activity_execution(db, "old-large-running", payload={"padding": "x" * (32 * 1024 * 1024)}) + for index in range(12): + _activity_execution(db, f"new-done-{index}", status="success", started=2000 + index) + # A fresh process excludes fixture allocations; tracemalloc alone cannot + # see SQLite's native buffers. Do not start the application or touch user DBs. + script = ''' +import importlib.util, json, resource, sys +from pathlib import Path +spec = importlib.util.spec_from_file_location("soc_native_rss_test", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +module.WORKFLOW_DB = Path(sys.argv[2]) +before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +events = module._get_workflow_recent_events("stream_alert_denoise") +after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +scale = 1024**2 if sys.platform == "darwin" else 1024 +print(json.dumps({"growth_mib": (after - before) / scale, "events": len(events)})) +''' + result = subprocess.run([sys.executable, "-B", "-c", script, handlers.__file__, str(db)], + check=True, text=True, capture_output=True, timeout=20) + measurement = json.loads(result.stdout) + assert measurement["events"] == 11 + assert measurement["growth_mib"] < 16, measurement + + def test_activity_repeated_projection_does_not_retain_payloads(tmp_path): handlers = _load_dashboard_handlers() handlers.WORKFLOW_DB = _activity_workflow_db(tmp_path) diff --git a/webui/src/utils/socDashboardPageRuntime.test.tsx b/webui/src/utils/socDashboardPageRuntime.test.tsx index e3240b556..0f425e94e 100644 --- a/webui/src/utils/socDashboardPageRuntime.test.tsx +++ b/webui/src/utils/socDashboardPageRuntime.test.tsx @@ -464,6 +464,87 @@ describe('SOC dashboard contract page runtime', () => { expect(container.querySelector('.ai-core')).toHaveClass('core-processing'); }); + it.each(['', 'batch'])('replaces a whole preview when the alert ID is missing or a legacy execution fallback (%s)', async (id) => { + vi.useFakeTimers(); + let event = workflowEvent('batch', 'running', { alert: { + id, threatName: '第一条告警', srcIp: '192.0.2.1', dstIp: '198.51.100.2', sourceType: 'ndr', + } }); + mockActivity(() => ({ workflowEvents: [event] })); + const { container } = render(); + await act(async () => {}); + event = { ...event, alert: { id, threatName: '第二条告警', srcIp: '203.0.113.9' } }; + await pollActivity(); + const cards = container.querySelector('.ai-evidence-field') as HTMLElement; + expect(within(cards).getByText('第二条告警')).toBeInTheDocument(); + expect(within(cards).getAllByText('未提供')).toHaveLength(2); + expect(within(cards).queryByText('198.51.100.2')).not.toBeInTheDocument(); + const rail = container.querySelector('.event-rail-list') as HTMLElement; + expect(within(rail).queryByText(/198\.51\.100\.2/)).not.toBeInTheDocument(); + }); + + it('releases an unconfirmed current task when busy snapshots keep omitting its final state', async () => { + vi.useFakeTimers(); + let events = [workflowEvent('old', 'running', { alert: { id: 'old-alert', threatName: '旧任务' } })]; + mockActivity(() => ({ workflowEvents: events, workflowSnapshotComplete: false })); + const { container } = render(); + await act(async () => {}); + events = Array.from({ length: 10 }, (_, index) => workflowEvent(`new-${index}`, 'running', { + alert: { id: `new-alert-${index}`, threatName: `新任务-${index}` }, + })); + for (let i = 0; i < 12; i += 1) await pollActivity(); + await act(async () => { await vi.advanceTimersByTimeAsync(500); }); + const cards = container.querySelector('.ai-evidence-field') as HTMLElement; + expect(within(cards).queryByText('旧任务')).not.toBeInTheDocument(); + expect(within(cards).getByText(/新任务-/)).toBeInTheDocument(); + expect(screen.getByText('AI 正在并行处理 10 个任务')).toBeInTheDocument(); + }); + + it('does not expire a long-running task whose unchanged status is still confirmed by polls', async () => { + vi.useFakeTimers(); + const event = workflowEvent('long-running'); + mockActivity(() => ({ workflowEvents: [event], workflowSnapshotComplete: false })); + const { container } = render(); + await act(async () => {}); + for (let i = 0; i < 25; i += 1) await pollActivity(); + expect(container.querySelector('.event-rail-item')).toHaveClass('state-processing'); + expect(container.querySelector('.ai-core')).toHaveClass('core-processing'); + expect(screen.queryByText('状态待确认')).not.toBeInTheDocument(); + }); + + it('marks tasks unconfirmed even during a hung request and restores them on a fresh confirmation', async () => { + vi.useFakeTimers(); + const event = workflowEvent('active'); + mockActivity(() => ({ workflowEvents: [event] })); + const { container } = render(); + await act(async () => {}); + const fallback = pageGetMock.getMockImplementation()!; + let resolveActivity: (value: any) => void = () => {}; + pageGetMock.mockImplementation((path: string, ...args: any[]) => path === '/activity' + ? new Promise((resolve) => { resolveActivity = resolve; }) : fallback(path, ...args)); + await act(async () => { await vi.advanceTimersByTimeAsync(33000); }); + expect(container.querySelector('.event-rail-item')).toHaveClass('state-unconfirmed'); + expect(container.querySelector('.ai-core')).not.toHaveClass('core-processing'); + expect(within(container.querySelector('.event-rail-list') as HTMLElement).getByText('状态待确认')).toBeInTheDocument(); + await act(async () => { resolveActivity({ data: { workflowEvents: [event], workflowSnapshotComplete: true } }); }); + expect(container.querySelector('.event-rail-item')).toHaveClass('state-processing'); + expect(container.querySelector('.ai-core')).toHaveClass('core-processing'); + }); + + it('removes unconfirmed tasks when an authoritative snapshot confirms their absence', async () => { + vi.useFakeTimers(); + let data: any = { workflowEvents: [workflowEvent('old')] }; + mockActivity(() => data); + const { container } = render(); + await act(async () => {}); + data = { workflowEvents: [], workflowSnapshotComplete: false }; + await act(async () => { await vi.advanceTimersByTimeAsync(33000); }); + expect(container.querySelector('.event-rail-item')).toHaveClass('state-unconfirmed'); + data = { workflowEvents: [], workflowSnapshotComplete: true }; + await pollActivity(); + await act(async () => { await vi.advanceTimersByTimeAsync(500); }); + expect(container.querySelectorAll('.event-rail-item')).toHaveLength(0); + }); + it('ignores an in-flight activity response after unmount', async () => { vi.useFakeTimers(); const fallback = pageGetMock.getMockImplementation()!; From ff17fcd5d7ff17338dc7b8e5bba55d5b2eff3091 Mon Sep 17 00:00:00 2001 From: stephamie7 <1223696150@qq.com> Date: Mon, 14 Sep 2026 15:08:59 +0800 Subject: [PATCH 6/7] feat(provider): add DeepSeek V4.1 Flash to ThreatBook catalog --- flocks/provider/catalog.json | 58 ++++++++++++++++++++++++ tests/provider/test_chinese_providers.py | 6 ++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/flocks/provider/catalog.json b/flocks/provider/catalog.json index 665ef4682..8b360cf80 100644 --- a/flocks/provider/catalog.json +++ b/flocks/provider/catalog.json @@ -52,6 +52,35 @@ "THREATBOOK_CN_LLM_API_KEY" ], "models": { + "deepseek-v4.1-flash": { + "name": "deepseek-v4.1-flash", + "family": "deepseek-v4", + "capabilities": { + "supports_tools": true, + "supports_vision": true, + "supports_reasoning": true, + "thinking_level_map": { + "minimal": "low", + "low": "low", + "medium": "high", + "high": "high", + "xhigh": "high", + "max": "max" + }, + "supports_streaming": true + }, + "limits": { + "context_window": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 384000 + }, + "pricing": { + "input": 1.0, + "output": 4.0, + "cache_read": 0.1, + "currency": "CNY" + } + }, "deepseek-v4-flash-0731": { "name": "deepseek-v4-flash-0731", "family": "deepseek-v4", @@ -348,6 +377,35 @@ "THREATBOOK_IO_LLM_API_KEY" ], "models": { + "deepseek-v4.1-flash": { + "name": "deepseek-v4.1-flash", + "family": "deepseek-v4", + "capabilities": { + "supports_tools": true, + "supports_vision": true, + "supports_reasoning": true, + "thinking_level_map": { + "minimal": "low", + "low": "low", + "medium": "high", + "high": "high", + "xhigh": "high", + "max": "max" + }, + "supports_streaming": true + }, + "limits": { + "context_window": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 384000 + }, + "pricing": { + "input": 1.0, + "output": 4.0, + "cache_read": 0.1, + "currency": "CNY" + } + }, "deepseek-v4-flash-0731": { "name": "deepseek-v4-flash-0731", "family": "deepseek-v4", diff --git a/tests/provider/test_chinese_providers.py b/tests/provider/test_chinese_providers.py index 3b3945f2f..eb424507a 100644 --- a/tests/provider/test_chinese_providers.py +++ b/tests/provider/test_chinese_providers.py @@ -341,9 +341,10 @@ def test_threatbook_cn_llm_catalog(self): "kimi-k2.6", "deepseek-v4-flash", "deepseek-v4-flash-0731", + "deepseek-v4.1-flash", } - assert models[0].id == "deepseek-v4-flash-0731" + assert models[0].id == "deepseek-v4.1-flash" kimi_code = next(m for m in models if m.id == "kimi-k2.7-code") assert kimi_code.capabilities.supports_vision is True assert kimi_code.capabilities.supports_reasoning is True @@ -412,9 +413,10 @@ def test_threatbook_io_llm_catalog(self): "qwen3-max", "deepseek-v4-flash", "deepseek-v4-flash-0731", + "deepseek-v4.1-flash", } - assert models[0].id == "deepseek-v4-flash-0731" + assert models[0].id == "deepseek-v4.1-flash" kimi_code = next(m for m in models if m.id == "kimi-k2.7-code") assert kimi_code.capabilities.supports_vision is True assert kimi_code.capabilities.supports_reasoning is True From 676d2106b90cdbff3b9b27719d61111539eba44f Mon Sep 17 00:00:00 2001 From: stephamie7 <1223696150@qq.com> Date: Mon, 14 Sep 2026 16:16:36 +0800 Subject: [PATCH 7/7] chore/update-version-2026.9.14 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bc211e7c0..bacb4251e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "flocks" -version = "v2026.9.9" +version = "v2026.9.14" description = "AI-Native SecOps platform with multi-agent collaboration" authors = [ {name = "Flocks Team", email = "team@example.com"} diff --git a/uv.lock b/uv.lock index 5a4498bf1..c3f451a95 100644 --- a/uv.lock +++ b/uv.lock @@ -553,7 +553,7 @@ wheels = [ [[package]] name = "flocks" -version = "2026.9.9" +version = "2026.9.14" source = { editable = "." } dependencies = [ { name = "aiofiles" },