From cb9f24bc4720eb2befdecce583da390823c888a9 Mon Sep 17 00:00:00 2001 From: ankur-paan Date: Wed, 28 Oct 2026 11:10:37 +0000 Subject: [PATCH 1/4] fix: v1.0.0 security, fleet concurrency, wasm sandbox, and docs remediation --- README.md | 4 +- crates/rekuiper-core/src/manager.rs | 12 +- crates/rekuiper-core/src/wasm.rs | 213 +- crates/rekuiper-server/src/engine/mod.rs | 2552 +---------------- crates/rekuiper-server/src/engine/sources.rs | 2533 ++++++++++++++++ crates/rekuiper-server/src/engine/windows.rs | 20 +- crates/rekuiper-server/src/fleet.rs | 132 +- .../rekuiper-server/src/handlers/metrics.rs | 43 +- crates/rekuiper-server/src/lib.rs | 35 +- crates/rekuiper-server/src/routes.rs | 73 +- crates/rekuiper-server/src/state.rs | 97 +- .../rekuiper-server/tests/test_otlp_and_ui.rs | 50 + .../tests/test_phase4_fleet_control_plane.rs | 108 + crates/rule-engine/Cargo.toml | 8 - crates/rule-engine/src/bin/kuiperd.rs | 6 - crates/rule-engine/src/bin/rekuiperd.rs | 6 - docs/en_US/CONTRIBUTING.md | 8 +- docs/en_US/api/restapi/authentication.md | 31 + docs/en_US/benchmarks/overview.md | 23 +- docs/en_US/concepts/streaming/join.md | 25 +- docs/en_US/concepts/streaming/overview.md | 2 + docs/en_US/concepts/streaming/windowing.md | 4 + .../configuration/global_configurations.md | 31 +- docs/en_US/configuration/secrets.md | 24 + docs/en_US/extension/wasm/overview.md | 36 + docs/en_US/guide/rules/error_handling_dlq.md | 23 + docs/en_US/guide/rules/graph_rule.md | 2 + docs/en_US/guide/rules/rule_pipeline.md | 25 + docs/en_US/guide/ui/web_console_user_guide.md | 20 +- docs/en_US/index.md | 4 +- docs/en_US/operation/declarative_gitops.md | 27 + docs/en_US/operation/fleet_management.md | 104 +- docs/en_US/operation/observability_ui_top.md | 23 + docs/en_US/operation/overview.md | 26 + .../public/diagrams/mcp_architecture.svg | 2 +- 35 files changed, 3507 insertions(+), 2825 deletions(-) create mode 100644 crates/rekuiper-server/src/engine/sources.rs delete mode 100644 crates/rule-engine/src/bin/kuiperd.rs delete mode 100644 crates/rule-engine/src/bin/rekuiperd.rs diff --git a/README.md b/README.md index 0c38794a..724b9ea7 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,8 @@ All tests run in a container limited to one CPU core and 1 GiB of RAM. | Measure | Result | Tested on | | :--- | :--- | :--- | -| End-to-end streaming SQL throughput | 1,085,885 events per second (460 ms for 500k records) | 1.0.0-rc4 | -| Zero-parse in-process SQL pipeline | 2,207,367 events per second (226 ms for 500k records) | 1.0.0-rc4 | +| End-to-end streaming SQL throughput | 1,141,417 events per second (438 ms for 500k records) | 1.0.0-rc4 | +| Zero-parse in-process SQL pipeline | 2,464,711 events per second (203 ms for 500k records) | 1.0.0-rc4 | | Parallel rules on one stream | 1,000 rules, 500,000 evaluations per second | 0.505 | | MQTT rule throughput, single rule | 150,000 to 200,000 messages per second | 0.500 | | Heap memory under load | 4.4 to 10.2 MiB | 0.500 | diff --git a/crates/rekuiper-core/src/manager.rs b/crates/rekuiper-core/src/manager.rs index 931bb315..bfaf8bf6 100644 --- a/crates/rekuiper-core/src/manager.rs +++ b/crates/rekuiper-core/src/manager.rs @@ -117,7 +117,7 @@ pub type TableRow = HashMap; #[derive(Clone, Default)] pub struct TableManager { tables: Arc>>, - rows: Arc>>>, + rows: Arc>>>, kv: Option>, op_lock: Arc>, } @@ -179,16 +179,20 @@ impl TableManager { } } } - rows.push(row); + rows.push_back(row); let max_size = retain_size.unwrap_or(if key_col.is_some() { usize::MAX } else { 100 }); while rows.len() > max_size { - rows.remove(0); + rows.pop_front(); } } /// Returns all lookup rows stored for a table (empty when none). pub fn get_table_rows(&self, table: &str) -> Vec> { - self.rows.read().get(table).cloned().unwrap_or_default() + self.rows + .read() + .get(table) + .map(|r| r.iter().cloned().collect()) + .unwrap_or_default() } pub async fn create_table(&self, def: TableDefinition) -> Result<()> { diff --git a/crates/rekuiper-core/src/wasm.rs b/crates/rekuiper-core/src/wasm.rs index f832efc9..e8a05087 100644 --- a/crates/rekuiper-core/src/wasm.rs +++ b/crates/rekuiper-core/src/wasm.rs @@ -3,7 +3,10 @@ use parking_lot::{Mutex, RwLock}; use serde_json::Value; use std::collections::HashMap; use std::sync::Arc; -use wasmi::{Engine, Instance, Linker, Module, Store, Val}; +use wasmi::{Config, Engine, Instance, Linker, Module, Store, Val}; + +/// Default instruction fuel budget per WASM UDF invocation. +const DEFAULT_WASM_FUEL_BUDGET: u64 = 1_000_000; struct WasmRuntimeInstance { store: Store<()>, @@ -14,104 +17,137 @@ struct WasmRuntimeInstance { /// Compiled WebAssembly module ready for isolated, sandboxed execution. #[derive(Clone)] pub struct WasmModule { + engine: Engine, module: Module, - runtime: Arc>, + runtime_pool: Arc>>, } impl WasmModule { - /// Compiles and pre-instantiates a WebAssembly module from raw binary bytecode. - pub fn from_bytes(wasm_bytes: &[u8]) -> Result { - let engine = Engine::default(); - let module = Module::new(&engine, wasm_bytes) - .context("Failed to parse and validate WebAssembly module")?; - let mut store = Store::new(&engine, ()); - let linker = Linker::new(&engine); + fn instantiate_runtime(engine: &Engine, module: &Module) -> Result { + let mut store = Store::new(engine, ()); + store + .set_fuel(DEFAULT_WASM_FUEL_BUDGET) + .context("Failed to initialize WASM fuel budget")?; + let linker = Linker::new(engine); let instance = linker - .instantiate(&mut store, &module) + .instantiate(&mut store, module) .context("Failed to instantiate WASM module")? .start(&mut store) .context("Failed to start WASM instance")?; + Ok(WasmRuntimeInstance { + store, + instance, + params_buf: Vec::with_capacity(8), + }) + } + + /// Compiles and pre-instantiates a WebAssembly module from raw binary bytecode. + pub fn from_bytes(wasm_bytes: &[u8]) -> Result { + let mut config = Config::default(); + config.consume_fuel(true); + let engine = Engine::new(&config); + let module = Module::new(&engine, wasm_bytes) + .context("Failed to parse and validate WebAssembly module")?; + let initial_instance = Self::instantiate_runtime(&engine, &module)?; Ok(Self { + engine, module, - runtime: Arc::new(Mutex::new(WasmRuntimeInstance { - store, - instance, - params_buf: Vec::with_capacity(8), - })), + runtime_pool: Arc::new(Mutex::new(vec![initial_instance])), }) } - /// Invokes an exported scalar function by name reusing the pre-instantiated instance. + /// Invokes an exported scalar function by name using a pooled instance and bounded fuel. pub fn call_function(&self, func_name: &str, args: &[Value]) -> Result { - let mut rt = self.runtime.lock(); - let WasmRuntimeInstance { - store, - instance, - params_buf, - } = &mut *rt; - - let func = instance.get_func(&*store, func_name).ok_or_else(|| { - anyhow::anyhow!("Exported function '{}' not found in WASM module", func_name) - })?; - - let func_type = func.ty(&*store); - let param_types = func_type.params(); - - if args.len() != param_types.len() { - bail!( - "Function '{}' expects {} arguments, but received {}", - func_name, - param_types.len(), - args.len() - ); + let mut rt = { + let mut pool = self.runtime_pool.lock(); + pool.pop() } + .map(Ok) + .unwrap_or_else(|| Self::instantiate_runtime(&self.engine, &self.module))?; - params_buf.clear(); - for (idx, (arg, p_type)) in args.iter().zip(param_types).enumerate() { - match p_type { - wasmi::core::ValType::I32 => { - let i = arg - .as_i64() - .or_else(|| arg.as_f64().map(|f| f as i64)) - .unwrap_or(0) as i32; - params_buf.push(Val::I32(i)); - } - wasmi::core::ValType::I64 => { - let i = arg - .as_i64() - .or_else(|| arg.as_f64().map(|f| f as i64)) - .unwrap_or(0); - params_buf.push(Val::I64(i)); - } - wasmi::core::ValType::F32 => { - let f = arg.as_f64().unwrap_or(0.0) as f32; - params_buf.push(Val::F32(f.into())); - } - wasmi::core::ValType::F64 => { - let f = arg.as_f64().unwrap_or(0.0); - params_buf.push(Val::F64(f.into())); + let exec_res = (|| -> Result { + let WasmRuntimeInstance { + store, + instance, + params_buf, + } = &mut rt; + + store + .set_fuel(DEFAULT_WASM_FUEL_BUDGET) + .context("Failed to replenish WASM execution fuel")?; + + let func = instance.get_func(&*store, func_name).ok_or_else(|| { + anyhow::anyhow!("Exported function '{}' not found in WASM module", func_name) + })?; + + let func_type = func.ty(&*store); + let param_types = func_type.params(); + + if args.len() != param_types.len() { + bail!( + "Function '{}' expects {} arguments, but received {}", + func_name, + param_types.len(), + args.len() + ); + } + + params_buf.clear(); + for (idx, (arg, p_type)) in args.iter().zip(param_types).enumerate() { + match p_type { + wasmi::core::ValType::I32 => { + let i = arg + .as_i64() + .or_else(|| arg.as_f64().map(|f| f as i64)) + .unwrap_or(0) as i32; + params_buf.push(Val::I32(i)); + } + wasmi::core::ValType::I64 => { + let i = arg + .as_i64() + .or_else(|| arg.as_f64().map(|f| f as i64)) + .unwrap_or(0); + params_buf.push(Val::I64(i)); + } + wasmi::core::ValType::F32 => { + let f = arg.as_f64().unwrap_or(0.0) as f32; + params_buf.push(Val::F32(f.into())); + } + wasmi::core::ValType::F64 => { + let f = arg.as_f64().unwrap_or(0.0); + params_buf.push(Val::F64(f.into())); + } + _ => bail!("Unsupported WASM parameter type for argument {}", idx), } - _ => bail!("Unsupported WASM parameter type for argument {}", idx), } - } - let mut results = vec![Val::I32(0); func_type.results().len()]; - func.call(store, params_buf, &mut results) - .context("WASM execution trapped or encountered runtime error")?; + let mut results = vec![Val::I32(0); func_type.results().len()]; + func.call(store, params_buf, &mut results) + .context("WASM execution trapped or encountered runtime error")?; - if results.is_empty() { - return Ok(Value::Null); - } + if results.is_empty() { + return Ok(Value::Null); + } - let res_val = match results[0] { - Val::I32(i) => serde_json::json!(i), - Val::I64(i) => serde_json::json!(i), - Val::F32(f) => serde_json::json!(f.to_float()), - Val::F64(f) => serde_json::json!(f.to_float()), - _ => Value::Null, - }; + let res_val = match results[0] { + Val::I32(i) => serde_json::json!(i), + Val::I64(i) => serde_json::json!(i), + Val::F32(f) => serde_json::json!(f.to_float()), + Val::F64(f) => serde_json::json!(f.to_float()), + _ => Value::Null, + }; + + Ok(res_val) + })(); + + if exec_res.is_ok() { + let mut pool = self.runtime_pool.lock(); + if pool.len() < 16 { + pool.push(rt); + } + } - Ok(res_val) + exec_res } /// List all exported functions in this WASM module. @@ -272,4 +308,27 @@ mod tests { .expect("registry call"); assert_eq!(call_res, serde_json::json!(300)); } + + #[test] + fn test_wasm_infinite_loop_fuel_exhaustion() { + // Minimal WASM module exporting `spin() -> i32` containing `loop { br 0 }` + let wasm_spin = vec![ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, // Magic + version + 0x01, 0x05, 0x01, 0x60, 0x00, 0x01, 0x7f, // Type 0: () -> i32 + 0x03, 0x02, 0x01, 0x00, // Function 0 uses Type 0 + 0x07, 0x08, 0x01, 0x04, 0x73, 0x70, 0x69, 0x6e, 0x00, 0x00, // Export "spin" + 0x0a, 0x09, 0x01, 0x07, 0x00, 0x03, 0x7f, 0x0c, 0x00, 0x0b, + 0x0b, // Code: loop (result i32) { br 0 } end + ]; + + let module = WasmModule::from_bytes(&wasm_spin).expect("WASM spin module compilation"); + let err = module + .call_function("spin", &[]) + .expect_err("Infinite loop must trap on fuel exhaustion"); + assert!( + err.to_string().contains("WASM execution trapped"), + "Unexpected error message: {}", + err + ); + } } diff --git a/crates/rekuiper-server/src/engine/mod.rs b/crates/rekuiper-server/src/engine/mod.rs index 10c635f0..a20195ae 100644 --- a/crates/rekuiper-server/src/engine/mod.rs +++ b/crates/rekuiper-server/src/engine/mod.rs @@ -4,6 +4,7 @@ pub mod joins; pub mod operators; pub mod runtime; pub mod sinks; +pub mod sources; pub mod windows; pub use dag::*; @@ -12,1748 +13,22 @@ pub(crate) use joins::*; pub use operators::*; pub(crate) use runtime::*; pub(crate) use sinks::*; +pub use sources::restore_running_rules; +pub(crate) use sources::*; pub(crate) use windows::*; -use crate::handlers::{stmt_called_functions, stmt_referenced_fields}; use crate::sink_cache::{self, SinkCache}; -use crate::state::{maybe_trace_record, AppState, HttpPushEndpoint, TraceManager}; +use crate::state::{maybe_trace_record, TraceManager}; use parking_lot::RwLock; use rekuiper_conf::KuiperConfig; -use rekuiper_connectors::codec::extract_required_fields_from_expr_names; -use rekuiper_connectors::{ - FileSource, FileSourceConfig, HttpPullConfig, HttpPullSource, JsonFieldMask, KafkaConfig, - KafkaSource, ModbusSource, MqttConfig, MqttSource, OpcUaSource, PayloadFormat, RabbitMqConfig, - RabbitMqSource, RedisSubSource, SimulatorConfig, SimulatorSource, SqlConnectorConfig, - SqlSource, VideoSource, VideoSourceConfig, WebSocketConfig, WebSocketSource, ZmqSource, - ZmqSourceConfig, -}; -use rekuiper_core::model::{RuleStatus, StreamField, StreamRecord}; -use rekuiper_core::{ - RuleCounters, RuleManager, SchemaManager, StreamBus, StreamManager, StreamReceiver, - TableDefinition, TableManager, -}; -use rekuiper_sql::{Parser, SelectStmt, WindowDef}; -use serde_json::Value; -use std::collections::HashMap; -use std::sync::Arc; - -/// Starts background source producers (HTTP pull, WebSocket, RedisSub, Kafka, -/// simulator) for a rule based on its source stream type. Shared by rule -/// creation and daemon-bootstrap restore. -/// Resolve the MQTT source configuration for a rule's stream. -/// -/// MQTT is the default streaming source: streams with no `TYPE`, an empty -/// `TYPE`, or `TYPE="mqtt"` ingest from the broker. Any other non-empty -/// `TYPE` (recognized sources like `kafka`, or anything else) resolves to -/// `None` here so exactly one source bootstrap owns the stream. -pub(crate) fn resolve_mqtt_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - schemas: &SchemaManager, - stream_name: &str, - rule_id: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - if let Some(kind) = def.options.get("TYPE") { - if !kind.trim().is_empty() && !kind.eq_ignore_ascii_case("mqtt") { - return None; - } - } - let mut config = MqttConfig::default(); - // CONF_KEY lookup is case-insensitive (`CONF_KEY`, `conf_key`, - // `confKey`): SQL definitions and imported/JSON definitions disagree on - // case. Stored configs typically carry only connection parameters, so a - // failed full decode still salvages server/credentials field by field. - let conf_key = def - .options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("CONF_KEY") - || k.eq_ignore_ascii_case("confKey") - || k.eq_ignore_ascii_case("connectionSelector") - }) - .map(|(_, v)| v.trim()); - if let Some(key) = conf_key { - if !key.is_empty() { - let lookup1 = format!("mqtt/{}", key); - let configs_guard = source_configs.read(); - let conf_val = configs_guard - .get(&lookup1) - .or_else(|| configs_guard.get(key)) - .or_else(|| configs_guard.get(&format!("connections/{}", key))) - .cloned(); - drop(configs_guard); - if let Some(val) = conf_val { - if let Ok(stored) = serde_json::from_value::(val.clone()) { - config = stored; - } else if let Some(srv) = val.get("server").and_then(|v| v.as_str()) { - if !srv.trim().is_empty() { - config.server = srv.to_string(); - } - if let Some(u) = val.get("username").and_then(|v| v.as_str()) { - config.username = Some(u.to_string()); - } - if let Some(p) = val.get("password").and_then(|v| v.as_str()) { - config.password = Some(p.to_string()); - } - } else { - tracing::warn!( - "[RULE {}] invalid mqtt config '{}': no server field", - rule_id, - lookup1, - ); - } - } - } - } - // Direct SERVER option overrides the configuration key (case-insensitive). - let server_opt = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("SERVER")) - .map(|(_, v)| v.trim()); - if let Some(srv) = server_opt { - if !srv.is_empty() { - config.server = srv.to_string(); - } - } - // DATASOURCE / topic (case-insensitive). - let topic_opt = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("DATASOURCE") || k.eq_ignore_ascii_case("topic")) - .map(|(_, v)| v.trim()); - if let Some(top) = topic_opt { - if !top.is_empty() { - config.topic = top.to_string(); - } - } - if config.topic.trim().is_empty() { - config.topic = stream_name.to_string(); - } - let client_id = def - .options - .get("CLIENTID") - .or_else(|| def.options.get("CLIENT_ID")); - if let Some(id) = client_id { - if !id.trim().is_empty() { - config.client_id = Some(id.clone()); - } - } - if let Some(user) = def.options.get("USERNAME") { - if !user.is_empty() { - config.username = Some(user.clone()); - } - } - if let Some(pass) = def.options.get("PASSWORD") { - if !pass.is_empty() { - config.password = Some(pass.clone()); - } - } - if let Some(qos) = def.options.get("QOS") { - match qos.trim().parse::() { - Ok(q) => config.qos = q, - Err(_) => { - tracing::warn!( - "[RULE {}] invalid mqtt QOS '{}', keeping {}", - rule_id, - qos, - config.qos - ); - } - } - } - config.format = resolve_payload_format(schemas, &def, rule_id)?; - Some(config) -} - -/// Resolve the MQTT configuration and EdgeX payload codec for a stream declaring -/// `TYPE="edgex"`. Default server is `tcp://edgex-mqtt-broker:1883` and default topic -/// is `edgex/rules-events`, configured via `etc/sources/edgex.yaml` or `EDGEX__*` overlays. -pub(crate) fn resolve_edgex_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - _schemas: &SchemaManager, - stream_name: &str, - rule_id: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - let kind = def.options.get("TYPE")?; - if !kind.eq_ignore_ascii_case("edgex") { - return None; - } - - let mut config = MqttConfig { - server: "tcp://edgex-mqtt-broker:1883".to_string(), - topic: "edgex/rules-events".to_string(), - format: PayloadFormat::EdgeX, - ..Default::default() - }; - - let conf_key = def - .options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("CONF_KEY") - || k.eq_ignore_ascii_case("confKey") - || k.eq_ignore_ascii_case("connectionSelector") - }) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - .unwrap_or("default"); - - let lookup = format!("edgex/{}", conf_key); - let configs_guard = source_configs.read(); - let conf_val = configs_guard - .get(&lookup) - .or_else(|| configs_guard.get(&lookup.to_ascii_lowercase())) - .or_else(|| configs_guard.get("edgex/default")) - .or_else(|| configs_guard.get(conf_key)) - .cloned(); - drop(configs_guard); - - if let Some(val) = conf_val { - let srv = val - .get("server") - .and_then(|v| v.as_str()) - .unwrap_or("edgex-mqtt-broker"); - let port = val - .get("port") - .and_then(|v| { - if let Some(n) = v.as_u64() { - Some(n) - } else if let Some(s) = v.as_str() { - s.parse::().ok() - } else { - None - } - }) - .unwrap_or(1883); - let proto = val - .get("protocol") - .and_then(|v| v.as_str()) - .unwrap_or("tcp"); - - if srv.contains("://") { - config.server = srv.to_string(); - } else { - config.server = format!("{}://{}:{}", proto, srv, port); - } - - if let Some(t) = val.get("topic").and_then(|v| v.as_str()) { - if !t.is_empty() { - config.topic = t.to_string(); - } - } - if let Some(opt) = val.get("optional") { - if let Some(u) = opt - .get("Username") - .or_else(|| opt.get("username")) - .and_then(|v| v.as_str()) - { - config.username = Some(u.to_string()); - } - if let Some(p) = opt - .get("Password") - .or_else(|| opt.get("password")) - .and_then(|v| v.as_str()) - { - config.password = Some(p.to_string()); - } - if let Some(cid) = opt - .get("ClientId") - .or_else(|| opt.get("clientId")) - .and_then(|v| v.as_str()) - { - config.client_id = Some(cid.to_string()); - } - } - } - - // Direct stream options take precedence - if let Some(srv) = def.options.get("SERVER").filter(|s| !s.trim().is_empty()) { - if srv.contains("://") { - config.server = srv.to_string(); - } else { - config.server = format!("tcp://{}", srv); - } - } - if let Some(ds) = def - .options - .get("DATASOURCE") - .filter(|s| !s.trim().is_empty()) - { - config.topic = ds.trim().to_string(); - } - - tracing::info!( - "[RULE {}] resolved edgex source for '{}': broker '{}', topic '{}'", - rule_id, - stream_name, - config.server, - config.topic - ); - Some(config) -} - -/// Payload decoding for a message source from the stream `FORMAT` (eKuiper -/// names: json, binary, delimited, protobuf with `SCHEMAID`). A protobuf -/// stream whose schema cannot be resolved yields `None` so the stream fails -/// loudly instead of decoding garbage; unknown formats fall back to JSON. -pub(crate) fn resolve_payload_format( - schemas: &SchemaManager, - def: &rekuiper_core::model::StreamDefinition, - rule_id: &str, -) -> Option { - resolve_payload_format_options( - schemas, - &def.options, - &def.stream_fields, - &def.name, - rule_id, - ) -} - -pub(crate) fn resolve_payload_format_options( - schemas: &SchemaManager, - options: &HashMap, - stream_fields: &[StreamField], - source_name: &str, - rule_id: &str, -) -> Option { - use rekuiper_connectors::{DelimitedCodec, PayloadFormat}; - let opt = |name: &str| { - options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case(name)) - .map(|(_, v)| v.trim()) - }; - let format = opt("FORMAT").unwrap_or("").to_ascii_lowercase(); - match format.as_str() { - "" | "json" => Some(PayloadFormat::Json), - "binary" => Some(PayloadFormat::Binary), - "delimited" => { - let delimiter = opt("DELIMITER") - .filter(|d| !d.is_empty()) - .map(DelimitedCodec::delimiter_from_name) - .unwrap_or(','); - let headers = stream_fields.iter().map(|f| f.name.clone()).collect(); - Some(PayloadFormat::Delimited(DelimitedCodec::new( - delimiter, headers, - ))) - } - "protobuf" => match resolve_proto_message(schemas, opt("SCHEMAID").unwrap_or("")) { - Ok(message) => Some(PayloadFormat::Protobuf(Arc::new(message))), - Err(e) => { - tracing::warn!("[RULE {}] source '{}': {}", rule_id, source_name, e); - None - } - }, - other => { - tracing::warn!( - "[RULE {}] source '{}': FORMAT '{}' is not supported for this source; decoding as JSON", - rule_id, - source_name, - other - ); - Some(PayloadFormat::Json) - } - } -} - -/// Resolve `SCHEMAID` (`.`) against registered protobuf -/// schemas (inline content, else the schema file). -pub(crate) fn resolve_proto_message( - schemas: &SchemaManager, - schema_id: &str, -) -> Result { - let (schema, message) = schema_id.split_once('.').ok_or_else(|| { - format!( - "protobuf SCHEMAID '{}' must be .", - schema_id - ) - })?; - let def = schemas - .get_schema("protobuf", schema) - .ok_or_else(|| format!("protobuf schema '{}' is not registered", schema))?; - let text = match (&def.content, &def.file) { - (Some(content), _) if !content.trim().is_empty() => content.clone(), - (_, Some(path)) => std::fs::read_to_string(path) - .map_err(|e| format!("cannot read protobuf schema file '{}': {}", path, e))?, - _ => return Err(format!("protobuf schema '{}' has no content", schema)), - }; - rekuiper_connectors::parse_proto(&text)? - .remove(message) - .ok_or_else(|| { - format!( - "message '{}' not found in protobuf schema '{}'", - message, schema - ) - }) -} - -pub(crate) fn compute_union_field_mask( - rules_map: &HashMap>>, -) -> Option { - if rules_map.is_empty() { - return None; - } - let mut union_set = std::collections::HashSet::new(); - for rule_fields_opt in rules_map.values() { - let fields = rule_fields_opt.as_ref()?; - union_set.extend(fields.iter().cloned()); - } - if union_set.is_empty() { - None - } else { - Some(JsonFieldMask::new(union_set)) - } -} - -pub(crate) fn update_stream_field_mask( - state: &AppState, - stream_name: &str, - rule_id: &str, - field_mask: Option, -) -> Arc>> { - let rule_fields = field_mask.map(|m| (*m.fields).clone()); - let combined = { - let mut map = state.stream_rule_field_masks.write(); - let rules_map = map.entry(stream_name.to_string()).or_default(); - rules_map.insert(rule_id.to_string(), rule_fields); - compute_union_field_mask(rules_map) - }; - let mut masks_guard = state.stream_field_masks.write(); - if let Some(flag) = masks_guard.get(stream_name) { - *flag.write() = combined; - flag.clone() - } else { - let flag = Arc::new(parking_lot::RwLock::new(combined)); - masks_guard.insert(stream_name.to_string(), flag.clone()); - flag - } -} - -pub(crate) fn cancel_stream_sources(state: &AppState, stream_name: &str) { - if let Some(cancels) = state.stream_source_cancels.write().remove(stream_name) { - for tx in cancels { - let _ = tx.send(true); - } - } - state.stream_active_rules.write().remove(stream_name); - state.stream_attach_meta.write().remove(stream_name); - state.stream_field_masks.write().remove(stream_name); - state.stream_rule_field_masks.write().remove(stream_name); - state - .http_push_endpoints - .write() - .retain(|_, ep| ep.stream_name != stream_name); -} - -pub(crate) fn cancel_table_source(state: &AppState, table_name: &str) { - let table_key = format!("$table/{}", table_name); - if let Some(cancels) = state.stream_source_cancels.write().remove(&table_key) { - for tx in cancels { - let _ = tx.send(true); - } - } -} - -pub(crate) fn resolve_mqtt_table_source( - table_def: &TableDefinition, - source_configs: &Arc>>, - schemas: &SchemaManager, -) -> Option { - if let Some(kind) = table_def.options.get("TYPE") { - if !kind.trim().is_empty() && !kind.eq_ignore_ascii_case("mqtt") { - return None; - } - } - let mut config = MqttConfig::default(); - let conf_key = table_def - .options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("CONF_KEY") - || k.eq_ignore_ascii_case("confKey") - || k.eq_ignore_ascii_case("connectionSelector") - }) - .map(|(_, v)| v.trim()); - if let Some(key) = conf_key { - if !key.is_empty() { - let lookup1 = format!("mqtt/{}", key); - let configs_guard = source_configs.read(); - let conf_val = configs_guard - .get(&lookup1) - .or_else(|| configs_guard.get(key)) - .or_else(|| configs_guard.get(&format!("connections/{}", key))) - .cloned(); - drop(configs_guard); - if let Some(val) = conf_val { - if let Ok(stored) = serde_json::from_value::(val.clone()) { - config = stored; - } else if let Some(srv) = val.get("server").and_then(|v| v.as_str()) { - if !srv.trim().is_empty() { - config.server = srv.to_string(); - } - if let Some(u) = val.get("username").and_then(|v| v.as_str()) { - config.username = Some(u.to_string()); - } - if let Some(p) = val.get("password").and_then(|v| v.as_str()) { - config.password = Some(p.to_string()); - } - } - } - } - } - if let Some((_, srv)) = table_def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("SERVER")) - { - let srv = srv.trim(); - if !srv.is_empty() { - config.server = srv.to_string(); - } - } - if let Some((_, top)) = table_def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("DATASOURCE") || k.eq_ignore_ascii_case("topic")) - { - let top = top.trim(); - if !top.is_empty() { - config.topic = top.to_string(); - } - } - if config.topic.trim().is_empty() { - config.topic = table_def.name.clone(); - } - if let Some(id) = table_def - .options - .get("CLIENTID") - .or_else(|| table_def.options.get("CLIENT_ID")) - { - if !id.trim().is_empty() { - config.client_id = Some(id.clone()); - } - } - if let Some(user) = table_def.options.get("USERNAME") { - if !user.is_empty() { - config.username = Some(user.clone()); - } - } - if let Some(pass) = table_def.options.get("PASSWORD") { - if !pass.is_empty() { - config.password = Some(pass.clone()); - } - } - if let Some(qos) = table_def.options.get("QOS") { - if let Ok(q) = qos.trim().parse::() { - config.qos = q; - } - } - config.format = resolve_payload_format_options( - schemas, - &table_def.options, - &table_def.stream_fields, - &table_def.name, - &table_def.name, - )?; - Some(config) -} - -pub(crate) fn bootstrap_table_source(state: &AppState, table_name: &str) { - let table_key = format!("$table/{}", table_name); - if state.stream_source_cancels.read().contains_key(&table_key) { - return; - } - let Some(def) = state.table_manager.get_table(table_name) else { - return; - }; - let datasource = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("DATASOURCE") || k.eq_ignore_ascii_case("TOPIC")) - .map(|(_, v)| v.trim().to_string()) - .unwrap_or_else(|| table_name.to_string()); - let table_type = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("TYPE")) - .map(|(_, v)| v.trim().to_string()) - .unwrap_or_default(); - - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - state - .stream_source_cancels - .write() - .insert(table_key.clone(), vec![cancel_tx]); - - let t_name = table_name.to_string(); - let table_mgr = state.table_manager.clone(); - - // 1. Listen on the stream bus for this table's datasource topic (handles memory sinks and internal publication) - let mut bus_rx = state.stream_bus.subscribe(&datasource); - let mut table_bus_rx = if datasource != table_name { - Some(state.stream_bus.subscribe(table_name)) - } else { - None - }; - let mut bus_cancel_rx = cancel_rx.clone(); - let t_name_bus = t_name.clone(); - let table_mgr_bus = table_mgr.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - record = bus_rx.recv() => { - match record { - Some(rec) => table_mgr_bus.insert_table_row(&t_name_bus, rec.data.to_hashmap()), - None => break, - } - } - record = async { - if let Some(ref mut rx) = table_bus_rx { - rx.recv().await - } else { - std::future::pending().await - } - } => { - match record { - Some(rec) => table_mgr_bus.insert_table_row(&t_name_bus, rec.data.to_hashmap()), - None => break, - } - } - changed = bus_cancel_rx.changed() => { - if changed.is_err() || *bus_cancel_rx.borrow() { - break; - } - } - } - } - }); - - // 2. If table is MQTT source (TYPE="mqtt" or default when not memory/file/redis/sql): - let is_memory = table_type.eq_ignore_ascii_case("memory"); - let is_redis = table_type.eq_ignore_ascii_case("redis"); - let is_sql = table_type.eq_ignore_ascii_case("sql"); - let is_file = table_type.eq_ignore_ascii_case("file"); - if !is_memory && !is_redis && !is_sql && !is_file { - if let Some(config) = - resolve_mqtt_table_source(&def, &state.source_configs, &state.schema_manager) - { - let bus_topic = format!("$table_mqtt/{}", table_name); - let stream_tx = state.stream_bus.get_or_create(&bus_topic); - let mut mqtt_rx = state.stream_bus.subscribe(&bus_topic); - let mut mqtt_cancel = cancel_rx.clone(); - let t_name_mqtt = t_name.clone(); - let table_mgr_mqtt = table_mgr.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - record = mqtt_rx.recv() => { - match record { - Some(rec) => table_mgr_mqtt.insert_table_row(&t_name_mqtt, rec.data.to_hashmap()), - None => break, - } - } - changed = mqtt_cancel.changed() => { - if changed.is_err() || *mqtt_cancel.borrow() { - break; - } - } - } - } - }); - MqttSource::new(config, stream_tx).spawn(cancel_rx.clone()); - } - } -} - -pub(crate) fn resolve_httppush_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - let is_push = def - .options - .get("TYPE") - .is_some_and(|t| t.eq_ignore_ascii_case("httppush") || t.eq_ignore_ascii_case("http_push")); - if !is_push { - return None; - } - - let raw_path = def - .options - .get("DATASOURCE") - .or_else(|| def.options.get("ENDPOINT")) - .or_else(|| def.options.get("PATH")) - .map(|s| s.trim()) - .unwrap_or(""); - let path = if raw_path.is_empty() { - format!("/{}", stream_name) - } else { - format!("/{}", raw_path.trim_start_matches('/')) - }; - - let mut method = "POST".to_string(); - if let Some(m) = def.options.get("METHOD") { - if !m.trim().is_empty() { - method = m.trim().to_ascii_uppercase(); - } - } else if let Some(key) = def.options.get("CONF_KEY").map(|k| k.trim()) { - if !key.is_empty() { - let lookup = format!("httppush/{}", key); - let guard = source_configs.read(); - if let Some(val) = guard.get(&lookup).or_else(|| guard.get(key)) { - if let Some(m) = val.get("method").and_then(|v| v.as_str()) { - method = m.trim().to_ascii_uppercase(); - } - } - } - } - - let format = def - .options - .get("FORMAT") - .map(|f| f.trim().to_ascii_lowercase()) - .unwrap_or_else(|| "json".to_string()); - - Some(HttpPushEndpoint { - stream_name: stream_name.to_string(), - path, - method, - format, - }) -} - -pub(crate) fn register_rule_stream_source( - state: &AppState, - stream_name: &str, - rule_id: &str, - cancel_tx: tokio::sync::watch::Sender, -) { - state - .stream_source_cancels - .write() - .entry(stream_name.to_string()) - .or_default() - .push(cancel_tx.clone()); - state - .stream_active_rules - .write() - .entry(stream_name.to_string()) - .or_default() - .insert(rule_id.to_string()); - state - .rule_streams - .write() - .entry(rule_id.to_string()) - .or_default() - .insert(stream_name.to_string()); - let (rule_cancel_tx, _rule_cancel_rx) = tokio::sync::watch::channel(false); - state - .source_cancels - .write() - .entry(rule_id.to_string()) - .or_default() - .push(rule_cancel_tx); -} - -/// Start source producers for one stream (MQTT/file/HTTP-pull/WebSocket/ -/// RedisSub/Kafka/SQL/simulator, whichever its TYPE declares). `needs_meta` -/// asks message sources to attach per-message metadata for `meta()`. -/// Multiple rules reading from the same stream share a single underlying -/// source producer, preventing duplicate message ingestion. -pub(crate) fn bootstrap_stream_sources( - state: &AppState, - rule_id: &str, - stream_name: &str, - needs_meta: bool, - field_mask: Option, -) { - // If this rule is already registered for this stream, update its mask and meta flag. - if let Some(rules) = state.stream_active_rules.read().get(stream_name) { - if rules.contains(rule_id) { - if needs_meta { - if let Some(flag) = state.stream_attach_meta.read().get(stream_name) { - flag.store(true, std::sync::atomic::Ordering::Relaxed); - } - } - update_stream_field_mask(state, stream_name, rule_id, field_mask); - return; - } - } - - // If a source task is already running for this stream, register this rule as - // another consumer and union its required fields into the stream's JsonFieldMask. - if state.stream_source_cancels.read().contains_key(stream_name) { - state - .stream_active_rules - .write() - .entry(stream_name.to_string()) - .or_default() - .insert(rule_id.to_string()); - state - .rule_streams - .write() - .entry(rule_id.to_string()) - .or_default() - .insert(stream_name.to_string()); - if needs_meta { - if let Some(flag) = state.stream_attach_meta.read().get(stream_name) { - flag.store(true, std::sync::atomic::Ordering::Relaxed); - } - } - update_stream_field_mask(state, stream_name, rule_id, field_mask); - let (rule_cancel_tx, _rule_cancel_rx) = tokio::sync::watch::channel(false); - state - .source_cancels - .write() - .entry(rule_id.to_string()) - .or_default() - .push(rule_cancel_tx); - return; - } - - let mask_flag = update_stream_field_mask(state, stream_name, rule_id, field_mask); - let comms = state.comms_hub.clone(); - let status_cb: rekuiper_connectors::ConnectorStatusCallback = - Arc::new(move |id, kind, status, err| { - comms.publish_connector_status(id, kind, status, err.map(|s| s.to_string())); - }); - - // HTTP Push source streams listen on the configured HTTP data server endpoint. - if let Some(endpoint) = - resolve_httppush_source(&state.stream_manager, &state.source_configs, stream_name) - { - let (cancel_tx, mut cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - let path = endpoint.path.clone(); - state - .http_push_endpoints - .write() - .insert(path.clone(), endpoint); - - let state_clone = state.clone(); - let stream_name_clone = stream_name.to_string(); - tokio::spawn(async move { - let _ = cancel_rx.changed().await; - let mut guard = state_clone.http_push_endpoints.write(); - if let Some(ep) = guard.get(&path) { - if ep.stream_name == stream_name_clone { - guard.remove(&path); - } - } - }); - return; - } - - // MQTT is the default streaming source: typeless streams and TYPE="mqtt" - // subscribe to the broker topic and feed the rule pipeline. - if let Some(mut config) = resolve_mqtt_source( - &state.stream_manager, - &state.source_configs, - &state.schema_manager, - stream_name, - rule_id, - ) { - let meta_flag = Arc::new(std::sync::atomic::AtomicBool::new(needs_meta)); - config.attach_meta = needs_meta; - config.field_mask = mask_flag.read().clone(); - state - .stream_attach_meta - .write() - .insert(stream_name.to_string(), meta_flag.clone()); - - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - MqttSource::new(config, stream_tx) - .with_rule_counters(state.rule_manager.rule_counters(rule_id)) - .with_meta_flag(Some(meta_flag)) - .with_field_mask_flag(Some(mask_flag)) - .with_status_callback(Some(status_cb.clone())) - .spawn(cancel_rx); - return; - } - - // EdgeX source streams connect to EdgeX MQTT message bus and decode Event / Reading DTOs. - if let Some(mut config) = resolve_edgex_source( - &state.stream_manager, - &state.source_configs, - &state.schema_manager, - stream_name, - rule_id, - ) { - let meta_flag = Arc::new(std::sync::atomic::AtomicBool::new(needs_meta)); - config.attach_meta = needs_meta; - state - .stream_attach_meta - .write() - .insert(stream_name.to_string(), meta_flag.clone()); - - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - MqttSource::new(config, stream_tx) - .with_rule_counters(state.rule_manager.rule_counters(rule_id)) - .with_meta_flag(Some(meta_flag)) - .spawn(cancel_rx); - return; - } - - // File source streams tail a line-delimited file into the stream bus. - if let Some(config) = resolve_file_source( - &state.stream_manager, - &state.source_configs, - stream_name, - rule_id, - ) { - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - FileSource::new(config, stream_tx).spawn(cancel_rx); - return; - } - - // HTTP pull source streams poll a remote endpoint into the stream bus. - if let Some(conf) = resolve_httppull_config( - &state.stream_manager, - &state.source_configs, - stream_name, - rule_id, - ) { - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - HttpPullSource { - config: conf, - tx: stream_tx, - status_callback: Some(status_cb.clone()), - } - .spawn(cancel_rx); - return; - } - - // WebSocket source streams forward incoming messages into the stream bus. - if let Some(url) = resolve_websocket_url( - &state.stream_manager, - &state.source_configs, - stream_name, - rule_id, - ) { - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - WebSocketSource { url, tx: stream_tx }.spawn(cancel_rx); - return; - } - - // Redis subscription streams forward channel messages into the stream bus. - if let Some((url, channel)) = resolve_redissub_source( - &state.stream_manager, - &state.source_configs, - stream_name, - rule_id, - ) { - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - RedisSubSource { - url, - channel, - tx: stream_tx, - } - .spawn(cancel_rx); - return; - } - - // Kafka source streams consume a topic partition into the stream bus. - if let Some(config) = resolve_kafka_source( - &state.stream_manager, - &state.source_configs, - stream_name, - rule_id, - ) { - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - KafkaSource { - config, - tx: stream_tx, - status_callback: Some(status_cb.clone()), - } - .spawn(cancel_rx); - return; - } - - // SQL source streams poll a database table into the stream bus. - if let Some(config) = resolve_sql_source( - &state.stream_manager, - &state.source_configs, - stream_name, - rule_id, - ) { - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - SqlSource::new(config, stream_tx).spawn(cancel_rx); - return; - } - - // RabbitMQ source streams subscribe to an AMQP broker into the stream bus. - if let Some(config) = - resolve_rabbitmq_source(&state.stream_manager, &state.source_configs, stream_name) - { - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - RabbitMqSource::new(config, stream_tx).spawn(cancel_rx); - return; - } - - // ZeroMQ source streams subscribe to a ZeroMQ endpoint into the stream bus. - if let Some(config) = - resolve_zmq_source(&state.stream_manager, &state.source_configs, stream_name) - { - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - ZmqSource::new(config, stream_tx).spawn(cancel_rx); - return; - } - - // Video source streams extract camera frames via FFmpeg into the stream bus. - if let Some(config) = - resolve_video_source(&state.stream_manager, &state.source_configs, stream_name) - { - if let Err(e) = config.validate() { - tracing::error!( - "[RULE {}] video source validation failed for stream '{}': {}", - rule_id, - stream_name, - e - ); - if let Some(handle) = state.rule_manager.rule_status_handle(rule_id) { - let mut status = handle.write(); - status.status = "error".to_string(); - status.message = format!("Video source validation error: {}", e); - status.last_exception = e; - } - return; - } - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - VideoSource::new(config, stream_tx).spawn(cancel_rx); - return; - } - - // Modbus source streams poll industrial PLCs over TCP or RTU-over-TCP into the stream bus. - if let Some(config) = - resolve_modbus_source(&state.stream_manager, &state.source_configs, stream_name) - { - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - ModbusSource::new(config, stream_tx) - .with_status_callback(Some(status_cb.clone())) - .spawn(cancel_rx); - return; - } - - // OPC-UA source streams poll OPC-UA servers over opc.tcp into the stream bus. - if let Some(config) = - resolve_opcua_source(&state.stream_manager, &state.source_configs, stream_name) - { - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - OpcUaSource::new(config, stream_tx) - .with_status_callback(Some(status_cb)) - .spawn(cancel_rx); - return; - } - - // OTLP source streams receive or poll OpenTelemetry metrics/logs/traces into the stream bus. - if let Some(config) = - resolve_otlp_source(&state.stream_manager, &state.source_configs, stream_name) - { - let stream_tx = state.stream_bus.get_or_create(stream_name); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - rekuiper_connectors::OtlpSource::new(config, stream_tx).spawn(cancel_rx); - return; - } - - // Simulator / Random source streams replay configured data into the stream bus. - if let Some(conf) = resolve_simulator_source( - &state.stream_manager, - &state.source_configs, - stream_name, - rule_id, - ) { - if !conf.data.is_empty() { - let stream_name_str = stream_name.to_string(); - let bus = state.stream_bus.clone(); - let (cancel_tx, mut cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, stream_name, rule_id, cancel_tx); - tokio::spawn(async move { - let (tx, mut rx) = tokio::sync::mpsc::channel::(1024); - let sim_handle = - tokio::spawn(async move { SimulatorSource::new(conf).run(tx).await }); - loop { - tokio::select! { - record = rx.recv() => { - match record { - Some(record) => { - let _ = bus.publish_async(&stream_name_str, record).await; - } - None => break, - } - } - changed = cancel_rx.changed() => { - if changed.is_err() || *cancel_rx.borrow() { - sim_handle.abort(); - break; - } - } - } - } - let _ = sim_handle.await; - }); - } - } -} - -/// Start source producers for a rule: its FROM source plus every joined -/// source (streams fan in messages; table targets bootstrap their source listeners). -pub(crate) fn bootstrap_rule_sources(state: &AppState, rule_id: &str, select_stmt: &SelectStmt) { - let needs_meta = stmt_called_functions(select_stmt) - .iter() - .any(|f| f.eq_ignore_ascii_case("meta") || f.eq_ignore_ascii_case("mqtt")); - let (has_wildcard, idents) = stmt_referenced_fields(select_stmt); - let mask_for_stream = |stream_name: &str| { - let ts_field = state.stream_manager.get_stream(stream_name).and_then(|s| { - s.options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("TIMESTAMP")) - .map(|(_, v)| v.clone()) - }); - extract_required_fields_from_expr_names(has_wildcard, &idents, ts_field.as_deref()) - }; - if state.table_manager.get_table(&select_stmt.from).is_some() { - bootstrap_table_source(state, &select_stmt.from); - } else { - bootstrap_stream_sources( - state, - rule_id, - &select_stmt.from, - needs_meta, - mask_for_stream(&select_stmt.from), - ); - } - for join in &select_stmt.joins { - if state.table_manager.get_table(&join.target).is_some() { - bootstrap_table_source(state, &join.target); - } else if join.target != select_stmt.from { - bootstrap_stream_sources( - state, - rule_id, - &join.target, - needs_meta, - mask_for_stream(&join.target), - ); - } - } -} - -pub(crate) fn bootstrap_graph_sources( - state: &AppState, - rule_id: &str, - graph: &rekuiper_core::model::GraphDefinition, -) { - for node in graph.nodes.values() { - if node.node_category == "source" { - let stream_name = node - .props - .get("datasource") - .or_else(|| node.props.get("sourceName")) - .or_else(|| node.props.get("topic")) - .and_then(|v| v.as_str()); - let src = match stream_name { - Some(s) if !s.trim().is_empty() => s.trim(), - _ => continue, - }; - - if state.stream_manager.get_stream(src).is_some() { - bootstrap_stream_sources(state, rule_id, src, true, None); - } else { - let n_type = node.node_type.to_lowercase(); - let comms = state.comms_hub.clone(); - let status_cb: rekuiper_connectors::ConnectorStatusCallback = - Arc::new(move |id, kind, status, err| { - comms.publish_connector_status( - id, - kind, - status, - err.map(|s| s.to_string()), - ); - }); - let stream_tx = state.stream_bus.get_or_create(src); - let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); - register_rule_stream_source(state, src, rule_id, cancel_tx); - - if n_type == "mqtt" || n_type == "source" { - let mut config = MqttConfig::default(); - if let Some(srv) = node.props.get("server").and_then(|v| v.as_str()) { - config.server = srv.to_string(); - } - if let Some(topic) = node - .props - .get("topic") - .or_else(|| node.props.get("datasource")) - .and_then(|v| v.as_str()) - { - config.topic = topic.to_string(); - } - if let Some(u) = node.props.get("username").and_then(|v| v.as_str()) { - config.username = Some(u.to_string()); - } - if let Some(p) = node.props.get("password").and_then(|v| v.as_str()) { - config.password = Some(p.to_string()); - } - if let Some(qos) = node.props.get("qos").and_then(|v| v.as_u64()) { - config.qos = qos as u8; - } - config.attach_meta = true; - MqttSource::new(config, stream_tx) - .with_rule_counters(state.rule_manager.rule_counters(rule_id)) - .with_status_callback(Some(status_cb)) - .spawn(cancel_rx); - } else if n_type == "modbus" { - let conf_val = - serde_json::Value::Object(node.props.clone().into_iter().collect()); - if let Ok(config) = - serde_json::from_value::(conf_val) - { - ModbusSource::new(config, stream_tx) - .with_status_callback(Some(status_cb)) - .spawn(cancel_rx); - } - } else if n_type == "opcua" { - let conf_val = - serde_json::Value::Object(node.props.clone().into_iter().collect()); - if let Ok(config) = - serde_json::from_value::(conf_val) - { - OpcUaSource::new(config, stream_tx) - .with_status_callback(Some(status_cb)) - .spawn(cancel_rx); - } - } else { - bootstrap_stream_sources(state, rule_id, src, true, None); - } - } - } - } -} - -/// Respawns execution tasks (plus source producers) for every rule whose -/// persisted status is `running`, so a restarted daemon resumes processing -/// without manual intervention. -pub async fn restore_running_rules(state: &AppState) { - for table_name in state.table_manager.list_tables() { - bootstrap_table_source(state, &table_name); - } - for rule in state.rule_manager.list_rules() { - let running = state - .rule_manager - .get_rule_status(&rule.id) - .is_some_and(|s| s.status == "running"); - if !running { - continue; - } - if rule.sql.trim().is_empty() { - if let Some(ref graph) = rule.graph { - let is_native_dag = - graph.nodes.values().any(|n| { - matches!(n.node_type.as_str(), "switch" | "rate_limit" | "watchdog") - }) || graph - .topo - .edges - .values() - .any(|e| matches!(e, rekuiper_core::model::EdgeTargets::Multi(_))); - if is_native_dag { - let buffer_len = rule - .options - .as_ref() - .and_then(|o| o.get("bufferLength")) - .and_then(|v| v.as_u64()) - .map(|n| n as usize) - .unwrap_or(1024); - spawn_graph_rule_task( - &state.rule_manager, - &state.stream_bus, - rule.id.clone(), - graph.clone(), - state.comms_hub.clone(), - buffer_len, - ); - bootstrap_graph_sources(state, &rule.id, graph); - tracing::info!("Restored running graph rule {}", rule.id); - continue; - } - } - } - let mut parser = Parser::new(&rule.sql); - let select_stmt = match parser.parse_select() { - Ok(stmt) => stmt, - Err(e) => { - tracing::warn!("Skipping restore of rule {}: invalid SQL: {}", rule.id, e); - continue; - } - }; - spawn_rule_task( - &state.rule_manager, - &state.stream_bus, - &state.stream_manager, - &state.table_manager, - &state.source_configs, - &state.http_client, - &state.trace_manager, - &state.config, - rule.id.clone(), - select_stmt.clone(), - rule.actions.clone(), - rule.options.clone(), - ); - bootstrap_rule_sources(state, &rule.id, &select_stmt); - tracing::info!("Restored running rule {}", rule.id); - } -} - -/// Resolve the HTTP pull configuration for a rule whose source stream -/// declares `TYPE="httppull"` (or `"http_pull"`). -/// -/// Looks up `httppull/{conf_key}` then `http_pull/{conf_key}` in the stored -/// source configs; when no key matches, falls back to the stream's -/// `DATASOURCE` property as the poll URL with default method/interval. -pub(crate) fn resolve_httppull_config( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, - rule_id: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - let kind = def.options.get("TYPE")?; - if !(kind.eq_ignore_ascii_case("httppull") || kind.eq_ignore_ascii_case("http_pull")) { - return None; - } - if let Some(key) = def.options.get("CONF_KEY") { - if !key.is_empty() { - for prefix in ["httppull", "http_pull"] { - let lookup = format!("{}/{}", prefix, key); - if let Some(conf_val) = source_configs.read().get(&lookup).cloned() { - match serde_json::from_value::(conf_val) { - Ok(conf) => return Some(conf), - Err(e) => { - tracing::warn!( - "[RULE {}] invalid http pull config '{}': {}", - rule_id, - lookup, - e - ); - return None; - } - } - } - } - } - } - let url = def.options.get("DATASOURCE").cloned().unwrap_or_default(); - if url.is_empty() { - tracing::warn!( - "[RULE {}] http pull stream '{}' has neither CONF_KEY config nor DATASOURCE url", - rule_id, - stream_name - ); - return None; - } - Some(HttpPullConfig { - url, - method: "get".to_string(), - interval: 1000, - headers: HashMap::new(), - body: None, - }) -} - -/// Resolve the WebSocket URL for a rule whose source stream declares -/// `TYPE="websocket"`. -/// -/// Looks up `websocket/{conf_key}` in the stored source configs and builds -/// the URL from it; otherwise parses `DATASOURCE` (used directly when it -/// starts with `ws://` or `wss://`, else treated as a path on the default -/// local endpoint). -pub(crate) fn resolve_websocket_url( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, - rule_id: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - let kind = def.options.get("TYPE")?; - if !kind.eq_ignore_ascii_case("websocket") { - return None; - } - if let Some(key) = def.options.get("CONF_KEY") { - if !key.is_empty() { - let lookup = format!("websocket/{}", key); - if let Some(conf_val) = source_configs.read().get(&lookup).cloned() { - match serde_json::from_value::(conf_val) { - Ok(conf) => return Some(conf.target_url()), - Err(e) => { - tracing::warn!( - "[RULE {}] invalid websocket config '{}': {}", - rule_id, - lookup, - e - ); - return None; - } - } - } - } - } - let datasource = def.options.get("DATASOURCE").cloned().unwrap_or_default(); - if datasource.is_empty() { - tracing::warn!( - "[RULE {}] websocket stream '{}' has neither CONF_KEY config nor DATASOURCE", - rule_id, - stream_name - ); - return None; - } - let ds = datasource.trim(); - if ds.starts_with("ws://") || ds.starts_with("wss://") { - Some(ds.to_string()) - } else { - Some(format!( - "ws://127.0.0.1:8080/{}", - ds.trim_start_matches('/') - )) - } -} - -/// Resolve a Redis server address from a stored source config value, which -/// may be a full object (`{"addr": ...}`) or a bare address string. -/// Falls back to the local default when absent or unparseable. -pub(crate) fn resolve_redis_addr( - source_configs: &Arc>>, - conf_key: &str, -) -> String { - const DEFAULT: &str = "127.0.0.1:6379"; - if conf_key.is_empty() { - return DEFAULT.to_string(); - } - let stored = source_configs - .read() - .get(&format!("redis/{}", conf_key)) - .cloned(); - match stored { - Some(Value::Object(map)) => map - .get("addr") - .or_else(|| map.get("address")) - .or_else(|| map.get("url")) - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - .unwrap_or(DEFAULT) - .to_string(), - Some(Value::String(s)) if !s.is_empty() => s, - _ => DEFAULT.to_string(), - } -} - -pub(crate) fn resolve_redissub_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, - rule_id: &str, -) -> Option<(String, String)> { - let def = stream_manager.get_stream(stream_name)?; - let kind = def.options.get("TYPE")?; - if !(kind.eq_ignore_ascii_case("redissub") || kind.eq_ignore_ascii_case("redis_sub")) { - return None; - } - let channel = def.options.get("DATASOURCE").cloned().unwrap_or_default(); - if channel.is_empty() { - tracing::warn!( - "[RULE {}] redissub stream '{}' has no DATASOURCE channel", - rule_id, - stream_name - ); - return None; - } - let conf_key = def.options.get("CONF_KEY").cloned().unwrap_or_default(); - let addr = resolve_redis_addr(source_configs, &conf_key); - let url = if addr.contains("://") { - addr - } else { - format!("redis://{}", addr) - }; - Some((url, channel)) -} - -/// Resolve the [`KafkaConfig`] for a rule whose source stream declares -/// `TYPE="kafka"`: the topic comes from the stream `DATASOURCE` (falling back -/// to the config topic), brokers and friends from `kafka/{conf_key}` (or -/// defaults when no key matches). -pub(crate) fn resolve_kafka_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, - rule_id: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - let kind = def.options.get("TYPE")?; - if !kind.eq_ignore_ascii_case("kafka") { - return None; - } - let mut config = KafkaConfig { - brokers: "127.0.0.1:9092".to_string(), - topic: None, - group_id: None, - partition: 0, - key: None, - offset_reset: "latest".to_string(), - }; - if let Some(key) = def.options.get("CONF_KEY") { - if !key.is_empty() { - let lookup = format!("kafka/{}", key); - if let Some(conf_val) = source_configs.read().get(&lookup).cloned() { - match serde_json::from_value::(conf_val) { - Ok(parsed) => config = parsed, - Err(e) => { - tracing::warn!( - "[RULE {}] invalid kafka config '{}': {}", - rule_id, - lookup, - e - ); - return None; - } - } - } - } - } - if let Some(reset) = def - .options - .get("OFFSET_RESET") - .or_else(|| def.options.get("OFFSETRESET")) - { - if !reset.trim().is_empty() { - let r = reset.trim().to_ascii_lowercase(); - if r == "earliest" || r == "latest" { - config.offset_reset = r; - } else { - tracing::warn!( - "[RULE {}] invalid kafka offset_reset '{}' (must be 'earliest' or 'latest')", - rule_id, - reset - ); - return None; - } - } - } - if !config.offset_reset.eq_ignore_ascii_case("earliest") - && !config.offset_reset.eq_ignore_ascii_case("latest") - { - tracing::warn!( - "[RULE {}] invalid kafka offset_reset '{}' (must be 'earliest' or 'latest')", - rule_id, - config.offset_reset - ); - return None; - } - match def.options.get("DATASOURCE").cloned() { - Some(topic) if !topic.is_empty() => config.topic = Some(topic), - _ if config.topic.as_deref().is_some_and(|t| !t.is_empty()) => {} - _ => { - tracing::warn!( - "[RULE {}] kafka stream '{}' has no DATASOURCE topic", - rule_id, - stream_name - ); - return None; - } - } - if config.broker_list().is_empty() { - tracing::warn!( - "[RULE {}] kafka stream '{}' has no brokers configured", - rule_id, - stream_name - ); - return None; - } - Some(config) -} - -/// Resolve the [`FileSourceConfig`] for a rule whose source stream declares -/// `TYPE="file"`: the path comes from `DATASOURCE` (falling back to a stored -/// `file/{conf_key}` config), with `FORMAT`/`fileType`, `hasHeader` and -/// `delimiter` layered from stream options over config defaults. -pub(crate) fn resolve_file_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, - rule_id: &str, -) -> Option { - use rekuiper_connectors::DelimitedCodec; - - let def = stream_manager.get_stream(stream_name)?; - let kind = def.options.get("TYPE")?; - if !kind.eq_ignore_ascii_case("file") { - return None; - } - let mut config = FileSourceConfig { - path: String::new(), - format: "json".to_string(), - has_header: false, - delimiter: None, - interval: 0, - }; - if let Some(key) = def.options.get("CONF_KEY") { - if !key.is_empty() { - let lookup = format!("file/{}", key); - if let Some(conf_val) = source_configs.read().get(&lookup).cloned() { - match serde_json::from_value::(conf_val) { - Ok(parsed) => config = parsed, - Err(e) => { - tracing::warn!( - "[RULE {}] invalid file config '{}': {}", - rule_id, - lookup, - e - ); - return None; - } - } - } - } - } - if let Some(path) = def.options.get("DATASOURCE") { - if !path.trim().is_empty() { - config.path = path.clone(); - } - } - if config.path.trim().is_empty() { - tracing::warn!( - "[RULE {}] file stream '{}' has neither DATASOURCE path nor file config", - rule_id, - stream_name - ); - return None; - } - if let Some(format) = def - .options - .get("FORMAT") - .filter(|s| !s.trim().is_empty()) - .or_else(|| def.options.get("FILETYPE").filter(|s| !s.trim().is_empty())) - { - config.format = format.clone(); - } - if let Some(flag) = def - .options - .get("HASHEADER") - .or_else(|| def.options.get("HAS_HEADER")) - { - match flag.trim().to_ascii_lowercase().as_str() { - "true" | "1" | "yes" => config.has_header = true, - "false" | "0" | "no" | "" => config.has_header = false, - _ => {} - } - } - if let Some(delim) = def.options.get("DELIMITER").filter(|s| !s.is_empty()) { - config.delimiter = Some(DelimitedCodec::delimiter_from_name(delim)); - } - Some(config) -} - -/// Signal cancellation to a rule's background streaming sources. -/// Streams shared with other running rules remain active; when the last -/// rule consuming a stream stops, the stream's background source task -/// is cancelled and cleanly disconnected. -pub(crate) fn cancel_rule_source(state: &AppState, rule_id: &str) { - if let Some(txs) = state.source_cancels.write().remove(rule_id) { - for tx in txs { - let _ = tx.send(true); - } - } - let streams = state.rule_streams.write().remove(rule_id); - if let Some(streams) = streams { - let mut active_rules = state.stream_active_rules.write(); - let mut source_cancels = state.stream_source_cancels.write(); - let mut attach_meta = state.stream_attach_meta.write(); - for stream in streams { - let mut should_cleanup = false; - if let Some(rules) = active_rules.get_mut(&stream) { - rules.remove(rule_id); - should_cleanup = rules.is_empty(); - } - if should_cleanup { - active_rules.remove(&stream); - attach_meta.remove(&stream); - state.stream_field_masks.write().remove(&stream); - state.stream_rule_field_masks.write().remove(&stream); - if let Some(cancels) = source_cancels.remove(&stream) { - for tx in cancels { - let _ = tx.send(true); - } - } - state - .http_push_endpoints - .write() - .retain(|_, ep| ep.stream_name != stream); - } else { - let recomputed = { - let mut rule_masks = state.stream_rule_field_masks.write(); - if let Some(rm) = rule_masks.get_mut(&stream) { - rm.remove(rule_id); - compute_union_field_mask(rm) - } else { - None - } - }; - if let Some(flag) = state.stream_field_masks.read().get(&stream) { - *flag.write() = recomputed; - } - } - } - } -} - -/// Resolve the bus topic a rule actually subscribes to: memory-type streams -/// (`TYPE="memory"`) re-export another topic via `DATASOURCE`, mirroring the -/// eKuiper memory source (used to chain rules through memory sinks). -pub(crate) fn resolve_source_topic(stream_manager: &StreamManager, stream_name: &str) -> String { - if let Some(def) = stream_manager.get_stream(stream_name) { - if def - .options - .get("TYPE") - .is_some_and(|t| t.eq_ignore_ascii_case("memory")) - { - if let Some(ds) = def.options.get("DATASOURCE") { - if !ds.is_empty() { - return ds.clone(); - } - } - } - } - stream_name.to_string() -} +use rekuiper_core::model::{RuleStatus, StreamRecord}; +use rekuiper_core::{ + RuleCounters, RuleManager, StreamBus, StreamManager, StreamReceiver, TableManager, +}; +use rekuiper_sql::{SelectStmt, WindowDef}; +use serde_json::Value; +use std::collections::HashMap; +use std::sync::Arc; /// Extract an atomic StreamBus pre-filter from a SQL WHERE expression. /// For `A AND B`, even if `B` is a complex function call, extracting `A` @@ -2518,811 +793,16 @@ pub(crate) fn spawn_rule_task( rule_manager.set_rule_handle(&rule_id, handle); } -/// Resolve the polling config for a `TYPE="sql"` source stream: a matching -/// `sql/{conf_key}` source config wins, otherwise the stream options (`URL`, -/// falling back to `DATASOURCE`, plus `TABLE` or the stream name and an -/// optional `INTERVAL` poll period). Option names match case-insensitively. -pub(crate) fn resolve_sql_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, - rule_id: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - let kind = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("TYPE")) - .map(|(_, v)| v.clone()) - .unwrap_or_default(); - if !kind.eq_ignore_ascii_case("sql") { - return None; - } - let conf_key = def - .options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("CONF_KEY") - || k.eq_ignore_ascii_case("confKey") - || k.eq_ignore_ascii_case("connectionSelector") - }) - .map(|(_, v)| v.trim().to_string()); - if let Some(key) = conf_key { - if !key.is_empty() { - let lookup = format!("sql/{}", key); - let configs_guard = source_configs.read(); - let conf_val = configs_guard - .get(&lookup) - .or_else(|| configs_guard.get(&key)) - .or_else(|| configs_guard.get(&format!("connections/{}", key))) - .cloned(); - drop(configs_guard); - if let Some(val) = conf_val { - match serde_json::from_value::(val) { - Ok(mut conf) => { - // Documented plugin configs carry the URL as `dburl` - // and may omit the table: fall back to the stream - // options (DATASOURCE/TABLE) for whatever is missing. - if conf.url.trim().is_empty() { - conf.url = def - .options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("URL") - || k.eq_ignore_ascii_case("DATASOURCE") - }) - .map(|(_, v)| v.clone()) - .unwrap_or_default(); - } - if conf.table.trim().is_empty() { - conf.table = sql_table_fallback(&def.options, stream_name); - } - if conf.url.trim().is_empty() || conf.table.trim().is_empty() { - // A template-SQL config without any table anchor - // cannot poll; surface the misconfiguration. - tracing::warn!( - "[RULE {}] sql source config '{}' has no usable url/table", - rule_id, - lookup - ); - return None; - } - return Some(conf); - } - Err(e) => { - tracing::warn!( - "[RULE {}] invalid sql source config '{}': {}", - rule_id, - lookup, - e - ); - return None; - } - } - } - } - } - let url = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("URL") || k.eq_ignore_ascii_case("DATASOURCE")) - .map(|(_, v)| v.clone()) - .unwrap_or_default(); - if url.trim().is_empty() { - return None; - } - let table = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("TABLE")) - .map(|(_, v)| v.clone()) - .unwrap_or_else(|| stream_name.to_string()); - let interval = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("INTERVAL")) - .and_then(|(_, v)| v.trim().parse::().ok()) - .unwrap_or(1000) - .max(1); - Some(SqlConnectorConfig { - url, - table, - fields: Vec::new(), - interval, - template_sql_query_cfg: None, - internal_sql_query_cfg: None, - }) -} - -pub(crate) fn resolve_rabbitmq_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - let is_rmq = def - .options - .get("TYPE") - .is_some_and(|t| t.eq_ignore_ascii_case("rabbitmq") || t.eq_ignore_ascii_case("amqp")); - if !is_rmq { - return None; - } - let mut config = RabbitMqConfig::default(); - let conf_key = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("CONF_KEY") || k.eq_ignore_ascii_case("confKey")) - .map(|(_, v)| v.trim()); - if let Some(key) = conf_key { - if !key.is_empty() { - let lookup = format!("rabbitmq/{}", key); - let configs_guard = source_configs.read(); - let conf_val = configs_guard - .get(&lookup) - .or_else(|| configs_guard.get(key)) - .cloned(); - drop(configs_guard); - if let Some(val) = conf_val { - if let Ok(c) = serde_json::from_value::(val) { - config = c; - } - } - } - } - if let Some(srv) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("SERVER") || k.eq_ignore_ascii_case("URL")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.server = srv.to_string(); - } - if let Some(q) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("QUEUE") || k.eq_ignore_ascii_case("DATASOURCE")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.queue = q.to_string(); - } - if let Some(ex) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("EXCHANGE")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.exchange = ex.to_string(); - } - if let Some(rk) = def - .options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("ROUTINGKEY") || k.eq_ignore_ascii_case("ROUTING_KEY") - }) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.routing_key = rk.to_string(); - } - if let Some(u) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("USERNAME") || k.eq_ignore_ascii_case("USER")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.username = Some(u.to_string()); - } - if let Some(p) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("PASSWORD")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.password = Some(p.to_string()); - } - Some(config) -} - -pub(crate) fn resolve_zmq_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - let is_zmq = def - .options - .get("TYPE") - .is_some_and(|t| t.eq_ignore_ascii_case("zmq") || t.eq_ignore_ascii_case("zeromq")); - if !is_zmq { - return None; - } - let mut config = ZmqSourceConfig::default(); - let conf_key = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("CONF_KEY") || k.eq_ignore_ascii_case("confKey")) - .map(|(_, v)| v.trim()); - if let Some(key) = conf_key { - if !key.is_empty() { - let lookup = format!("zmq/{}", key); - let configs_guard = source_configs.read(); - let conf_val = configs_guard - .get(&lookup) - .or_else(|| configs_guard.get(key)) - .cloned(); - drop(configs_guard); - if let Some(val) = conf_val { - if let Ok(c) = serde_json::from_value::(val) { - config = c; - } - } - } - } - if let Some(srv) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("SERVER") || k.eq_ignore_ascii_case("URL")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.server = srv.to_string(); - } - if let Some(topic) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("DATASOURCE") || k.eq_ignore_ascii_case("TOPIC")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.topic = Some(topic.to_string()); - } - if let Some(mode) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("MODE")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.mode = mode.to_string(); - } - Some(config) -} - -pub(crate) fn resolve_video_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - let is_video = def - .options - .get("TYPE") - .is_some_and(|t| t.eq_ignore_ascii_case("video")); - if !is_video { - return None; - } - let mut config = VideoSourceConfig::default(); - let conf_key = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("CONF_KEY") || k.eq_ignore_ascii_case("confKey")) - .map(|(_, v)| v.trim()); - if let Some(key) = conf_key { - if !key.is_empty() { - let lookup = format!("video/{}", key); - let configs_guard = source_configs.read(); - let conf_val = configs_guard - .get(&lookup) - .or_else(|| configs_guard.get(key)) - .cloned(); - drop(configs_guard); - if let Some(val) = conf_val { - if let Ok(c) = serde_json::from_value::(val) { - config = c; - } - } - } - } - if let Some(url) = def - .options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("URL") - || k.eq_ignore_ascii_case("SERVER") - || k.eq_ignore_ascii_case("DATASOURCE") - }) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.url = url.to_string(); - } - if let Some(interval) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("INTERVAL")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.set_interval_str(interval); - } - if let Some(codec) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("CODEC")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.codec = codec.to_string(); - } - if let Some(debug) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("DEBUG_RESP") || k.eq_ignore_ascii_case("DEBUGRESP")) - .map(|(_, v)| v.trim()) - { - config.debug_resp = debug.eq_ignore_ascii_case("true"); - } - if let Some(ffmpeg) = def - .options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("FFMPEG_PATH") || k.eq_ignore_ascii_case("FFMPEGPATH") - }) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.ffmpeg_path = ffmpeg.to_string(); - } - Some(config) -} - -pub(crate) fn resolve_simulator_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, - rule_id: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - let is_sim = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("TYPE")) - .map(|(_, v)| v.trim()) - .is_some_and(|t| t.eq_ignore_ascii_case("simulator") || t.eq_ignore_ascii_case("random")); - if !is_sim { - return None; - } - let mut config = SimulatorConfig { - data: Vec::new(), - interval: Value::from(10), - loop_data: true, - }; - let conf_key = def - .options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("CONF_KEY") - || k.eq_ignore_ascii_case("confKey") - || k.eq_ignore_ascii_case("connectionSelector") - }) - .map(|(_, v)| v.trim()); - let key = conf_key.unwrap_or("default"); - if !key.is_empty() { - let lookup = format!("simulator/{}", key); - let lookup_random = format!("random/{}", key); - let configs_guard = source_configs.read(); - let conf_val = configs_guard - .get(&lookup) - .or_else(|| configs_guard.get(&lookup_random)) - .or_else(|| configs_guard.get(key)) - .cloned(); - drop(configs_guard); - if let Some(val) = conf_val { - match serde_json::from_value::(val) { - Ok(c) => config = c, - Err(e) => { - tracing::warn!( - "[RULE {}] invalid simulator config '{}': {}", - rule_id, - lookup, - e - ); - } - } - } - } - if let Some(interval) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("INTERVAL")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - if let Ok(num) = interval.parse::() { - config.interval = Value::from(num); - } else { - config.interval = Value::String(interval.to_string()); - } - } - if let Some(loop_opt) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("LOOP")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - match loop_opt.to_ascii_lowercase().as_str() { - "true" | "1" | "yes" => config.loop_data = true, - "false" | "0" | "no" => config.loop_data = false, - _ => {} - } - } - if let Some(data_str) = def - .options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("DATA")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - if let Ok(parsed_data) = serde_json::from_str::>>(data_str) { - config.data = parsed_data; - } else if let Ok(single) = serde_json::from_str::>(data_str) { - config.data = vec![single]; - } - } - Some(config) -} - -pub(crate) fn resolve_modbus_options( - options: &HashMap, - source_configs: &Arc>>, -) -> Option { - let is_modbus = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("TYPE")) - .map(|(_, v)| v.trim()) - .is_some_and(|t| t.eq_ignore_ascii_case("modbus")); - if !is_modbus { - return None; - } - let mut config = rekuiper_connectors::ModbusConfig::default(); - if let Some(key) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("CONF_KEY") || k.eq_ignore_ascii_case("confKey")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - let lookup = format!("modbus/{}", key); - let lookup_src = format!("sources/modbus/{}", key); - let guard = source_configs.read(); - let conf_val = guard - .get(&lookup) - .or_else(|| guard.get(&lookup_src)) - .or_else(|| guard.get(key)) - .cloned(); - drop(guard); - if let Some(val) = conf_val { - if let Ok(c) = serde_json::from_value::(val) { - config = c; - } - } - } - if let Some(ep) = options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("ENDPOINT") - || k.eq_ignore_ascii_case("URL") - || k.eq_ignore_ascii_case("SERVER") - || k.eq_ignore_ascii_case("DATASOURCE") - }) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.endpoint = ep.to_string(); - } - if let Some(tp) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("TRANSPORT")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.transport = tp.to_string(); - } - if let Some(u) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("UNIT_ID") || k.eq_ignore_ascii_case("SLAVE_ID")) - .and_then(|(_, v)| v.trim().parse::().ok()) - { - config.unit_id = u; - } - if let Some(iv) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("INTERVAL")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.interval_ms = - rekuiper_connectors::simulator::parse_interval_ms(&Value::String(iv.to_string())) - .as_millis() - .max(1) as u64; - } - if let Some(to) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("TIMEOUT")) - .and_then(|(_, v)| v.trim().parse::().ok()) - { - config.timeout_ms = to; - } - if let Some(ab) = options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("ADDRESS_BASE") || k.eq_ignore_ascii_case("ADDRESSBASE") - }) - .and_then(|(_, v)| v.trim().parse::().ok()) - { - config.address_base = Some(ab); - } - if let Some(plc) = options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("PLC_ADDRESSING") || k.eq_ignore_ascii_case("PLCADDRESSING") - }) - .map(|(_, v)| v.trim().eq_ignore_ascii_case("true") || v.trim() == "1") - { - config.plc_addressing = plc; - } - if let Some(mg) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("MAX_GAP") || k.eq_ignore_ascii_case("MAXGAP")) - .and_then(|(_, v)| v.trim().parse::().ok()) - { - config.max_gap = mg; - } - if let Some(regs) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("REGISTERS")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - if let Ok(r) = serde_json::from_str::>(regs) { - config.registers = r; - } - } - Some(config) -} - -pub(crate) fn resolve_modbus_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - resolve_modbus_options(&def.options, source_configs) -} - -pub(crate) fn resolve_modbus_table_config( - table_manager: &TableManager, - source_configs: &Arc>>, - table_name: &str, -) -> Option { - let def = table_manager.get_table(table_name)?; - resolve_modbus_options(&def.options, source_configs) -} - -pub(crate) fn resolve_opcua_options( - options: &HashMap, - source_configs: &Arc>>, -) -> Option { - let is_opcua = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("TYPE")) - .map(|(_, v)| v.trim()) - .is_some_and(|t| { - t.eq_ignore_ascii_case("opcua") - || t.eq_ignore_ascii_case("opc_ua") - || t.eq_ignore_ascii_case("opc-ua") - }); - if !is_opcua { - return None; - } - let mut config = rekuiper_connectors::OpcUaConfig::default(); - if let Some(key) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("CONF_KEY") || k.eq_ignore_ascii_case("confKey")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - let lookup = format!("opcua/{}", key); - let lookup_src = format!("sources/opcua/{}", key); - let guard = source_configs.read(); - let conf_val = guard - .get(&lookup) - .or_else(|| guard.get(&lookup_src)) - .or_else(|| guard.get(key)) - .cloned(); - drop(guard); - if let Some(val) = conf_val { - if let Ok(c) = serde_json::from_value::(val) { - config = c; - } - } - } - if let Some(ep) = options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("ENDPOINT") - || k.eq_ignore_ascii_case("URL") - || k.eq_ignore_ascii_case("SERVER") - || k.eq_ignore_ascii_case("DATASOURCE") - }) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.endpoint = ep.to_string(); - } - if let Some(iv) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("INTERVAL")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.interval_ms = - rekuiper_connectors::simulator::parse_interval_ms(&Value::String(iv.to_string())) - .as_millis() - .max(1) as u64; - } - if let Some(to) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("TIMEOUT")) - .and_then(|(_, v)| v.trim().parse::().ok()) - { - config.timeout_ms = to; - } - if let Some(sp) = options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("SECURITY_POLICY") || k.eq_ignore_ascii_case("SECURITYPOLICY") - }) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - config.security_policy = sp.to_string(); - } - if let Some(u) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("USERNAME") || k.eq_ignore_ascii_case("USER")) - .map(|(_, v)| v.trim().to_string()) - .filter(|s| !s.is_empty()) - { - config.username = Some(u); - } - if let Some(p) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("PASSWORD") || k.eq_ignore_ascii_case("PASS")) - .map(|(_, v)| v.trim().to_string()) - .filter(|s| !s.is_empty()) - { - config.password = Some(p); - } - if let Some(nodes) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("NODES")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - if let Ok(n) = serde_json::from_str::>(nodes) { - config.nodes = n; - } - } - Some(config) -} - -pub(crate) fn resolve_opcua_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - resolve_opcua_options(&def.options, source_configs) -} - -pub(crate) fn resolve_opcua_table_config( - table_manager: &TableManager, - source_configs: &Arc>>, - table_name: &str, -) -> Option { - let def = table_manager.get_table(table_name)?; - resolve_opcua_options(&def.options, source_configs) -} - -pub(crate) fn resolve_otlp_options( - options: &HashMap, - source_configs: &Arc>>, -) -> Option { - let is_otlp = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("TYPE")) - .map(|(_, v)| v.trim()) - .is_some_and(|t| { - t.eq_ignore_ascii_case("otlp") - || t.eq_ignore_ascii_case("opentelemetry") - || t.eq_ignore_ascii_case("otel") - }); - if !is_otlp { - return None; - } - let mut config = rekuiper_connectors::OtlpSourceConfig { - bind: None, - endpoint: None, - interval: None, - }; - if let Some(key) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("CONF_KEY") || k.eq_ignore_ascii_case("confKey")) - .map(|(_, v)| v.trim()) - .filter(|s| !s.is_empty()) - { - let lookup = format!("otlp/{}", key); - let lookup_src = format!("sources/otlp/{}", key); - let guard = source_configs.read(); - let conf_val = guard - .get(&lookup) - .or_else(|| guard.get(&lookup_src)) - .or_else(|| guard.get(key)) - .cloned(); - drop(guard); - if let Some(v) = conf_val { - if let Ok(c) = serde_json::from_value::(v) { - config = c; - } - } - } - if let Some(ep) = options - .iter() - .find(|(k, _)| { - k.eq_ignore_ascii_case("ENDPOINT") - || k.eq_ignore_ascii_case("BIND") - || k.eq_ignore_ascii_case("SERVER") - }) - .map(|(_, v)| v.trim().to_string()) - .filter(|s| !s.is_empty()) - { - config.bind = Some(ep.clone()); - config.endpoint = Some(ep); - } - if let Some(iv) = options - .iter() - .find(|(k, _)| k.eq_ignore_ascii_case("INTERVAL")) - .map(|(_, v)| v.trim().to_string()) - .filter(|s| !s.is_empty()) - { - config.interval = Some(iv); - } - Some(config) -} - -pub(crate) fn resolve_otlp_source( - stream_manager: &StreamManager, - source_configs: &Arc>>, - stream_name: &str, -) -> Option { - let def = stream_manager.get_stream(stream_name)?; - resolve_otlp_options(&def.options, source_configs) -} - #[cfg(test)] mod tests { use super::*; use crate::state::{ der_read_tlv, load_config_maps, parse_pem_block, persist_config_entry, spki_to_pkcs1, - unpersist_config_entry, verify_jwt_raw, + unpersist_config_entry, verify_jwt_raw, AppState, }; - use rekuiper_core::{KvStore, StreamDefinition}; + use rekuiper_connectors::PayloadFormat; + use rekuiper_core::{KvStore, SchemaManager, StreamDefinition, TableDefinition}; + use rekuiper_sql::Parser; use serde_json::json; async fn test_stream_manager(options: &[(&str, &str)]) -> StreamManager { diff --git a/crates/rekuiper-server/src/engine/sources.rs b/crates/rekuiper-server/src/engine/sources.rs new file mode 100644 index 00000000..cfc80f87 --- /dev/null +++ b/crates/rekuiper-server/src/engine/sources.rs @@ -0,0 +1,2533 @@ +use super::*; +use crate::handlers::{stmt_called_functions, stmt_referenced_fields}; +use crate::state::{AppState, HttpPushEndpoint}; +use parking_lot::RwLock; +use rekuiper_connectors::codec::extract_required_fields_from_expr_names; +use rekuiper_connectors::{ + FileSource, FileSourceConfig, HttpPullConfig, HttpPullSource, JsonFieldMask, KafkaConfig, + KafkaSource, ModbusSource, MqttConfig, MqttSource, OpcUaSource, PayloadFormat, RabbitMqConfig, + RabbitMqSource, RedisSubSource, SimulatorConfig, SimulatorSource, SqlConnectorConfig, + SqlSource, VideoSource, VideoSourceConfig, WebSocketConfig, WebSocketSource, ZmqSource, + ZmqSourceConfig, +}; +use rekuiper_core::model::{StreamField, StreamRecord}; +use rekuiper_core::{SchemaManager, StreamManager, TableDefinition, TableManager}; +use rekuiper_sql::{Parser, SelectStmt}; +use serde_json::Value; +use std::collections::HashMap; +use std::sync::Arc; + +/// Starts background source producers (HTTP pull, WebSocket, RedisSub, Kafka, +/// simulator) for a rule based on its source stream type. Shared by rule +/// creation and daemon-bootstrap restore. +/// Resolve the MQTT source configuration for a rule's stream. +/// +/// MQTT is the default streaming source: streams with no `TYPE`, an empty +/// `TYPE`, or `TYPE="mqtt"` ingest from the broker. Any other non-empty +/// `TYPE` (recognized sources like `kafka`, or anything else) resolves to +/// `None` here so exactly one source bootstrap owns the stream. +pub(crate) fn resolve_mqtt_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + schemas: &SchemaManager, + stream_name: &str, + rule_id: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + if let Some(kind) = def.options.get("TYPE") { + if !kind.trim().is_empty() && !kind.eq_ignore_ascii_case("mqtt") { + return None; + } + } + let mut config = MqttConfig::default(); + // CONF_KEY lookup is case-insensitive (`CONF_KEY`, `conf_key`, + // `confKey`): SQL definitions and imported/JSON definitions disagree on + // case. Stored configs typically carry only connection parameters, so a + // failed full decode still salvages server/credentials field by field. + let conf_key = def + .options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("CONF_KEY") + || k.eq_ignore_ascii_case("confKey") + || k.eq_ignore_ascii_case("connectionSelector") + }) + .map(|(_, v)| v.trim()); + if let Some(key) = conf_key { + if !key.is_empty() { + let lookup1 = format!("mqtt/{}", key); + let configs_guard = source_configs.read(); + let conf_val = configs_guard + .get(&lookup1) + .or_else(|| configs_guard.get(key)) + .or_else(|| configs_guard.get(&format!("connections/{}", key))) + .cloned(); + drop(configs_guard); + if let Some(val) = conf_val { + if let Ok(stored) = serde_json::from_value::(val.clone()) { + config = stored; + } else if let Some(srv) = val.get("server").and_then(|v| v.as_str()) { + if !srv.trim().is_empty() { + config.server = srv.to_string(); + } + if let Some(u) = val.get("username").and_then(|v| v.as_str()) { + config.username = Some(u.to_string()); + } + if let Some(p) = val.get("password").and_then(|v| v.as_str()) { + config.password = Some(p.to_string()); + } + } else { + tracing::warn!( + "[RULE {}] invalid mqtt config '{}': no server field", + rule_id, + lookup1, + ); + } + } + } + } + // Direct SERVER option overrides the configuration key (case-insensitive). + let server_opt = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("SERVER")) + .map(|(_, v)| v.trim()); + if let Some(srv) = server_opt { + if !srv.is_empty() { + config.server = srv.to_string(); + } + } + // DATASOURCE / topic (case-insensitive). + let topic_opt = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("DATASOURCE") || k.eq_ignore_ascii_case("topic")) + .map(|(_, v)| v.trim()); + if let Some(top) = topic_opt { + if !top.is_empty() { + config.topic = top.to_string(); + } + } + if config.topic.trim().is_empty() { + config.topic = stream_name.to_string(); + } + let client_id = def + .options + .get("CLIENTID") + .or_else(|| def.options.get("CLIENT_ID")); + if let Some(id) = client_id { + if !id.trim().is_empty() { + config.client_id = Some(id.clone()); + } + } + if let Some(user) = def.options.get("USERNAME") { + if !user.is_empty() { + config.username = Some(user.clone()); + } + } + if let Some(pass) = def.options.get("PASSWORD") { + if !pass.is_empty() { + config.password = Some(pass.clone()); + } + } + if let Some(qos) = def.options.get("QOS") { + match qos.trim().parse::() { + Ok(q) => config.qos = q, + Err(_) => { + tracing::warn!( + "[RULE {}] invalid mqtt QOS '{}', keeping {}", + rule_id, + qos, + config.qos + ); + } + } + } + config.format = resolve_payload_format(schemas, &def, rule_id)?; + Some(config) +} + +/// Resolve the MQTT configuration and EdgeX payload codec for a stream declaring +/// `TYPE="edgex"`. Default server is `tcp://edgex-mqtt-broker:1883` and default topic +/// is `edgex/rules-events`, configured via `etc/sources/edgex.yaml` or `EDGEX__*` overlays. +pub(crate) fn resolve_edgex_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + _schemas: &SchemaManager, + stream_name: &str, + rule_id: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + let kind = def.options.get("TYPE")?; + if !kind.eq_ignore_ascii_case("edgex") { + return None; + } + + let mut config = MqttConfig { + server: "tcp://edgex-mqtt-broker:1883".to_string(), + topic: "edgex/rules-events".to_string(), + format: PayloadFormat::EdgeX, + ..Default::default() + }; + + let conf_key = def + .options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("CONF_KEY") + || k.eq_ignore_ascii_case("confKey") + || k.eq_ignore_ascii_case("connectionSelector") + }) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + .unwrap_or("default"); + + let lookup = format!("edgex/{}", conf_key); + let configs_guard = source_configs.read(); + let conf_val = configs_guard + .get(&lookup) + .or_else(|| configs_guard.get(&lookup.to_ascii_lowercase())) + .or_else(|| configs_guard.get("edgex/default")) + .or_else(|| configs_guard.get(conf_key)) + .cloned(); + drop(configs_guard); + + if let Some(val) = conf_val { + let srv = val + .get("server") + .and_then(|v| v.as_str()) + .unwrap_or("edgex-mqtt-broker"); + let port = val + .get("port") + .and_then(|v| { + if let Some(n) = v.as_u64() { + Some(n) + } else if let Some(s) = v.as_str() { + s.parse::().ok() + } else { + None + } + }) + .unwrap_or(1883); + let proto = val + .get("protocol") + .and_then(|v| v.as_str()) + .unwrap_or("tcp"); + + if srv.contains("://") { + config.server = srv.to_string(); + } else { + config.server = format!("{}://{}:{}", proto, srv, port); + } + + if let Some(t) = val.get("topic").and_then(|v| v.as_str()) { + if !t.is_empty() { + config.topic = t.to_string(); + } + } + if let Some(opt) = val.get("optional") { + if let Some(u) = opt + .get("Username") + .or_else(|| opt.get("username")) + .and_then(|v| v.as_str()) + { + config.username = Some(u.to_string()); + } + if let Some(p) = opt + .get("Password") + .or_else(|| opt.get("password")) + .and_then(|v| v.as_str()) + { + config.password = Some(p.to_string()); + } + if let Some(cid) = opt + .get("ClientId") + .or_else(|| opt.get("clientId")) + .and_then(|v| v.as_str()) + { + config.client_id = Some(cid.to_string()); + } + } + } + + // Direct stream options take precedence + if let Some(srv) = def.options.get("SERVER").filter(|s| !s.trim().is_empty()) { + if srv.contains("://") { + config.server = srv.to_string(); + } else { + config.server = format!("tcp://{}", srv); + } + } + if let Some(ds) = def + .options + .get("DATASOURCE") + .filter(|s| !s.trim().is_empty()) + { + config.topic = ds.trim().to_string(); + } + + tracing::info!( + "[RULE {}] resolved edgex source for '{}': broker '{}', topic '{}'", + rule_id, + stream_name, + config.server, + config.topic + ); + Some(config) +} + +/// Payload decoding for a message source from the stream `FORMAT` (eKuiper +/// names: json, binary, delimited, protobuf with `SCHEMAID`). A protobuf +/// stream whose schema cannot be resolved yields `None` so the stream fails +/// loudly instead of decoding garbage; unknown formats fall back to JSON. +pub(crate) fn resolve_payload_format( + schemas: &SchemaManager, + def: &rekuiper_core::model::StreamDefinition, + rule_id: &str, +) -> Option { + resolve_payload_format_options( + schemas, + &def.options, + &def.stream_fields, + &def.name, + rule_id, + ) +} + +pub(crate) fn resolve_payload_format_options( + schemas: &SchemaManager, + options: &HashMap, + stream_fields: &[StreamField], + source_name: &str, + rule_id: &str, +) -> Option { + use rekuiper_connectors::{DelimitedCodec, PayloadFormat}; + let opt = |name: &str| { + options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case(name)) + .map(|(_, v)| v.trim()) + }; + let format = opt("FORMAT").unwrap_or("").to_ascii_lowercase(); + match format.as_str() { + "" | "json" => Some(PayloadFormat::Json), + "binary" => Some(PayloadFormat::Binary), + "delimited" => { + let delimiter = opt("DELIMITER") + .filter(|d| !d.is_empty()) + .map(DelimitedCodec::delimiter_from_name) + .unwrap_or(','); + let headers = stream_fields.iter().map(|f| f.name.clone()).collect(); + Some(PayloadFormat::Delimited(DelimitedCodec::new( + delimiter, headers, + ))) + } + "protobuf" => match resolve_proto_message(schemas, opt("SCHEMAID").unwrap_or("")) { + Ok(message) => Some(PayloadFormat::Protobuf(Arc::new(message))), + Err(e) => { + tracing::warn!("[RULE {}] source '{}': {}", rule_id, source_name, e); + None + } + }, + other => { + tracing::warn!( + "[RULE {}] source '{}': FORMAT '{}' is not supported for this source; decoding as JSON", + rule_id, + source_name, + other + ); + Some(PayloadFormat::Json) + } + } +} + +/// Resolve `SCHEMAID` (`.`) against registered protobuf +/// schemas (inline content, else the schema file). +pub(crate) fn resolve_proto_message( + schemas: &SchemaManager, + schema_id: &str, +) -> Result { + let (schema, message) = schema_id.split_once('.').ok_or_else(|| { + format!( + "protobuf SCHEMAID '{}' must be .", + schema_id + ) + })?; + let def = schemas + .get_schema("protobuf", schema) + .ok_or_else(|| format!("protobuf schema '{}' is not registered", schema))?; + let text = match (&def.content, &def.file) { + (Some(content), _) if !content.trim().is_empty() => content.clone(), + (_, Some(path)) => std::fs::read_to_string(path) + .map_err(|e| format!("cannot read protobuf schema file '{}': {}", path, e))?, + _ => return Err(format!("protobuf schema '{}' has no content", schema)), + }; + rekuiper_connectors::parse_proto(&text)? + .remove(message) + .ok_or_else(|| { + format!( + "message '{}' not found in protobuf schema '{}'", + message, schema + ) + }) +} + +pub(crate) fn compute_union_field_mask( + rules_map: &HashMap>>, +) -> Option { + if rules_map.is_empty() { + return None; + } + let mut union_set = std::collections::HashSet::new(); + for rule_fields_opt in rules_map.values() { + let fields = rule_fields_opt.as_ref()?; + union_set.extend(fields.iter().cloned()); + } + if union_set.is_empty() { + None + } else { + Some(JsonFieldMask::new(union_set)) + } +} + +pub(crate) fn update_stream_field_mask( + state: &AppState, + stream_name: &str, + rule_id: &str, + field_mask: Option, +) -> Arc>> { + let rule_fields = field_mask.map(|m| (*m.fields).clone()); + let combined = { + let mut map = state.stream_rule_field_masks.write(); + let rules_map = map.entry(stream_name.to_string()).or_default(); + rules_map.insert(rule_id.to_string(), rule_fields); + compute_union_field_mask(rules_map) + }; + let mut masks_guard = state.stream_field_masks.write(); + if let Some(flag) = masks_guard.get(stream_name) { + *flag.write() = combined; + flag.clone() + } else { + let flag = Arc::new(parking_lot::RwLock::new(combined)); + masks_guard.insert(stream_name.to_string(), flag.clone()); + flag + } +} + +pub(crate) fn cancel_stream_sources(state: &AppState, stream_name: &str) { + if let Some(cancels) = state.stream_source_cancels.write().remove(stream_name) { + for tx in cancels { + let _ = tx.send(true); + } + } + state.stream_active_rules.write().remove(stream_name); + state.stream_attach_meta.write().remove(stream_name); + state.stream_field_masks.write().remove(stream_name); + state.stream_rule_field_masks.write().remove(stream_name); + state + .http_push_endpoints + .write() + .retain(|_, ep| ep.stream_name != stream_name); +} + +pub(crate) fn cancel_table_source(state: &AppState, table_name: &str) { + let table_key = format!("$table/{}", table_name); + if let Some(cancels) = state.stream_source_cancels.write().remove(&table_key) { + for tx in cancels { + let _ = tx.send(true); + } + } +} + +pub(crate) fn resolve_mqtt_table_source( + table_def: &TableDefinition, + source_configs: &Arc>>, + schemas: &SchemaManager, +) -> Option { + if let Some(kind) = table_def.options.get("TYPE") { + if !kind.trim().is_empty() && !kind.eq_ignore_ascii_case("mqtt") { + return None; + } + } + let mut config = MqttConfig::default(); + let conf_key = table_def + .options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("CONF_KEY") + || k.eq_ignore_ascii_case("confKey") + || k.eq_ignore_ascii_case("connectionSelector") + }) + .map(|(_, v)| v.trim()); + if let Some(key) = conf_key { + if !key.is_empty() { + let lookup1 = format!("mqtt/{}", key); + let configs_guard = source_configs.read(); + let conf_val = configs_guard + .get(&lookup1) + .or_else(|| configs_guard.get(key)) + .or_else(|| configs_guard.get(&format!("connections/{}", key))) + .cloned(); + drop(configs_guard); + if let Some(val) = conf_val { + if let Ok(stored) = serde_json::from_value::(val.clone()) { + config = stored; + } else if let Some(srv) = val.get("server").and_then(|v| v.as_str()) { + if !srv.trim().is_empty() { + config.server = srv.to_string(); + } + if let Some(u) = val.get("username").and_then(|v| v.as_str()) { + config.username = Some(u.to_string()); + } + if let Some(p) = val.get("password").and_then(|v| v.as_str()) { + config.password = Some(p.to_string()); + } + } + } + } + } + if let Some((_, srv)) = table_def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("SERVER")) + { + let srv = srv.trim(); + if !srv.is_empty() { + config.server = srv.to_string(); + } + } + if let Some((_, top)) = table_def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("DATASOURCE") || k.eq_ignore_ascii_case("topic")) + { + let top = top.trim(); + if !top.is_empty() { + config.topic = top.to_string(); + } + } + if config.topic.trim().is_empty() { + config.topic = table_def.name.clone(); + } + if let Some(id) = table_def + .options + .get("CLIENTID") + .or_else(|| table_def.options.get("CLIENT_ID")) + { + if !id.trim().is_empty() { + config.client_id = Some(id.clone()); + } + } + if let Some(user) = table_def.options.get("USERNAME") { + if !user.is_empty() { + config.username = Some(user.clone()); + } + } + if let Some(pass) = table_def.options.get("PASSWORD") { + if !pass.is_empty() { + config.password = Some(pass.clone()); + } + } + if let Some(qos) = table_def.options.get("QOS") { + if let Ok(q) = qos.trim().parse::() { + config.qos = q; + } + } + config.format = resolve_payload_format_options( + schemas, + &table_def.options, + &table_def.stream_fields, + &table_def.name, + &table_def.name, + )?; + Some(config) +} + +pub(crate) fn bootstrap_table_source(state: &AppState, table_name: &str) { + let table_key = format!("$table/{}", table_name); + if state.stream_source_cancels.read().contains_key(&table_key) { + return; + } + let Some(def) = state.table_manager.get_table(table_name) else { + return; + }; + let datasource = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("DATASOURCE") || k.eq_ignore_ascii_case("TOPIC")) + .map(|(_, v)| v.trim().to_string()) + .unwrap_or_else(|| table_name.to_string()); + let table_type = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("TYPE")) + .map(|(_, v)| v.trim().to_string()) + .unwrap_or_default(); + + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + state + .stream_source_cancels + .write() + .insert(table_key.clone(), vec![cancel_tx]); + + let t_name = table_name.to_string(); + let table_mgr = state.table_manager.clone(); + + // 1. Listen on the stream bus for this table's datasource topic (handles memory sinks and internal publication) + let mut bus_rx = state.stream_bus.subscribe(&datasource); + let mut table_bus_rx = if datasource != table_name { + Some(state.stream_bus.subscribe(table_name)) + } else { + None + }; + let mut bus_cancel_rx = cancel_rx.clone(); + let t_name_bus = t_name.clone(); + let table_mgr_bus = table_mgr.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + record = bus_rx.recv() => { + match record { + Some(rec) => table_mgr_bus.insert_table_row(&t_name_bus, rec.data.to_hashmap()), + None => break, + } + } + record = async { + if let Some(ref mut rx) = table_bus_rx { + rx.recv().await + } else { + std::future::pending().await + } + } => { + match record { + Some(rec) => table_mgr_bus.insert_table_row(&t_name_bus, rec.data.to_hashmap()), + None => break, + } + } + changed = bus_cancel_rx.changed() => { + if changed.is_err() || *bus_cancel_rx.borrow() { + break; + } + } + } + } + }); + + // 2. If table is MQTT source (TYPE="mqtt" or default when not memory/file/redis/sql): + let is_memory = table_type.eq_ignore_ascii_case("memory"); + let is_redis = table_type.eq_ignore_ascii_case("redis"); + let is_sql = table_type.eq_ignore_ascii_case("sql"); + let is_file = table_type.eq_ignore_ascii_case("file"); + if !is_memory && !is_redis && !is_sql && !is_file { + if let Some(config) = + resolve_mqtt_table_source(&def, &state.source_configs, &state.schema_manager) + { + let bus_topic = format!("$table_mqtt/{}", table_name); + let stream_tx = state.stream_bus.get_or_create(&bus_topic); + let mut mqtt_rx = state.stream_bus.subscribe(&bus_topic); + let mut mqtt_cancel = cancel_rx.clone(); + let t_name_mqtt = t_name.clone(); + let table_mgr_mqtt = table_mgr.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + record = mqtt_rx.recv() => { + match record { + Some(rec) => table_mgr_mqtt.insert_table_row(&t_name_mqtt, rec.data.to_hashmap()), + None => break, + } + } + changed = mqtt_cancel.changed() => { + if changed.is_err() || *mqtt_cancel.borrow() { + break; + } + } + } + } + }); + MqttSource::new(config, stream_tx).spawn(cancel_rx.clone()); + } + } +} + +pub(crate) fn resolve_httppush_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + let is_push = def + .options + .get("TYPE") + .is_some_and(|t| t.eq_ignore_ascii_case("httppush") || t.eq_ignore_ascii_case("http_push")); + if !is_push { + return None; + } + + let raw_path = def + .options + .get("DATASOURCE") + .or_else(|| def.options.get("ENDPOINT")) + .or_else(|| def.options.get("PATH")) + .map(|s| s.trim()) + .unwrap_or(""); + let path = if raw_path.is_empty() { + format!("/{}", stream_name) + } else { + format!("/{}", raw_path.trim_start_matches('/')) + }; + + let mut method = "POST".to_string(); + if let Some(m) = def.options.get("METHOD") { + if !m.trim().is_empty() { + method = m.trim().to_ascii_uppercase(); + } + } else if let Some(key) = def.options.get("CONF_KEY").map(|k| k.trim()) { + if !key.is_empty() { + let lookup = format!("httppush/{}", key); + let guard = source_configs.read(); + if let Some(val) = guard.get(&lookup).or_else(|| guard.get(key)) { + if let Some(m) = val.get("method").and_then(|v| v.as_str()) { + method = m.trim().to_ascii_uppercase(); + } + } + } + } + + let format = def + .options + .get("FORMAT") + .map(|f| f.trim().to_ascii_lowercase()) + .unwrap_or_else(|| "json".to_string()); + + Some(HttpPushEndpoint { + stream_name: stream_name.to_string(), + path, + method, + format, + }) +} + +pub(crate) fn register_rule_stream_source( + state: &AppState, + stream_name: &str, + rule_id: &str, + cancel_tx: tokio::sync::watch::Sender, +) { + state + .stream_source_cancels + .write() + .entry(stream_name.to_string()) + .or_default() + .push(cancel_tx.clone()); + state + .stream_active_rules + .write() + .entry(stream_name.to_string()) + .or_default() + .insert(rule_id.to_string()); + state + .rule_streams + .write() + .entry(rule_id.to_string()) + .or_default() + .insert(stream_name.to_string()); + let (rule_cancel_tx, _rule_cancel_rx) = tokio::sync::watch::channel(false); + state + .source_cancels + .write() + .entry(rule_id.to_string()) + .or_default() + .push(rule_cancel_tx); +} + +/// Start source producers for one stream (MQTT/file/HTTP-pull/WebSocket/ +/// RedisSub/Kafka/SQL/simulator, whichever its TYPE declares). `needs_meta` +/// asks message sources to attach per-message metadata for `meta()`. +/// Multiple rules reading from the same stream share a single underlying +/// source producer, preventing duplicate message ingestion. +pub(crate) fn bootstrap_stream_sources( + state: &AppState, + rule_id: &str, + stream_name: &str, + needs_meta: bool, + field_mask: Option, +) { + // If this rule is already registered for this stream, update its mask and meta flag. + if let Some(rules) = state.stream_active_rules.read().get(stream_name) { + if rules.contains(rule_id) { + if needs_meta { + if let Some(flag) = state.stream_attach_meta.read().get(stream_name) { + flag.store(true, std::sync::atomic::Ordering::Relaxed); + } + } + update_stream_field_mask(state, stream_name, rule_id, field_mask); + return; + } + } + + // If a source task is already running for this stream, register this rule as + // another consumer and union its required fields into the stream's JsonFieldMask. + if state.stream_source_cancels.read().contains_key(stream_name) { + state + .stream_active_rules + .write() + .entry(stream_name.to_string()) + .or_default() + .insert(rule_id.to_string()); + state + .rule_streams + .write() + .entry(rule_id.to_string()) + .or_default() + .insert(stream_name.to_string()); + if needs_meta { + if let Some(flag) = state.stream_attach_meta.read().get(stream_name) { + flag.store(true, std::sync::atomic::Ordering::Relaxed); + } + } + update_stream_field_mask(state, stream_name, rule_id, field_mask); + let (rule_cancel_tx, _rule_cancel_rx) = tokio::sync::watch::channel(false); + state + .source_cancels + .write() + .entry(rule_id.to_string()) + .or_default() + .push(rule_cancel_tx); + return; + } + + let mask_flag = update_stream_field_mask(state, stream_name, rule_id, field_mask); + let comms = state.comms_hub.clone(); + let status_cb: rekuiper_connectors::ConnectorStatusCallback = + Arc::new(move |id, kind, status, err| { + comms.publish_connector_status(id, kind, status, err.map(|s| s.to_string())); + }); + + // HTTP Push source streams listen on the configured HTTP data server endpoint. + if let Some(endpoint) = + resolve_httppush_source(&state.stream_manager, &state.source_configs, stream_name) + { + let (cancel_tx, mut cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + let path = endpoint.path.clone(); + state + .http_push_endpoints + .write() + .insert(path.clone(), endpoint); + + let state_clone = state.clone(); + let stream_name_clone = stream_name.to_string(); + tokio::spawn(async move { + let _ = cancel_rx.changed().await; + let mut guard = state_clone.http_push_endpoints.write(); + if let Some(ep) = guard.get(&path) { + if ep.stream_name == stream_name_clone { + guard.remove(&path); + } + } + }); + return; + } + + // MQTT is the default streaming source: typeless streams and TYPE="mqtt" + // subscribe to the broker topic and feed the rule pipeline. + if let Some(mut config) = resolve_mqtt_source( + &state.stream_manager, + &state.source_configs, + &state.schema_manager, + stream_name, + rule_id, + ) { + let meta_flag = Arc::new(std::sync::atomic::AtomicBool::new(needs_meta)); + config.attach_meta = needs_meta; + config.field_mask = mask_flag.read().clone(); + state + .stream_attach_meta + .write() + .insert(stream_name.to_string(), meta_flag.clone()); + + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + MqttSource::new(config, stream_tx) + .with_rule_counters(state.rule_manager.rule_counters(rule_id)) + .with_meta_flag(Some(meta_flag)) + .with_field_mask_flag(Some(mask_flag)) + .with_status_callback(Some(status_cb.clone())) + .spawn(cancel_rx); + return; + } + + // EdgeX source streams connect to EdgeX MQTT message bus and decode Event / Reading DTOs. + if let Some(mut config) = resolve_edgex_source( + &state.stream_manager, + &state.source_configs, + &state.schema_manager, + stream_name, + rule_id, + ) { + let meta_flag = Arc::new(std::sync::atomic::AtomicBool::new(needs_meta)); + config.attach_meta = needs_meta; + state + .stream_attach_meta + .write() + .insert(stream_name.to_string(), meta_flag.clone()); + + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + MqttSource::new(config, stream_tx) + .with_rule_counters(state.rule_manager.rule_counters(rule_id)) + .with_meta_flag(Some(meta_flag)) + .spawn(cancel_rx); + return; + } + + // File source streams tail a line-delimited file into the stream bus. + if let Some(config) = resolve_file_source( + &state.stream_manager, + &state.source_configs, + stream_name, + rule_id, + ) { + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + FileSource::new(config, stream_tx).spawn(cancel_rx); + return; + } + + // HTTP pull source streams poll a remote endpoint into the stream bus. + if let Some(conf) = resolve_httppull_config( + &state.stream_manager, + &state.source_configs, + stream_name, + rule_id, + ) { + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + HttpPullSource { + config: conf, + tx: stream_tx, + status_callback: Some(status_cb.clone()), + } + .spawn(cancel_rx); + return; + } + + // WebSocket source streams forward incoming messages into the stream bus. + if let Some(url) = resolve_websocket_url( + &state.stream_manager, + &state.source_configs, + stream_name, + rule_id, + ) { + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + WebSocketSource { url, tx: stream_tx }.spawn(cancel_rx); + return; + } + + // Redis subscription streams forward channel messages into the stream bus. + if let Some((url, channel)) = resolve_redissub_source( + &state.stream_manager, + &state.source_configs, + stream_name, + rule_id, + ) { + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + RedisSubSource { + url, + channel, + tx: stream_tx, + } + .spawn(cancel_rx); + return; + } + + // Kafka source streams consume a topic partition into the stream bus. + if let Some(config) = resolve_kafka_source( + &state.stream_manager, + &state.source_configs, + stream_name, + rule_id, + ) { + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + KafkaSource { + config, + tx: stream_tx, + status_callback: Some(status_cb.clone()), + } + .spawn(cancel_rx); + return; + } + + // SQL source streams poll a database table into the stream bus. + if let Some(config) = resolve_sql_source( + &state.stream_manager, + &state.source_configs, + stream_name, + rule_id, + ) { + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + SqlSource::new(config, stream_tx).spawn(cancel_rx); + return; + } + + // RabbitMQ source streams subscribe to an AMQP broker into the stream bus. + if let Some(config) = + resolve_rabbitmq_source(&state.stream_manager, &state.source_configs, stream_name) + { + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + RabbitMqSource::new(config, stream_tx).spawn(cancel_rx); + return; + } + + // ZeroMQ source streams subscribe to a ZeroMQ endpoint into the stream bus. + if let Some(config) = + resolve_zmq_source(&state.stream_manager, &state.source_configs, stream_name) + { + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + ZmqSource::new(config, stream_tx).spawn(cancel_rx); + return; + } + + // Video source streams extract camera frames via FFmpeg into the stream bus. + if let Some(config) = + resolve_video_source(&state.stream_manager, &state.source_configs, stream_name) + { + if let Err(e) = config.validate() { + tracing::error!( + "[RULE {}] video source validation failed for stream '{}': {}", + rule_id, + stream_name, + e + ); + if let Some(handle) = state.rule_manager.rule_status_handle(rule_id) { + let mut status = handle.write(); + status.status = "error".to_string(); + status.message = format!("Video source validation error: {}", e); + status.last_exception = e; + } + return; + } + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + VideoSource::new(config, stream_tx).spawn(cancel_rx); + return; + } + + // Modbus source streams poll industrial PLCs over TCP or RTU-over-TCP into the stream bus. + if let Some(config) = + resolve_modbus_source(&state.stream_manager, &state.source_configs, stream_name) + { + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + ModbusSource::new(config, stream_tx) + .with_status_callback(Some(status_cb.clone())) + .spawn(cancel_rx); + return; + } + + // OPC-UA source streams poll OPC-UA servers over opc.tcp into the stream bus. + if let Some(config) = + resolve_opcua_source(&state.stream_manager, &state.source_configs, stream_name) + { + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + OpcUaSource::new(config, stream_tx) + .with_status_callback(Some(status_cb)) + .spawn(cancel_rx); + return; + } + + // OTLP source streams receive or poll OpenTelemetry metrics/logs/traces into the stream bus. + if let Some(config) = + resolve_otlp_source(&state.stream_manager, &state.source_configs, stream_name) + { + let stream_tx = state.stream_bus.get_or_create(stream_name); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + rekuiper_connectors::OtlpSource::new(config, stream_tx).spawn(cancel_rx); + return; + } + + // Simulator / Random source streams replay configured data into the stream bus. + if let Some(conf) = resolve_simulator_source( + &state.stream_manager, + &state.source_configs, + stream_name, + rule_id, + ) { + if !conf.data.is_empty() { + let stream_name_str = stream_name.to_string(); + let bus = state.stream_bus.clone(); + let (cancel_tx, mut cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, stream_name, rule_id, cancel_tx); + tokio::spawn(async move { + let (tx, mut rx) = tokio::sync::mpsc::channel::(1024); + let sim_handle = + tokio::spawn(async move { SimulatorSource::new(conf).run(tx).await }); + loop { + tokio::select! { + record = rx.recv() => { + match record { + Some(record) => { + let _ = bus.publish_async(&stream_name_str, record).await; + } + None => break, + } + } + changed = cancel_rx.changed() => { + if changed.is_err() || *cancel_rx.borrow() { + sim_handle.abort(); + break; + } + } + } + } + let _ = sim_handle.await; + }); + } + } +} + +/// Start source producers for a rule: its FROM source plus every joined +/// source (streams fan in messages; table targets bootstrap their source listeners). +pub(crate) fn bootstrap_rule_sources(state: &AppState, rule_id: &str, select_stmt: &SelectStmt) { + let needs_meta = stmt_called_functions(select_stmt) + .iter() + .any(|f| f.eq_ignore_ascii_case("meta") || f.eq_ignore_ascii_case("mqtt")); + let (has_wildcard, idents) = stmt_referenced_fields(select_stmt); + let mask_for_stream = |stream_name: &str| { + let ts_field = state.stream_manager.get_stream(stream_name).and_then(|s| { + s.options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("TIMESTAMP")) + .map(|(_, v)| v.clone()) + }); + extract_required_fields_from_expr_names(has_wildcard, &idents, ts_field.as_deref()) + }; + if state.table_manager.get_table(&select_stmt.from).is_some() { + bootstrap_table_source(state, &select_stmt.from); + } else { + bootstrap_stream_sources( + state, + rule_id, + &select_stmt.from, + needs_meta, + mask_for_stream(&select_stmt.from), + ); + } + for join in &select_stmt.joins { + if state.table_manager.get_table(&join.target).is_some() { + bootstrap_table_source(state, &join.target); + } else if join.target != select_stmt.from { + bootstrap_stream_sources( + state, + rule_id, + &join.target, + needs_meta, + mask_for_stream(&join.target), + ); + } + } +} + +pub(crate) fn bootstrap_graph_sources( + state: &AppState, + rule_id: &str, + graph: &rekuiper_core::model::GraphDefinition, +) { + for node in graph.nodes.values() { + if node.node_category == "source" { + let stream_name = node + .props + .get("datasource") + .or_else(|| node.props.get("sourceName")) + .or_else(|| node.props.get("topic")) + .and_then(|v| v.as_str()); + let src = match stream_name { + Some(s) if !s.trim().is_empty() => s.trim(), + _ => continue, + }; + + if state.stream_manager.get_stream(src).is_some() { + bootstrap_stream_sources(state, rule_id, src, true, None); + } else { + let n_type = node.node_type.to_lowercase(); + let comms = state.comms_hub.clone(); + let status_cb: rekuiper_connectors::ConnectorStatusCallback = + Arc::new(move |id, kind, status, err| { + comms.publish_connector_status( + id, + kind, + status, + err.map(|s| s.to_string()), + ); + }); + let stream_tx = state.stream_bus.get_or_create(src); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + register_rule_stream_source(state, src, rule_id, cancel_tx); + + if n_type == "mqtt" || n_type == "source" { + let mut config = MqttConfig::default(); + if let Some(srv) = node.props.get("server").and_then(|v| v.as_str()) { + config.server = srv.to_string(); + } + if let Some(topic) = node + .props + .get("topic") + .or_else(|| node.props.get("datasource")) + .and_then(|v| v.as_str()) + { + config.topic = topic.to_string(); + } + if let Some(u) = node.props.get("username").and_then(|v| v.as_str()) { + config.username = Some(u.to_string()); + } + if let Some(p) = node.props.get("password").and_then(|v| v.as_str()) { + config.password = Some(p.to_string()); + } + if let Some(qos) = node.props.get("qos").and_then(|v| v.as_u64()) { + config.qos = qos as u8; + } + config.attach_meta = true; + MqttSource::new(config, stream_tx) + .with_rule_counters(state.rule_manager.rule_counters(rule_id)) + .with_status_callback(Some(status_cb)) + .spawn(cancel_rx); + } else if n_type == "modbus" { + let conf_val = + serde_json::Value::Object(node.props.clone().into_iter().collect()); + if let Ok(config) = + serde_json::from_value::(conf_val) + { + ModbusSource::new(config, stream_tx) + .with_status_callback(Some(status_cb)) + .spawn(cancel_rx); + } + } else if n_type == "opcua" { + let conf_val = + serde_json::Value::Object(node.props.clone().into_iter().collect()); + if let Ok(config) = + serde_json::from_value::(conf_val) + { + OpcUaSource::new(config, stream_tx) + .with_status_callback(Some(status_cb)) + .spawn(cancel_rx); + } + } else { + bootstrap_stream_sources(state, rule_id, src, true, None); + } + } + } + } +} + +/// Respawns execution tasks (plus source producers) for every rule whose +/// persisted status is `running`, so a restarted daemon resumes processing +/// without manual intervention. +pub async fn restore_running_rules(state: &AppState) { + for table_name in state.table_manager.list_tables() { + bootstrap_table_source(state, &table_name); + } + for rule in state.rule_manager.list_rules() { + let running = state + .rule_manager + .get_rule_status(&rule.id) + .is_some_and(|s| s.status == "running"); + if !running { + continue; + } + if rule.sql.trim().is_empty() { + if let Some(ref graph) = rule.graph { + let is_native_dag = + graph.nodes.values().any(|n| { + matches!(n.node_type.as_str(), "switch" | "rate_limit" | "watchdog") + }) || graph + .topo + .edges + .values() + .any(|e| matches!(e, rekuiper_core::model::EdgeTargets::Multi(_))); + if is_native_dag { + let buffer_len = rule + .options + .as_ref() + .and_then(|o| o.get("bufferLength")) + .and_then(|v| v.as_u64()) + .map(|n| n as usize) + .unwrap_or(1024); + spawn_graph_rule_task( + &state.rule_manager, + &state.stream_bus, + rule.id.clone(), + graph.clone(), + state.comms_hub.clone(), + buffer_len, + ); + bootstrap_graph_sources(state, &rule.id, graph); + tracing::info!("Restored running graph rule {}", rule.id); + continue; + } + } + } + let mut parser = Parser::new(&rule.sql); + let select_stmt = match parser.parse_select() { + Ok(stmt) => stmt, + Err(e) => { + tracing::warn!("Skipping restore of rule {}: invalid SQL: {}", rule.id, e); + continue; + } + }; + spawn_rule_task( + &state.rule_manager, + &state.stream_bus, + &state.stream_manager, + &state.table_manager, + &state.source_configs, + &state.http_client, + &state.trace_manager, + &state.config, + rule.id.clone(), + select_stmt.clone(), + rule.actions.clone(), + rule.options.clone(), + ); + bootstrap_rule_sources(state, &rule.id, &select_stmt); + tracing::info!("Restored running rule {}", rule.id); + } +} + +/// Resolve the HTTP pull configuration for a rule whose source stream +/// declares `TYPE="httppull"` (or `"http_pull"`). +/// +/// Looks up `httppull/{conf_key}` then `http_pull/{conf_key}` in the stored +/// source configs; when no key matches, falls back to the stream's +/// `DATASOURCE` property as the poll URL with default method/interval. +pub(crate) fn resolve_httppull_config( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, + rule_id: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + let kind = def.options.get("TYPE")?; + if !(kind.eq_ignore_ascii_case("httppull") || kind.eq_ignore_ascii_case("http_pull")) { + return None; + } + if let Some(key) = def.options.get("CONF_KEY") { + if !key.is_empty() { + for prefix in ["httppull", "http_pull"] { + let lookup = format!("{}/{}", prefix, key); + if let Some(conf_val) = source_configs.read().get(&lookup).cloned() { + match serde_json::from_value::(conf_val) { + Ok(conf) => return Some(conf), + Err(e) => { + tracing::warn!( + "[RULE {}] invalid http pull config '{}': {}", + rule_id, + lookup, + e + ); + return None; + } + } + } + } + } + } + let url = def.options.get("DATASOURCE").cloned().unwrap_or_default(); + if url.is_empty() { + tracing::warn!( + "[RULE {}] http pull stream '{}' has neither CONF_KEY config nor DATASOURCE url", + rule_id, + stream_name + ); + return None; + } + Some(HttpPullConfig { + url, + method: "get".to_string(), + interval: 1000, + headers: HashMap::new(), + body: None, + }) +} + +/// Resolve the WebSocket URL for a rule whose source stream declares +/// `TYPE="websocket"`. +/// +/// Looks up `websocket/{conf_key}` in the stored source configs and builds +/// the URL from it; otherwise parses `DATASOURCE` (used directly when it +/// starts with `ws://` or `wss://`, else treated as a path on the default +/// local endpoint). +pub(crate) fn resolve_websocket_url( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, + rule_id: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + let kind = def.options.get("TYPE")?; + if !kind.eq_ignore_ascii_case("websocket") { + return None; + } + if let Some(key) = def.options.get("CONF_KEY") { + if !key.is_empty() { + let lookup = format!("websocket/{}", key); + if let Some(conf_val) = source_configs.read().get(&lookup).cloned() { + match serde_json::from_value::(conf_val) { + Ok(conf) => return Some(conf.target_url()), + Err(e) => { + tracing::warn!( + "[RULE {}] invalid websocket config '{}': {}", + rule_id, + lookup, + e + ); + return None; + } + } + } + } + } + let datasource = def.options.get("DATASOURCE").cloned().unwrap_or_default(); + if datasource.is_empty() { + tracing::warn!( + "[RULE {}] websocket stream '{}' has neither CONF_KEY config nor DATASOURCE", + rule_id, + stream_name + ); + return None; + } + let ds = datasource.trim(); + if ds.starts_with("ws://") || ds.starts_with("wss://") { + Some(ds.to_string()) + } else { + Some(format!( + "ws://127.0.0.1:8080/{}", + ds.trim_start_matches('/') + )) + } +} + +/// Resolve a Redis server address from a stored source config value, which +/// may be a full object (`{"addr": ...}`) or a bare address string. +/// Falls back to the local default when absent or unparseable. +pub(crate) fn resolve_redis_addr( + source_configs: &Arc>>, + conf_key: &str, +) -> String { + const DEFAULT: &str = "127.0.0.1:6379"; + if conf_key.is_empty() { + return DEFAULT.to_string(); + } + let stored = source_configs + .read() + .get(&format!("redis/{}", conf_key)) + .cloned(); + match stored { + Some(Value::Object(map)) => map + .get("addr") + .or_else(|| map.get("address")) + .or_else(|| map.get("url")) + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .unwrap_or(DEFAULT) + .to_string(), + Some(Value::String(s)) if !s.is_empty() => s, + _ => DEFAULT.to_string(), + } +} + +pub(crate) fn resolve_redissub_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, + rule_id: &str, +) -> Option<(String, String)> { + let def = stream_manager.get_stream(stream_name)?; + let kind = def.options.get("TYPE")?; + if !(kind.eq_ignore_ascii_case("redissub") || kind.eq_ignore_ascii_case("redis_sub")) { + return None; + } + let channel = def.options.get("DATASOURCE").cloned().unwrap_or_default(); + if channel.is_empty() { + tracing::warn!( + "[RULE {}] redissub stream '{}' has no DATASOURCE channel", + rule_id, + stream_name + ); + return None; + } + let conf_key = def.options.get("CONF_KEY").cloned().unwrap_or_default(); + let addr = resolve_redis_addr(source_configs, &conf_key); + let url = if addr.contains("://") { + addr + } else { + format!("redis://{}", addr) + }; + Some((url, channel)) +} + +/// Resolve the [`KafkaConfig`] for a rule whose source stream declares +/// `TYPE="kafka"`: the topic comes from the stream `DATASOURCE` (falling back +/// to the config topic), brokers and friends from `kafka/{conf_key}` (or +/// defaults when no key matches). +pub(crate) fn resolve_kafka_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, + rule_id: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + let kind = def.options.get("TYPE")?; + if !kind.eq_ignore_ascii_case("kafka") { + return None; + } + let mut config = KafkaConfig { + brokers: "127.0.0.1:9092".to_string(), + topic: None, + group_id: None, + partition: 0, + key: None, + offset_reset: "latest".to_string(), + }; + if let Some(key) = def.options.get("CONF_KEY") { + if !key.is_empty() { + let lookup = format!("kafka/{}", key); + if let Some(conf_val) = source_configs.read().get(&lookup).cloned() { + match serde_json::from_value::(conf_val) { + Ok(parsed) => config = parsed, + Err(e) => { + tracing::warn!( + "[RULE {}] invalid kafka config '{}': {}", + rule_id, + lookup, + e + ); + return None; + } + } + } + } + } + if let Some(reset) = def + .options + .get("OFFSET_RESET") + .or_else(|| def.options.get("OFFSETRESET")) + { + if !reset.trim().is_empty() { + let r = reset.trim().to_ascii_lowercase(); + if r == "earliest" || r == "latest" { + config.offset_reset = r; + } else { + tracing::warn!( + "[RULE {}] invalid kafka offset_reset '{}' (must be 'earliest' or 'latest')", + rule_id, + reset + ); + return None; + } + } + } + if !config.offset_reset.eq_ignore_ascii_case("earliest") + && !config.offset_reset.eq_ignore_ascii_case("latest") + { + tracing::warn!( + "[RULE {}] invalid kafka offset_reset '{}' (must be 'earliest' or 'latest')", + rule_id, + config.offset_reset + ); + return None; + } + match def.options.get("DATASOURCE").cloned() { + Some(topic) if !topic.is_empty() => config.topic = Some(topic), + _ if config.topic.as_deref().is_some_and(|t| !t.is_empty()) => {} + _ => { + tracing::warn!( + "[RULE {}] kafka stream '{}' has no DATASOURCE topic", + rule_id, + stream_name + ); + return None; + } + } + if config.broker_list().is_empty() { + tracing::warn!( + "[RULE {}] kafka stream '{}' has no brokers configured", + rule_id, + stream_name + ); + return None; + } + Some(config) +} + +/// Resolve the [`FileSourceConfig`] for a rule whose source stream declares +/// `TYPE="file"`: the path comes from `DATASOURCE` (falling back to a stored +/// `file/{conf_key}` config), with `FORMAT`/`fileType`, `hasHeader` and +/// `delimiter` layered from stream options over config defaults. +pub(crate) fn resolve_file_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, + rule_id: &str, +) -> Option { + use rekuiper_connectors::DelimitedCodec; + + let def = stream_manager.get_stream(stream_name)?; + let kind = def.options.get("TYPE")?; + if !kind.eq_ignore_ascii_case("file") { + return None; + } + let mut config = FileSourceConfig { + path: String::new(), + format: "json".to_string(), + has_header: false, + delimiter: None, + interval: 0, + }; + if let Some(key) = def.options.get("CONF_KEY") { + if !key.is_empty() { + let lookup = format!("file/{}", key); + if let Some(conf_val) = source_configs.read().get(&lookup).cloned() { + match serde_json::from_value::(conf_val) { + Ok(parsed) => config = parsed, + Err(e) => { + tracing::warn!( + "[RULE {}] invalid file config '{}': {}", + rule_id, + lookup, + e + ); + return None; + } + } + } + } + } + if let Some(path) = def.options.get("DATASOURCE") { + if !path.trim().is_empty() { + config.path = path.clone(); + } + } + if config.path.trim().is_empty() { + tracing::warn!( + "[RULE {}] file stream '{}' has neither DATASOURCE path nor file config", + rule_id, + stream_name + ); + return None; + } + if let Some(format) = def + .options + .get("FORMAT") + .filter(|s| !s.trim().is_empty()) + .or_else(|| def.options.get("FILETYPE").filter(|s| !s.trim().is_empty())) + { + config.format = format.clone(); + } + if let Some(flag) = def + .options + .get("HASHEADER") + .or_else(|| def.options.get("HAS_HEADER")) + { + match flag.trim().to_ascii_lowercase().as_str() { + "true" | "1" | "yes" => config.has_header = true, + "false" | "0" | "no" | "" => config.has_header = false, + _ => {} + } + } + if let Some(delim) = def.options.get("DELIMITER").filter(|s| !s.is_empty()) { + config.delimiter = Some(DelimitedCodec::delimiter_from_name(delim)); + } + Some(config) +} + +/// Signal cancellation to a rule's background streaming sources. +/// Streams shared with other running rules remain active; when the last +/// rule consuming a stream stops, the stream's background source task +/// is cancelled and cleanly disconnected. +pub(crate) fn cancel_rule_source(state: &AppState, rule_id: &str) { + if let Some(txs) = state.source_cancels.write().remove(rule_id) { + for tx in txs { + let _ = tx.send(true); + } + } + let streams = state.rule_streams.write().remove(rule_id); + if let Some(streams) = streams { + let mut active_rules = state.stream_active_rules.write(); + let mut source_cancels = state.stream_source_cancels.write(); + let mut attach_meta = state.stream_attach_meta.write(); + for stream in streams { + let mut should_cleanup = false; + if let Some(rules) = active_rules.get_mut(&stream) { + rules.remove(rule_id); + should_cleanup = rules.is_empty(); + } + if should_cleanup { + active_rules.remove(&stream); + attach_meta.remove(&stream); + state.stream_field_masks.write().remove(&stream); + state.stream_rule_field_masks.write().remove(&stream); + if let Some(cancels) = source_cancels.remove(&stream) { + for tx in cancels { + let _ = tx.send(true); + } + } + state + .http_push_endpoints + .write() + .retain(|_, ep| ep.stream_name != stream); + } else { + let recomputed = { + let mut rule_masks = state.stream_rule_field_masks.write(); + if let Some(rm) = rule_masks.get_mut(&stream) { + rm.remove(rule_id); + compute_union_field_mask(rm) + } else { + None + } + }; + if let Some(flag) = state.stream_field_masks.read().get(&stream) { + *flag.write() = recomputed; + } + } + } + } +} + +/// Resolve the bus topic a rule actually subscribes to: memory-type streams +/// (`TYPE="memory"`) re-export another topic via `DATASOURCE`, mirroring the +/// eKuiper memory source (used to chain rules through memory sinks). +pub(crate) fn resolve_source_topic(stream_manager: &StreamManager, stream_name: &str) -> String { + if let Some(def) = stream_manager.get_stream(stream_name) { + if def + .options + .get("TYPE") + .is_some_and(|t| t.eq_ignore_ascii_case("memory")) + { + if let Some(ds) = def.options.get("DATASOURCE") { + if !ds.is_empty() { + return ds.clone(); + } + } + } + } + stream_name.to_string() +} + +/// Resolve the polling config for a `TYPE="sql"` source stream: a matching +/// `sql/{conf_key}` source config wins, otherwise the stream options (`URL`, +/// falling back to `DATASOURCE`, plus `TABLE` or the stream name and an +/// optional `INTERVAL` poll period). Option names match case-insensitively. +pub(crate) fn resolve_sql_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, + rule_id: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + let kind = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("TYPE")) + .map(|(_, v)| v.clone()) + .unwrap_or_default(); + if !kind.eq_ignore_ascii_case("sql") { + return None; + } + let conf_key = def + .options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("CONF_KEY") + || k.eq_ignore_ascii_case("confKey") + || k.eq_ignore_ascii_case("connectionSelector") + }) + .map(|(_, v)| v.trim().to_string()); + if let Some(key) = conf_key { + if !key.is_empty() { + let lookup = format!("sql/{}", key); + let configs_guard = source_configs.read(); + let conf_val = configs_guard + .get(&lookup) + .or_else(|| configs_guard.get(&key)) + .or_else(|| configs_guard.get(&format!("connections/{}", key))) + .cloned(); + drop(configs_guard); + if let Some(val) = conf_val { + match serde_json::from_value::(val) { + Ok(mut conf) => { + // Documented plugin configs carry the URL as `dburl` + // and may omit the table: fall back to the stream + // options (DATASOURCE/TABLE) for whatever is missing. + if conf.url.trim().is_empty() { + conf.url = def + .options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("URL") + || k.eq_ignore_ascii_case("DATASOURCE") + }) + .map(|(_, v)| v.clone()) + .unwrap_or_default(); + } + if conf.table.trim().is_empty() { + conf.table = sql_table_fallback(&def.options, stream_name); + } + if conf.url.trim().is_empty() || conf.table.trim().is_empty() { + // A template-SQL config without any table anchor + // cannot poll; surface the misconfiguration. + tracing::warn!( + "[RULE {}] sql source config '{}' has no usable url/table", + rule_id, + lookup + ); + return None; + } + return Some(conf); + } + Err(e) => { + tracing::warn!( + "[RULE {}] invalid sql source config '{}': {}", + rule_id, + lookup, + e + ); + return None; + } + } + } + } + } + let url = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("URL") || k.eq_ignore_ascii_case("DATASOURCE")) + .map(|(_, v)| v.clone()) + .unwrap_or_default(); + if url.trim().is_empty() { + return None; + } + let table = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("TABLE")) + .map(|(_, v)| v.clone()) + .unwrap_or_else(|| stream_name.to_string()); + let interval = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("INTERVAL")) + .and_then(|(_, v)| v.trim().parse::().ok()) + .unwrap_or(1000) + .max(1); + Some(SqlConnectorConfig { + url, + table, + fields: Vec::new(), + interval, + template_sql_query_cfg: None, + internal_sql_query_cfg: None, + }) +} + +pub(crate) fn resolve_rabbitmq_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + let is_rmq = def + .options + .get("TYPE") + .is_some_and(|t| t.eq_ignore_ascii_case("rabbitmq") || t.eq_ignore_ascii_case("amqp")); + if !is_rmq { + return None; + } + let mut config = RabbitMqConfig::default(); + let conf_key = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("CONF_KEY") || k.eq_ignore_ascii_case("confKey")) + .map(|(_, v)| v.trim()); + if let Some(key) = conf_key { + if !key.is_empty() { + let lookup = format!("rabbitmq/{}", key); + let configs_guard = source_configs.read(); + let conf_val = configs_guard + .get(&lookup) + .or_else(|| configs_guard.get(key)) + .cloned(); + drop(configs_guard); + if let Some(val) = conf_val { + if let Ok(c) = serde_json::from_value::(val) { + config = c; + } + } + } + } + if let Some(srv) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("SERVER") || k.eq_ignore_ascii_case("URL")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.server = srv.to_string(); + } + if let Some(q) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("QUEUE") || k.eq_ignore_ascii_case("DATASOURCE")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.queue = q.to_string(); + } + if let Some(ex) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("EXCHANGE")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.exchange = ex.to_string(); + } + if let Some(rk) = def + .options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("ROUTINGKEY") || k.eq_ignore_ascii_case("ROUTING_KEY") + }) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.routing_key = rk.to_string(); + } + if let Some(u) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("USERNAME") || k.eq_ignore_ascii_case("USER")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.username = Some(u.to_string()); + } + if let Some(p) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("PASSWORD")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.password = Some(p.to_string()); + } + Some(config) +} + +pub(crate) fn resolve_zmq_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + let is_zmq = def + .options + .get("TYPE") + .is_some_and(|t| t.eq_ignore_ascii_case("zmq") || t.eq_ignore_ascii_case("zeromq")); + if !is_zmq { + return None; + } + let mut config = ZmqSourceConfig::default(); + let conf_key = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("CONF_KEY") || k.eq_ignore_ascii_case("confKey")) + .map(|(_, v)| v.trim()); + if let Some(key) = conf_key { + if !key.is_empty() { + let lookup = format!("zmq/{}", key); + let configs_guard = source_configs.read(); + let conf_val = configs_guard + .get(&lookup) + .or_else(|| configs_guard.get(key)) + .cloned(); + drop(configs_guard); + if let Some(val) = conf_val { + if let Ok(c) = serde_json::from_value::(val) { + config = c; + } + } + } + } + if let Some(srv) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("SERVER") || k.eq_ignore_ascii_case("URL")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.server = srv.to_string(); + } + if let Some(topic) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("DATASOURCE") || k.eq_ignore_ascii_case("TOPIC")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.topic = Some(topic.to_string()); + } + if let Some(mode) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("MODE")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.mode = mode.to_string(); + } + Some(config) +} + +pub(crate) fn resolve_video_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + let is_video = def + .options + .get("TYPE") + .is_some_and(|t| t.eq_ignore_ascii_case("video")); + if !is_video { + return None; + } + let mut config = VideoSourceConfig::default(); + let conf_key = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("CONF_KEY") || k.eq_ignore_ascii_case("confKey")) + .map(|(_, v)| v.trim()); + if let Some(key) = conf_key { + if !key.is_empty() { + let lookup = format!("video/{}", key); + let configs_guard = source_configs.read(); + let conf_val = configs_guard + .get(&lookup) + .or_else(|| configs_guard.get(key)) + .cloned(); + drop(configs_guard); + if let Some(val) = conf_val { + if let Ok(c) = serde_json::from_value::(val) { + config = c; + } + } + } + } + if let Some(url) = def + .options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("URL") + || k.eq_ignore_ascii_case("SERVER") + || k.eq_ignore_ascii_case("DATASOURCE") + }) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.url = url.to_string(); + } + if let Some(interval) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("INTERVAL")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.set_interval_str(interval); + } + if let Some(codec) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("CODEC")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.codec = codec.to_string(); + } + if let Some(debug) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("DEBUG_RESP") || k.eq_ignore_ascii_case("DEBUGRESP")) + .map(|(_, v)| v.trim()) + { + config.debug_resp = debug.eq_ignore_ascii_case("true"); + } + if let Some(ffmpeg) = def + .options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("FFMPEG_PATH") || k.eq_ignore_ascii_case("FFMPEGPATH") + }) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.ffmpeg_path = ffmpeg.to_string(); + } + Some(config) +} + +pub(crate) fn resolve_simulator_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, + rule_id: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + let is_sim = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("TYPE")) + .map(|(_, v)| v.trim()) + .is_some_and(|t| t.eq_ignore_ascii_case("simulator") || t.eq_ignore_ascii_case("random")); + if !is_sim { + return None; + } + let mut config = SimulatorConfig { + data: Vec::new(), + interval: Value::from(10), + loop_data: true, + }; + let conf_key = def + .options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("CONF_KEY") + || k.eq_ignore_ascii_case("confKey") + || k.eq_ignore_ascii_case("connectionSelector") + }) + .map(|(_, v)| v.trim()); + let key = conf_key.unwrap_or("default"); + if !key.is_empty() { + let lookup = format!("simulator/{}", key); + let lookup_random = format!("random/{}", key); + let configs_guard = source_configs.read(); + let conf_val = configs_guard + .get(&lookup) + .or_else(|| configs_guard.get(&lookup_random)) + .or_else(|| configs_guard.get(key)) + .cloned(); + drop(configs_guard); + if let Some(val) = conf_val { + match serde_json::from_value::(val) { + Ok(c) => config = c, + Err(e) => { + tracing::warn!( + "[RULE {}] invalid simulator config '{}': {}", + rule_id, + lookup, + e + ); + } + } + } + } + if let Some(interval) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("INTERVAL")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + if let Ok(num) = interval.parse::() { + config.interval = Value::from(num); + } else { + config.interval = Value::String(interval.to_string()); + } + } + if let Some(loop_opt) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("LOOP")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + match loop_opt.to_ascii_lowercase().as_str() { + "true" | "1" | "yes" => config.loop_data = true, + "false" | "0" | "no" => config.loop_data = false, + _ => {} + } + } + if let Some(data_str) = def + .options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("DATA")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + if let Ok(parsed_data) = serde_json::from_str::>>(data_str) { + config.data = parsed_data; + } else if let Ok(single) = serde_json::from_str::>(data_str) { + config.data = vec![single]; + } + } + Some(config) +} + +pub(crate) fn resolve_modbus_options( + options: &HashMap, + source_configs: &Arc>>, +) -> Option { + let is_modbus = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("TYPE")) + .map(|(_, v)| v.trim()) + .is_some_and(|t| t.eq_ignore_ascii_case("modbus")); + if !is_modbus { + return None; + } + let mut config = rekuiper_connectors::ModbusConfig::default(); + if let Some(key) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("CONF_KEY") || k.eq_ignore_ascii_case("confKey")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + let lookup = format!("modbus/{}", key); + let lookup_src = format!("sources/modbus/{}", key); + let guard = source_configs.read(); + let conf_val = guard + .get(&lookup) + .or_else(|| guard.get(&lookup_src)) + .or_else(|| guard.get(key)) + .cloned(); + drop(guard); + if let Some(val) = conf_val { + if let Ok(c) = serde_json::from_value::(val) { + config = c; + } + } + } + if let Some(ep) = options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("ENDPOINT") + || k.eq_ignore_ascii_case("URL") + || k.eq_ignore_ascii_case("SERVER") + || k.eq_ignore_ascii_case("DATASOURCE") + }) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.endpoint = ep.to_string(); + } + if let Some(tp) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("TRANSPORT")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.transport = tp.to_string(); + } + if let Some(u) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("UNIT_ID") || k.eq_ignore_ascii_case("SLAVE_ID")) + .and_then(|(_, v)| v.trim().parse::().ok()) + { + config.unit_id = u; + } + if let Some(iv) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("INTERVAL")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.interval_ms = + rekuiper_connectors::simulator::parse_interval_ms(&Value::String(iv.to_string())) + .as_millis() + .max(1) as u64; + } + if let Some(to) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("TIMEOUT")) + .and_then(|(_, v)| v.trim().parse::().ok()) + { + config.timeout_ms = to; + } + if let Some(ab) = options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("ADDRESS_BASE") || k.eq_ignore_ascii_case("ADDRESSBASE") + }) + .and_then(|(_, v)| v.trim().parse::().ok()) + { + config.address_base = Some(ab); + } + if let Some(plc) = options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("PLC_ADDRESSING") || k.eq_ignore_ascii_case("PLCADDRESSING") + }) + .map(|(_, v)| v.trim().eq_ignore_ascii_case("true") || v.trim() == "1") + { + config.plc_addressing = plc; + } + if let Some(mg) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("MAX_GAP") || k.eq_ignore_ascii_case("MAXGAP")) + .and_then(|(_, v)| v.trim().parse::().ok()) + { + config.max_gap = mg; + } + if let Some(regs) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("REGISTERS")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + if let Ok(r) = serde_json::from_str::>(regs) { + config.registers = r; + } + } + Some(config) +} + +pub(crate) fn resolve_modbus_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + resolve_modbus_options(&def.options, source_configs) +} + +pub(crate) fn resolve_modbus_table_config( + table_manager: &TableManager, + source_configs: &Arc>>, + table_name: &str, +) -> Option { + let def = table_manager.get_table(table_name)?; + resolve_modbus_options(&def.options, source_configs) +} + +pub(crate) fn resolve_opcua_options( + options: &HashMap, + source_configs: &Arc>>, +) -> Option { + let is_opcua = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("TYPE")) + .map(|(_, v)| v.trim()) + .is_some_and(|t| { + t.eq_ignore_ascii_case("opcua") + || t.eq_ignore_ascii_case("opc_ua") + || t.eq_ignore_ascii_case("opc-ua") + }); + if !is_opcua { + return None; + } + let mut config = rekuiper_connectors::OpcUaConfig::default(); + if let Some(key) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("CONF_KEY") || k.eq_ignore_ascii_case("confKey")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + let lookup = format!("opcua/{}", key); + let lookup_src = format!("sources/opcua/{}", key); + let guard = source_configs.read(); + let conf_val = guard + .get(&lookup) + .or_else(|| guard.get(&lookup_src)) + .or_else(|| guard.get(key)) + .cloned(); + drop(guard); + if let Some(val) = conf_val { + if let Ok(c) = serde_json::from_value::(val) { + config = c; + } + } + } + if let Some(ep) = options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("ENDPOINT") + || k.eq_ignore_ascii_case("URL") + || k.eq_ignore_ascii_case("SERVER") + || k.eq_ignore_ascii_case("DATASOURCE") + }) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.endpoint = ep.to_string(); + } + if let Some(iv) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("INTERVAL")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.interval_ms = + rekuiper_connectors::simulator::parse_interval_ms(&Value::String(iv.to_string())) + .as_millis() + .max(1) as u64; + } + if let Some(to) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("TIMEOUT")) + .and_then(|(_, v)| v.trim().parse::().ok()) + { + config.timeout_ms = to; + } + if let Some(sp) = options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("SECURITY_POLICY") || k.eq_ignore_ascii_case("SECURITYPOLICY") + }) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + config.security_policy = sp.to_string(); + } + if let Some(u) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("USERNAME") || k.eq_ignore_ascii_case("USER")) + .map(|(_, v)| v.trim().to_string()) + .filter(|s| !s.is_empty()) + { + config.username = Some(u); + } + if let Some(p) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("PASSWORD") || k.eq_ignore_ascii_case("PASS")) + .map(|(_, v)| v.trim().to_string()) + .filter(|s| !s.is_empty()) + { + config.password = Some(p); + } + if let Some(nodes) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("NODES")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + if let Ok(n) = serde_json::from_str::>(nodes) { + config.nodes = n; + } + } + Some(config) +} + +pub(crate) fn resolve_opcua_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + resolve_opcua_options(&def.options, source_configs) +} + +pub(crate) fn resolve_opcua_table_config( + table_manager: &TableManager, + source_configs: &Arc>>, + table_name: &str, +) -> Option { + let def = table_manager.get_table(table_name)?; + resolve_opcua_options(&def.options, source_configs) +} + +pub(crate) fn resolve_otlp_options( + options: &HashMap, + source_configs: &Arc>>, +) -> Option { + let is_otlp = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("TYPE")) + .map(|(_, v)| v.trim()) + .is_some_and(|t| { + t.eq_ignore_ascii_case("otlp") + || t.eq_ignore_ascii_case("opentelemetry") + || t.eq_ignore_ascii_case("otel") + }); + if !is_otlp { + return None; + } + let mut config = rekuiper_connectors::OtlpSourceConfig { + bind: None, + endpoint: None, + interval: None, + }; + if let Some(key) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("CONF_KEY") || k.eq_ignore_ascii_case("confKey")) + .map(|(_, v)| v.trim()) + .filter(|s| !s.is_empty()) + { + let lookup = format!("otlp/{}", key); + let lookup_src = format!("sources/otlp/{}", key); + let guard = source_configs.read(); + let conf_val = guard + .get(&lookup) + .or_else(|| guard.get(&lookup_src)) + .or_else(|| guard.get(key)) + .cloned(); + drop(guard); + if let Some(v) = conf_val { + if let Ok(c) = serde_json::from_value::(v) { + config = c; + } + } + } + if let Some(ep) = options + .iter() + .find(|(k, _)| { + k.eq_ignore_ascii_case("ENDPOINT") + || k.eq_ignore_ascii_case("BIND") + || k.eq_ignore_ascii_case("SERVER") + }) + .map(|(_, v)| v.trim().to_string()) + .filter(|s| !s.is_empty()) + { + config.bind = Some(ep.clone()); + config.endpoint = Some(ep); + } + if let Some(iv) = options + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("INTERVAL")) + .map(|(_, v)| v.trim().to_string()) + .filter(|s| !s.is_empty()) + { + config.interval = Some(iv); + } + Some(config) +} + +pub(crate) fn resolve_otlp_source( + stream_manager: &StreamManager, + source_configs: &Arc>>, + stream_name: &str, +) -> Option { + let def = stream_manager.get_stream(stream_name)?; + resolve_otlp_options(&def.options, source_configs) +} diff --git a/crates/rekuiper-server/src/engine/windows.rs b/crates/rekuiper-server/src/engine/windows.rs index d22de0e2..95b4dfb9 100644 --- a/crates/rekuiper-server/src/engine/windows.rs +++ b/crates/rekuiper-server/src/engine/windows.rs @@ -249,7 +249,7 @@ pub(crate) async fn run_count_window_rule( .state .write() .insert("__rule_start__".to_string(), Value::from(start_time_ms)); - let mut buffer: Vec = Vec::new(); + let mut buffer: std::collections::VecDeque = std::collections::VecDeque::new(); let mut events_since_trigger: usize = 0; let from_source = select_stmt.from.clone(); let mut join_rx = spawn_join_forwarders(join_rxs); @@ -274,7 +274,7 @@ pub(crate) async fn run_count_window_rule( if !window_ingest_passes(&select_stmt, &record.data) { continue; } - buffer.push(TaggedRow { source: from_source.clone(), data: record.data }); + buffer.push_back(TaggedRow { source: from_source.clone(), data: record.data }); events_since_trigger += 1; if hop <= count { // Standard count window (tumbling when hop == count, overlapping when hop < count) @@ -283,7 +283,7 @@ pub(crate) async fn run_count_window_rule( buffer.drain(0..count).collect() } else { // Overlapping: discard only the oldest `hop` records. - let batch = buffer[0..count].to_vec(); + let batch: Vec = buffer.iter().take(count).cloned().collect(); buffer.drain(0..hop.min(buffer.len())); batch }; @@ -293,12 +293,13 @@ pub(crate) async fn run_count_window_rule( } else { // Sparsely sampled count window with gap (hop > count) if buffer.len() > count { - buffer.remove(0); + buffer.pop_front(); } if events_since_trigger >= hop { if !buffer.is_empty() { let now_ms = chrono::Utc::now().timestamp_millis(); - emit_window_batch(&counters, &table_manager, &source_configs, &select_stmt, std::mem::take(&mut buffer), false, Some((now_ms, now_ms)), &sink, Some(&rule_state)).await; + let batch: Vec = buffer.drain(..).collect(); + emit_window_batch(&counters, &table_manager, &source_configs, &select_stmt, batch, false, Some((now_ms, now_ms)), &sink, Some(&rule_state)).await; } events_since_trigger = 0; buffer.clear(); @@ -325,14 +326,14 @@ pub(crate) async fn run_count_window_rule( if !window_ingest_passes(&select_stmt, &tagged.data) { continue; } - buffer.push(tagged); + buffer.push_back(tagged); events_since_trigger += 1; if hop <= count { if buffer.len() >= count { let batch: Vec = if hop == count { buffer.drain(0..count).collect() } else { - let batch = buffer[0..count].to_vec(); + let batch: Vec = buffer.iter().take(count).cloned().collect(); buffer.drain(0..hop.min(buffer.len())); batch }; @@ -341,11 +342,12 @@ pub(crate) async fn run_count_window_rule( } } else if events_since_trigger >= hop { if buffer.len() > count { - buffer.remove(0); + buffer.pop_front(); } if !buffer.is_empty() { let now_ms = chrono::Utc::now().timestamp_millis(); - emit_window_batch(&counters, &table_manager, &source_configs, &select_stmt, std::mem::take(&mut buffer), false, Some((now_ms, now_ms)), &sink, Some(&rule_state)).await; + let batch: Vec = buffer.drain(..).collect(); + emit_window_batch(&counters, &table_manager, &source_configs, &select_stmt, batch, false, Some((now_ms, now_ms)), &sink, Some(&rule_state)).await; } events_since_trigger = 0; buffer.clear(); diff --git a/crates/rekuiper-server/src/fleet.rs b/crates/rekuiper-server/src/fleet.rs index 00d7a0c7..13b77c4e 100644 --- a/crates/rekuiper-server/src/fleet.rs +++ b/crates/rekuiper-server/src/fleet.rs @@ -119,8 +119,11 @@ pub enum FleetServerCmd { ), } +static NEXT_CONN_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + #[derive(Clone)] pub struct ConnectedDeviceHandle { + pub conn_id: u64, pub cmd_tx: tokio::sync::mpsc::Sender, pub connected_at: i64, } @@ -163,6 +166,62 @@ impl FleetManager { mgr } + pub async fn load_from_kv(&self, kv: &Arc) -> Result<()> { + if let Ok(devs) = kv.list_all("fleet_devices").await { + let mut guard = self.devices.write(); + for (k, v) in devs { + if let Ok(mut dev) = serde_json::from_str::(&v) { + if dev.status == "online" { + dev.status = "offline".to_string(); + } + guard.insert(k, dev); + } + } + } + if let Ok(snaps) = kv.list_all("fleet_snapshots").await { + let mut guard = self.snapshots.write(); + for (k, v) in snaps { + if let Ok(snap) = serde_json::from_str::(&v) { + guard.insert(k, snap); + } + } + } + if let Ok(rollouts) = kv.list_all("fleet_rollouts").await { + let mut guard = self.rollouts.write(); + for (k, v) in rollouts { + if let Ok(ro) = serde_json::from_str::(&v) { + guard.insert(k, ro); + } + } + } + if let Ok(audits) = kv.list_all("fleet_audit").await { + let mut entries = Vec::new(); + for (_k, v) in audits { + if let Ok(entry) = serde_json::from_str::(&v) { + entries.push(entry); + } + } + entries.sort_by_key(|e| e.timestamp); + if entries.len() > 1000 { + let drain_count = entries.len() - 1000; + entries.drain(0..drain_count); + } + *self.audit_log.write() = entries; + } + Ok(()) + } + + fn persist_rollout(&self, rollout: &FleetRollout) { + if let Some(kv) = &self.kv { + let kv_clone = kv.clone(); + let key = rollout.rollout_id.clone(); + let json_str = serde_json::to_string(rollout).unwrap_or_default(); + tokio::spawn(async move { + let _ = kv_clone.set("fleet_rollouts", &key, &json_str).await; + }); + } + } + pub fn record_audit(&self, action: &str, actor: &str, target_id: &str, details: Value) { let entry = FleetAuditEntry { id: format!("aud_{}", uuid::Uuid::new_v4().simple()), @@ -383,6 +442,7 @@ impl FleetManager { self.rollouts .write() .insert(rollout_id.clone(), rollout.clone()); + self.persist_rollout(&rollout); self.record_audit( "rollout_started", "admin", @@ -454,6 +514,7 @@ impl FleetManager { r.status = RolloutStatus::RolledBack; r.message = halt_msg.clone(); r.updated_at = chrono::Utc::now().timestamp(); + self.persist_rollout(r); } } self.record_audit( @@ -514,6 +575,7 @@ impl FleetManager { r.message = "Staged rollout promoted to 100% production successfully.".to_string(); r.updated_at = chrono::Utc::now().timestamp(); + self.persist_rollout(r); } } self.record_audit( @@ -564,6 +626,7 @@ impl FleetManager { r.status = RolloutStatus::Completed; r.message = "Rollout completed successfully across entire fleet.".to_string(); r.updated_at = chrono::Utc::now().timestamp(); + self.persist_rollout(r); } } @@ -585,6 +648,7 @@ impl FleetManager { r.status = RolloutStatus::RolledBack; r.message = "Rollout halted & rolled back manually by administrator.".to_string(); r.updated_at = chrono::Utc::now().timestamp(); + self.persist_rollout(r); (r.canary_devices.clone(), r.production_devices.clone()) }; @@ -970,31 +1034,40 @@ pub async fn handle_fleet_ws_client(socket: AxumWebSocket, state: AppState) { let device_id = initial_hb.agent_id.clone(); state.fleet.register_or_update_heartbeat(initial_hb); - // Register active connection + // Register active connection with monotonic session ID + let conn_id = NEXT_CONN_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed); state.fleet.active_connections.write().insert( device_id.clone(), ConnectedDeviceHandle { + conn_id, cmd_tx: cmd_tx.clone(), connected_at: chrono::Utc::now().timestamp(), }, ); - state - .fleet - .record_audit("device_connected", "system", &device_id, json!({})); + state.fleet.record_audit( + "device_connected", + "system", + &device_id, + json!({"conn_id": conn_id}), + ); - // Stream 0 session loop: manages incoming telemetry frames and pending command dispatches + // Stream 0 session loop: manages incoming telemetry frames and pending command dispatches (FIFO queue) let fleet_clone_session = state.fleet.clone(); - type PendingReplySlot = - Arc>>>>; - let pending_reply: PendingReplySlot = Arc::new(tokio::sync::Mutex::new(None)); - - let pending_reply_reader = pending_reply.clone(); + type PendingReplyQueue = Arc< + tokio::sync::Mutex< + std::collections::VecDeque>>, + >, + >; + let pending_replies: PendingReplyQueue = + Arc::new(tokio::sync::Mutex::new(std::collections::VecDeque::new())); + + let pending_reply_reader = pending_replies.clone(); let reader_task = tokio::spawn(async move { while let Ok(bytes) = read_frame(&mut read_half).await { if let Ok(val) = serde_json::from_slice::(&bytes) { if val.get("agent_id").is_none() && val.get("status").is_some() { let mut guard = pending_reply_reader.lock().await; - if let Some(reply) = guard.take() { + if let Some(reply) = guard.pop_front() { let _ = reply.send(Ok(val)); continue; } @@ -1006,17 +1079,17 @@ pub async fn handle_fleet_ws_client(socket: AxumWebSocket, state: AppState) { } }); - let pending_reply_writer = pending_reply.clone(); + let pending_reply_writer = pending_replies.clone(); let write_task = tokio::spawn(async move { while let Some((cmd, reply)) = ctrl_cmd_rx.recv().await { { let mut guard = pending_reply_writer.lock().await; - *guard = Some(reply); + guard.push_back(reply); } let payload = serde_json::to_vec(&cmd).unwrap_or_default(); if let Err(e) = send_frame(&mut write_half, &payload).await { let mut guard = pending_reply_writer.lock().await; - if let Some(r) = guard.take() { + if let Some(r) = guard.pop_back() { let _ = r.send(Err(format!("Failed to write command to Stream 0: {}", e))); } break; @@ -1033,10 +1106,33 @@ pub async fn handle_fleet_ws_client(socket: AxumWebSocket, state: AppState) { reader_task.abort(); write_task.abort(); - // Cleanup on disconnect - fleet_clone.active_connections.write().remove(&dev_id_clone); - fleet_clone.set_device_status(&dev_id_clone, "offline"); - fleet_clone.record_audit("device_disconnected", "system", &dev_id_clone, json!({})); + // Drain any in-flight command replies so callers fail immediately instead of waiting for timeout + { + let mut guard = pending_replies.lock().await; + while let Some(reply) = guard.pop_front() { + let _ = reply.send(Err("Device connection closed".to_string())); + } + } + + // Cleanup on disconnect only if this connection is still the active handle + let was_current = { + let mut conns = fleet_clone.active_connections.write(); + if conns.get(&dev_id_clone).map(|h| h.conn_id) == Some(conn_id) { + conns.remove(&dev_id_clone); + true + } else { + false + } + }; + if was_current { + fleet_clone.set_device_status(&dev_id_clone, "offline"); + fleet_clone.record_audit( + "device_disconnected", + "system", + &dev_id_clone, + json!({"conn_id": conn_id}), + ); + } } // ============================================================================ diff --git a/crates/rekuiper-server/src/handlers/metrics.rs b/crates/rekuiper-server/src/handlers/metrics.rs index 951f1ce5..7915fc21 100644 --- a/crates/rekuiper-server/src/handlers/metrics.rs +++ b/crates/rekuiper-server/src/handlers/metrics.rs @@ -2,24 +2,42 @@ use crate::handlers::current_process_stats; use crate::state::AppState; use axum::{extract::State, http::StatusCode, response::IntoResponse, Json}; use serde_json::json; +use std::sync::LazyLock; +use std::time::{Duration, Instant}; use sysinfo::{CpuRefreshKind, MemoryRefreshKind, RefreshKind, System}; -pub async fn ping_handler() -> impl IntoResponse { - StatusCode::OK -} - -pub async fn root_handler(State(state): State) -> impl IntoResponse { +static HOST_STATS_CACHE: LazyLock> = LazyLock::new(|| { let mut sys = System::new_with_specifics( RefreshKind::new() .with_cpu(CpuRefreshKind::everything()) .with_memory(MemoryRefreshKind::everything()), ); - sys.refresh_all(); + sys.refresh_cpu_usage(); + sys.refresh_memory(); + parking_lot::Mutex::new((Instant::now(), sys)) +}); + +fn cached_host_stats() -> (f32, u64, u64) { + let mut guard = HOST_STATS_CACHE.lock(); + if guard.0.elapsed() >= Duration::from_millis(500) { + guard.1.refresh_cpu_usage(); + guard.1.refresh_memory(); + guard.0 = Instant::now(); + } + ( + guard.1.global_cpu_info().cpu_usage(), + guard.1.used_memory(), + guard.1.total_memory(), + ) +} +pub async fn ping_handler() -> impl IntoResponse { + StatusCode::OK +} + +pub async fn root_handler(State(state): State) -> impl IntoResponse { let uptime = state.start_time.elapsed().as_secs(); - let cpu_usage = sys.global_cpu_info().cpu_usage(); - let memory_used = sys.used_memory(); - let memory_total = sys.total_memory(); + let (cpu_usage, memory_used, memory_total) = cached_host_stats(); let arch = match std::env::consts::ARCH { "x86_64" => "amd64", @@ -58,16 +76,15 @@ pub async fn metrics_dump_check() -> impl IntoResponse { pub async fn metrics_dump(State(state): State) -> impl IntoResponse { let (cpu, memory) = current_process_stats(); - let mut sys = System::new_all(); - sys.refresh_all(); + let (_host_cpu, used_memory, total_memory) = cached_host_stats(); Json(json!({ "metrics": { "cpu": cpu, "cpu_usage": cpu, "memory": memory, "memory_bytes": memory, - "total_memory": sys.total_memory(), - "used_memory": sys.used_memory(), + "total_memory": total_memory, + "used_memory": used_memory, "uptime_seconds": state.start_time.elapsed().as_secs(), }, "cpu": cpu, diff --git a/crates/rekuiper-server/src/lib.rs b/crates/rekuiper-server/src/lib.rs index 97efae20..3c4e50a0 100644 --- a/crates/rekuiper-server/src/lib.rs +++ b/crates/rekuiper-server/src/lib.rs @@ -144,6 +144,7 @@ pub async fn start_server_with_options( state.rule_manager.load_from_kv(&kv).await?; state.schema_manager.load_from_kv(&kv).await?; state.plugin_manager.load_from_kv(&kv).await?; + state.fleet.load_from_kv(&kv).await?; let proto_schemas = state .schema_manager .load_proto_dir(std::path::Path::new("etc/schemas")); @@ -226,18 +227,18 @@ pub async fn start_server_with_options( // Dedicated Prometheus listener when enabled on its own port, mirroring // eKuiper's prome_init behaviour. if config.basic.prometheus && config.basic.prometheus_port != config.basic.rest_port { + let prom_ip = config.basic.rest_ip.clone(); let prom_port = config.basic.prometheus_port; let prom_app = axum::Router::new().route("/metrics", axum::routing::get(prometheus_metrics_handler)); let prom_app = prom_app.with_state(state.clone()); tokio::spawn(async move { - let addr = SocketAddr::from(([0, 0, 0, 0], prom_port)); + let addr: SocketAddr = format!("{}:{}", prom_ip, prom_port) + .parse() + .unwrap_or_else(|_| SocketAddr::from(([127, 0, 0, 1], prom_port))); match tokio::net::TcpListener::bind(addr).await { Ok(listener) => { - tracing::info!( - "Serving prometheus metrics on port http://localhost:{}/metrics", - prom_port - ); + tracing::info!("Serving prometheus metrics on port http://{}/metrics", addr); if let Err(e) = axum::serve(listener, prom_app).await { tracing::warn!("Prometheus metrics server error: {}", e); } @@ -266,22 +267,32 @@ pub async fn start_server_with_options( // Support EdgeX dual port (59720) concurrently alongside default port (9081) let edgex_port = 59720; if config.basic.rest_port != edgex_port { + let edgex_ip = config.basic.rest_ip.clone(); let edgex_app = app.clone(); tokio::spawn(async move { - let edgex_addr = SocketAddr::from(([0, 0, 0, 0], edgex_port)); + let edgex_addr: SocketAddr = format!("{}:{}", edgex_ip, edgex_port) + .parse() + .unwrap_or_else(|_| SocketAddr::from(([127, 0, 0, 1], edgex_port))); if let Ok(listener) = tokio::net::TcpListener::bind(edgex_addr).await { - tracing::info!( - "Serving EdgeX dual-port REST API on http://0.0.0.0:{}", - edgex_port - ); - let _ = axum::serve(listener, edgex_app).tcp_nodelay(true).await; + tracing::info!("Serving EdgeX dual-port REST API on http://{}", edgex_addr); + let _ = axum::serve( + listener, + edgex_app.into_make_service_with_connect_info::(), + ) + .tcp_nodelay(true) + .await; } }); } let listener = tokio::net::TcpListener::bind(addr).await?; println!("Server initialized and ready in {:?}", init_start.elapsed()); - axum::serve(listener, app).tcp_nodelay(true).await?; + axum::serve( + listener, + app.into_make_service_with_connect_info::(), + ) + .tcp_nodelay(true) + .await?; Ok(()) } diff --git a/crates/rekuiper-server/src/routes.rs b/crates/rekuiper-server/src/routes.rs index acf8f916..d79b89d6 100644 --- a/crates/rekuiper-server/src/routes.rs +++ b/crates/rekuiper-server/src/routes.rs @@ -454,26 +454,31 @@ pub fn ip_matches_entry(client_ip: &std::net::IpAddr, entry: &str) -> bool { false } -pub fn extract_client_ip(headers: &axum::http::HeaderMap) -> Option { - if let Some(x_forwarded) = headers.get("x-forwarded-for").and_then(|v| v.to_str().ok()) { - if let Some(first_ip) = x_forwarded.split(',').next() { - if let Ok(ip) = first_ip.trim().parse::() { - return Some(ip); +pub fn extract_client_ip( + headers: &axum::http::HeaderMap, + peer_addr: Option, +) -> Option { + let trust_forwarded = peer_addr.map(|a| a.ip().is_loopback()).unwrap_or(true); + if trust_forwarded { + if let Some(x_forwarded) = headers.get("x-forwarded-for").and_then(|v| v.to_str().ok()) { + if let Some(first_ip) = x_forwarded.split(',').next() { + if let Ok(ip) = first_ip.trim().parse::() { + return Some(ip); + } } } - } - if let Some(x_real_ip) = headers.get("x-real-ip").and_then(|v| v.to_str().ok()) { - if let Ok(ip) = x_real_ip.trim().parse::() { - return Some(ip); + if let Some(x_real_ip) = headers.get("x-real-ip").and_then(|v| v.to_str().ok()) { + if let Ok(ip) = x_real_ip.trim().parse::() { + return Some(ip); + } } } - // Loopback fallback for local connections - Some(std::net::IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1))) + peer_addr.map(|a| a.ip()) } pub async fn ui_handler( axum::extract::State(state): axum::extract::State, - headers: axum::http::HeaderMap, + req: axum::extract::Request, ) -> axum::response::Response { use axum::response::IntoResponse; let cfg = state.config.read(); @@ -490,8 +495,21 @@ pub async fn ui_handler( // 2. Check IP allowlist (if configured) if !cfg.basic.ui_ip_allowlist.is_empty() { - let client_ip = extract_client_ip(&headers) - .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1))); + let peer_addr = req + .extensions() + .get::>() + .map(|ci| ci.0); + let Some(client_ip) = extract_client_ip(req.headers(), peer_addr) else { + return ( + axum::http::StatusCode::FORBIDDEN, + [( + axum::http::header::CONTENT_TYPE, + "text/plain; charset=utf-8", + )], + "Access denied: client IP could not be verified against ui_ip_allowlist\n", + ) + .into_response(); + }; let mut allowed = false; for entry in &cfg.basic.ui_ip_allowlist { if ip_matches_entry(&client_ip, entry) { @@ -514,19 +532,12 @@ pub async fn ui_handler( drop(cfg); - for candidate in &[ - "etc/ui.html", - "/home/paanday/fleet/control/etc/ui.html", - "/home/paanday/fleet/ui.html", - "/tmp/fleet/control/etc/ui.html", - ] { - if let Ok(content) = std::fs::read_to_string(candidate) { - return ( - [(axum::http::header::CONTENT_TYPE, "text/html; charset=utf-8")], - content, - ) - .into_response(); - } + if let Ok(content) = std::fs::read_to_string("etc/ui.html") { + return ( + [(axum::http::header::CONTENT_TYPE, "text/html; charset=utf-8")], + content, + ) + .into_response(); } ( [(axum::http::header::CONTENT_TYPE, "text/html; charset=utf-8")], @@ -570,9 +581,9 @@ pub async fn ui_login( let cfg = state.config.read(); let token = payload.token.trim(); - // 1. Check custom UI password + // 1. Check custom UI password (constant-time) if let Some(ref pw) = cfg.basic.ui_password { - if token == pw { + if crate::state::constant_time_eq(token, pw) { return ( axum::http::StatusCode::OK, axum::Json(serde_json::json!({ "status": "ok", "token": token })), @@ -581,9 +592,9 @@ pub async fn ui_login( } } - // 2. Check fleet token + // 2. Check fleet token (constant-time) if let Some(ref ft) = cfg.basic.fleet_token { - if token == ft { + if crate::state::constant_time_eq(token, ft) { return ( axum::http::StatusCode::OK, axum::Json(serde_json::json!({ "status": "ok", "token": token })), diff --git a/crates/rekuiper-server/src/state.rs b/crates/rekuiper-server/src/state.rs index 917d23ac..c3321f2f 100644 --- a/crates/rekuiper-server/src/state.rs +++ b/crates/rekuiper-server/src/state.rs @@ -648,9 +648,25 @@ pub fn apply_edgex_env_overlays(source_configs: &Arc bool { + let a_bytes = a.as_bytes(); + let b_bytes = b.as_bytes(); + if a_bytes.len() != b_bytes.len() { + return false; + } + let mut diff: u8 = 0; + for (&x, &y) in a_bytes.iter().zip(b_bytes.iter()) { + diff |= std::hint::black_box(x ^ y); + } + std::hint::black_box(diff) == 0 +} + /// JWT authorization guard for deployments with `basic.authentication: /// true`. eKuiper accepts only a raw RS256 JWT (no `Bearer ` prefix); `/` -/// and `/ping` stay public. +/// and `/ping` stay public. When `ui_password` or `fleet_token` are also +/// configured, valid UI/fleet tokens are accepted alongside JWT. pub async fn auth_guard( State(state): State, request: axum::http::Request, @@ -680,36 +696,6 @@ pub async fn auth_guard( return next.run(request).await; } - // If upstream JWT authentication is enabled, enforce strict upstream wire-compatibility: - if auth_enabled { - let Some(header_val) = request - .headers() - .get(header::AUTHORIZATION) - .and_then(|v| v.to_str().ok()) - else { - return (StatusCode::UNAUTHORIZED, "Missing authorization header\n").into_response(); - }; - if header_val.starts_with("Bearer ") || header_val.starts_with("bearer ") { - return ( - StatusCode::UNAUTHORIZED, - "Bearer token is not supported, please use raw JWT token\n", - ) - .into_response(); - } - let Some(key_der) = load_auth_public_key() else { - return ( - StatusCode::UNAUTHORIZED, - "Authentication misconfigured: public key not found\n", - ) - .into_response(); - }; - return match verify_jwt_raw(header_val.trim(), &key_der) { - Ok(()) => next.run(request).await, - Err(msg) => (StatusCode::UNAUTHORIZED, msg).into_response(), - }; - } - - // Otherwise, check UI password and fleet token authentication: let query_token = request.uri().query().and_then(|q| { q.split('&').find_map(|pair| { let mut parts = pair.split('='); @@ -739,22 +725,47 @@ pub async fn auth_guard( query_token }; - let Some(token) = token_candidate else { - return (StatusCode::UNAUTHORIZED, "Missing authorization header\n").into_response(); - }; - - // If a UI custom password is configured, check if it matches - if let Some(ref pw) = ui_password { - if token == pw { - return next.run(request).await; + // Check UI custom password or fleet provisioning token first (constant-time) + if let Some(token) = token_candidate { + if let Some(ref pw) = ui_password { + if constant_time_eq(token, pw) { + return next.run(request).await; + } + } + if let Some(ref ft) = fleet_token { + if constant_time_eq(token, ft) { + return next.run(request).await; + } } } - // If a fleet bearer/provisioning token is configured, check if it matches - if let Some(ref ft) = fleet_token { - if token == ft { - return next.run(request).await; + // If upstream JWT authentication is enabled, enforce strict upstream wire-compatibility: + if auth_enabled { + let Some(header_val) = header_val else { + return (StatusCode::UNAUTHORIZED, "Missing authorization header\n").into_response(); + }; + if header_val.starts_with("Bearer ") || header_val.starts_with("bearer ") { + return ( + StatusCode::UNAUTHORIZED, + "Bearer token is not supported, please use raw JWT token\n", + ) + .into_response(); } + let Some(key_der) = load_auth_public_key() else { + return ( + StatusCode::UNAUTHORIZED, + "Authentication misconfigured: public key not found\n", + ) + .into_response(); + }; + return match verify_jwt_raw(header_val.trim(), &key_der) { + Ok(()) => next.run(request).await, + Err(msg) => (StatusCode::UNAUTHORIZED, msg).into_response(), + }; + } + + if token_candidate.is_none() { + return (StatusCode::UNAUTHORIZED, "Missing authorization header\n").into_response(); } ( diff --git a/crates/rekuiper-server/tests/test_otlp_and_ui.rs b/crates/rekuiper-server/tests/test_otlp_and_ui.rs index 9b79a3f6..cc51c60b 100644 --- a/crates/rekuiper-server/tests/test_otlp_and_ui.rs +++ b/crates/rekuiper-server/tests/test_otlp_and_ui.rs @@ -551,3 +551,53 @@ async fn test_otlp_sink_400_error_routes_to_dlq() { let error_msg = dlq_rec.data.get("error").and_then(|v| v.as_str()).unwrap(); assert!(error_msg.contains("HTTP 400")); } + +#[tokio::test] +async fn test_ui_ip_allowlist_fails_closed_and_jwt_coexistence() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + + let stream_bus = StreamBus::new(); + let stream_manager = StreamManager::new(); + let rule_manager = RuleManager::new(stream_bus.clone()); + + let mut config = KuiperConfig::default(); + config.basic.enable_ui = true; + config.basic.ui_ip_allowlist = vec!["127.0.0.1/32".to_string()]; + // Enable both basic.authentication (JWT) and ui_password simultaneously + config.basic.authentication = true; + config.basic.ui_password = Some("CoexistingAdminToken123".to_string()); + + let state = AppState::new( + "test_ui_coexist".to_string(), + config, + stream_manager, + TableManager::new(), + rule_manager, + stream_bus, + ); + + let app = create_router(state); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let client = reqwest::Client::new(); + + // 1. Without ConnectInfo or X-Forwarded-For, allowlist must fail closed (403 Forbidden), NOT default to 127.0.0.1 + let no_ip_resp = client + .get(format!("http://{}/ui", local_addr)) + .send() + .await + .unwrap(); + assert_eq!(no_ip_resp.status(), reqwest::StatusCode::FORBIDDEN); + + // 2. With basic.authentication = true AND ui_password set, Bearer must succeed + let auth_resp = client + .get(format!("http://{}/streams", local_addr)) + .header("Authorization", "Bearer CoexistingAdminToken123") + .send() + .await + .unwrap(); + assert_eq!(auth_resp.status(), reqwest::StatusCode::OK); +} diff --git a/crates/rekuiper-server/tests/test_phase4_fleet_control_plane.rs b/crates/rekuiper-server/tests/test_phase4_fleet_control_plane.rs index 9db42582..8803644a 100644 --- a/crates/rekuiper-server/tests/test_phase4_fleet_control_plane.rs +++ b/crates/rekuiper-server/tests/test_phase4_fleet_control_plane.rs @@ -435,3 +435,111 @@ async fn test_fleet_auth_enforcement() { "Authenticated WebSocket handshake must succeed" ); } + +#[tokio::test] +async fn test_fleet_concurrent_commands_and_kv_reload() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let ws_url = format!("ws://127.0.0.1:{}/fleet/ws", port); + + let kv: std::sync::Arc = std::sync::Arc::new(MemKvStore::new()); + let mut cp_state = build_test_state("control_plane_concurrent"); + cp_state.fleet = Arc::new(rekuiper_server::fleet::FleetManager::new_with_kv( + kv.clone(), + )); + + let cp_router = create_router(cp_state.clone()); + tokio::spawn(async move { + let _ = axum::serve(listener, cp_router).await; + }); + + // Spawn 1 edge agent + let edge_state = build_test_state("edge_concurrent_1"); + let edge_router = create_router(edge_state.clone()); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + tokio::spawn(async move { + let _ = connect_outbound_ws_agent( + &ws_url, + edge_state, + edge_router, + AgentMetadata { + device_id: "edge-conc-01".to_string(), + name: "Concurrent Gateway".to_string(), + plant: "Plant-Austin".to_string(), + region: "US-South".to_string(), + device_type: "IPC-Edge".to_string(), + version: "1.0.0".to_string(), + }, + Duration::from_millis(100), + cancel_rx, + ) + .await; + }); + + // Wait for registration + for _ in 0..30 { + if cp_state + .fleet + .active_connections + .read() + .contains_key("edge-conc-01") + { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!(cp_state + .fleet + .active_connections + .read() + .contains_key("edge-conc-01")); + + // Fire 5 concurrent SendControl (Ping) commands on Stream 0 and verify none are dropped/overwritten + let mut handles = Vec::new(); + for _ in 0..5 { + let fleet = cp_state.fleet.clone(); + handles.push(tokio::spawn(async move { + fleet + .send_device_control( + "edge-conc-01", + rekuiper_server::agent::AgentControlCommand::Ping, + ) + .await + })); + } + for h in handles { + let res = h.await.unwrap().expect("Concurrent Ping must succeed"); + assert_eq!(res["status"], "pong"); + } + + // Create a snapshot and verify KV reload restores devices, snapshots, and audit logs + let snap = cp_state + .fleet + .create_snapshot( + "kv-snap".to_string(), + "desc".to_string(), + json!({"version": "v1"}), + ) + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + let reloaded_fleet = rekuiper_server::fleet::FleetManager::new_with_kv(kv.clone()); + reloaded_fleet.load_from_kv(&kv).await.unwrap(); + assert!(reloaded_fleet.devices.read().contains_key("edge-conc-01")); + // Reloaded online devices reset to "offline" until heartbeats resume + assert_eq!( + reloaded_fleet + .devices + .read() + .get("edge-conc-01") + .unwrap() + .status, + "offline" + ); + assert!(reloaded_fleet + .snapshots + .read() + .contains_key(&snap.snapshot_id)); + + let _ = cancel_tx.send(true); +} diff --git a/crates/rule-engine/Cargo.toml b/crates/rule-engine/Cargo.toml index eb031375..7ebed151 100644 --- a/crates/rule-engine/Cargo.toml +++ b/crates/rule-engine/Cargo.toml @@ -20,14 +20,6 @@ path = "src/lib.rs" name = "rule-engine" path = "src/main.rs" -[[bin]] -name = "kuiperd" -path = "src/bin/kuiperd.rs" - -[[bin]] -name = "rekuiperd" -path = "src/bin/rekuiperd.rs" - [dependencies] kuiperd = { workspace = true } rekuiper-sql = { workspace = true } diff --git a/crates/rule-engine/src/bin/kuiperd.rs b/crates/rule-engine/src/bin/kuiperd.rs deleted file mode 100644 index 83ad340e..00000000 --- a/crates/rule-engine/src/bin/kuiperd.rs +++ /dev/null @@ -1,6 +0,0 @@ -use anyhow::Result; - -#[tokio::main] -async fn main() -> Result<()> { - kuiperd::run_daemon().await -} diff --git a/crates/rule-engine/src/bin/rekuiperd.rs b/crates/rule-engine/src/bin/rekuiperd.rs deleted file mode 100644 index 83ad340e..00000000 --- a/crates/rule-engine/src/bin/rekuiperd.rs +++ /dev/null @@ -1,6 +0,0 @@ -use anyhow::Result; - -#[tokio::main] -async fn main() -> Result<()> { - kuiperd::run_daemon().await -} diff --git a/docs/en_US/CONTRIBUTING.md b/docs/en_US/CONTRIBUTING.md index 91ee498d..9e0fa22d 100644 --- a/docs/en_US/CONTRIBUTING.md +++ b/docs/en_US/CONTRIBUTING.md @@ -60,7 +60,7 @@ Report security vulnerabilities privately to the **I-Dacs Labs Security Team**: ## Cargo Workspace Architecture -`rekuiper` is structured as a modular Cargo workspace containing 8 crates: +`rekuiper` is structured as a modular Cargo workspace containing 10 crates: | Crate | Directory | Purpose | | :--- | :--- | :--- | @@ -71,7 +71,9 @@ Report security vulnerabilities privately to the **I-Dacs Labs Security Team**: | `rekuiper-server` | `crates/rekuiper-server` | REST API, 100% OpenAPI 3.0 route handlers, Prometheus metrics server, pipeline runner. | | `rekuiper-cli` | `crates/rekuiper-cli` | Command-line interface (`kuiper`) drop-in client binary. | | `rekuiper-mcp` | `crates/rekuiper-mcp` | Model Context Protocol server exposing AI assistant tools for rule validation and control. | -| `kuiperd` | `crates/kuiperd` | Server daemon executable entrypoint (`kuiperd`). | +| `kuiperd` | `crates/kuiperd` | Server daemon executable entrypoint (`kuiperd` and `rekuiperd`). | +| `rule-engine` | `crates/rule-engine` | High-level embeddable library façade for Rust applications. | +| `ruleengine` | `crates/ruleengine` | Crate alias re-exporting `rule-engine`. | --- @@ -104,7 +106,7 @@ Or run directly through Cargo: cargo run --bin kuiperd -- --etc etc ``` -The server listens on `http://0.0.0.0:9081`. Prometheus metrics are available at `http://0.0.0.0:20499/metrics`. +The server listens on `http://127.0.0.1:9081` by default (configurable via `basic.restIp` in `etc/kuiper.yaml`). Prometheus metrics are available at `http://127.0.0.1:20499/metrics`. ### Use the CLI Client Run client commands against the local daemon: diff --git a/docs/en_US/api/restapi/authentication.md b/docs/en_US/api/restapi/authentication.md index 09231b88..17b89edb 100644 --- a/docs/en_US/api/restapi/authentication.md +++ b/docs/en_US/api/restapi/authentication.md @@ -50,3 +50,34 @@ Ensure that the public key file `sample_key.pub` exists in the `etc/mgmt` direct ## JWT Signature Sign the token using your RSA private key. rekuiper verifies the signature against the matching public key located in `etc/mgmt`. + +## Coexistence with Console Password and Fleet Token + +When `basic.authentication: true` is enabled alongside `uiPassword` or `fleetToken`: +- **RS256 JWT Tokens**: Accepted in the `Authorization` header (either raw `` or `Bearer `). +- **Web Console Password (`uiPassword`)**: Requests carrying `Authorization: Bearer ` are verified using constant-time byte comparison and permitted for Web Console sessions. +- **Fleet Control Plane Token (`fleetToken`)**: Requests to `/fleet/*` or `/api/v1/fleet/*` carrying `Authorization: Bearer ` or `X-Fleet-Token: ` are verified in constant time. + + + + REST API Authentication Guard (auth_guard) + + Incoming HTTP Request + Authorization Header + or X-Fleet-Token + Public: /, /ping, /ui, /ui/auth + + + + Credential Verification Pipeline + 1. Fleet Route Check: Constant-Time fleetToken Match + 2. Console Bearer Check: Constant-Time uiPassword Match + 3. JWT Guard: RS256 Signature & Claims (etc/mgmt/*.pub) + Zero Timing Side-Channels (constant_time_eq) + + + + Protected Route Handler + Valid: 200 OK / Route Execution + Invalid: 401 Unauthorized + diff --git a/docs/en_US/benchmarks/overview.md b/docs/en_US/benchmarks/overview.md index 34773045..d46b4992 100644 --- a/docs/en_US/benchmarks/overview.md +++ b/docs/en_US/benchmarks/overview.md @@ -51,7 +51,7 @@ Different benchmark suites evaluate specific capabilities across versions: | Suite Name | Evaluated Version | Workload Type | Key Result | Link | | :--- | :---: | :--- | :--- | :--- | -| **In-Process Engine Throughput** | **rekuiper 1.0.0-rc4** | Continuous stream execution of 500k events on 1 core | Measures **2,207,367 events/s** (zero-parse pipeline) and **1,085,885 events/s** (JSON pipeline) with active dead-letter queue support. | [View Benchmark](#version-1-0-0-rc4-in-process-engine-throughput-benchmark) | +| **In-Process Engine Throughput** | **rekuiper 1.0.0-rc4** | Continuous stream execution of 500k events on 1 core | Measures **2,464,711 events/s** (zero-parse pipeline) and **1,141,417 events/s** (JSON pipeline) with active dead-letter queue support. | [View Benchmark](#version-1-0-0-rc4-in-process-engine-throughput-benchmark) | | **Concurrent Parallel Rules** | **rekuiper 0.505-beta** | Stream fan-out to $N$ parallel SQL rules | Sustains **1,000 parallel rules** (500,000 evals/s) on 1 core; eKuiper ceiling is 100 rules. | [View Benchmark](./parallel_rules.md) | | **Single-Rule MQTT Throughput** | **rekuiper 0.500-beta** | Ingestion of 5 industrial MQTT shapes | Sustains **150,000 to 200,000 msg/s** loss-free on 1 core; 6.3x to 10x higher than competitors. | [View Benchmark](./throughput.md) | | **120-Second Bounded Sustained** | **rekuiper 0.426-beta** | Long-duration MQTT ingest (12M events) | Sustains **100,000 msg/s for 120s** across all five workloads with 0.00% loss. | [View Benchmark](./throughput.md#version-0-426-bounded-sustained-ladder) | @@ -89,10 +89,27 @@ The engine delivers projected records to a bounded in-memory subscriber channel. rekuiper version 1.0.0-rc4 recorded the following performance metrics: + + + Single-Core Throughput & Resource Profile (1 Physical Core, 1 GiB RAM Limit) + Zero-Parse In-Process SQL + + 2,464,711 eps (202.86 ms) + End-to-End JSON SQL + + 1,141,417 eps (438.05 ms) + Parallel 1,000-Rule Fan-Out + + 500,000 evals/s (1,000 rules) + Sustained MQTT Broker I/O + + 150,000 - 200,000 msg/s (4.4 - 10.2 MiB RSS, 0.00% Loss) + + | Pipeline Architecture | Evaluated Version | Records Processed | Elapsed Time | Throughput | Memory Model | Drain Delay | | :--- | :---: | :---: | :---: | :---: | :--- | :---: | -| **Zero-Parse In-Process Engine Pipeline** | **v1.0.0-rc4** | 500,000 | **226.51 ms** | **2,207,367 events/s** | Zero heap allocation, zero GC pauses | **0.0 ms** | -| **End-to-End JSON Streaming Pipeline** | **v1.0.0-rc4** | 500,000 | **460.45 ms** | **1,085,885 events/s** | Dynamic JSON parse and serialize | **0.0 ms** | +| **Zero-Parse In-Process Engine Pipeline** | **v1.0.0-rc4** | 500,000 | **202.86 ms** | **2,464,711 events/s** | Zero heap allocation, zero GC pauses | **0.0 ms** | +| **End-to-End JSON Streaming Pipeline** | **v1.0.0-rc4** | 500,000 | **438.05 ms** | **1,141,417 events/s** | Dynamic JSON parse and serialize | **0.0 ms** | #### Key Enabling Optimizations in Version 1.0.0-rc4 diff --git a/docs/en_US/concepts/streaming/join.md b/docs/en_US/concepts/streaming/join.md index 82c7bd49..cf4ebad4 100644 --- a/docs/en_US/concepts/streaming/join.md +++ b/docs/en_US/concepts/streaming/join.md @@ -7,5 +7,28 @@ rekuiper supports two join combinations: - **Stream-to-Stream Join**: Joins multiple unbounded streams. Stream-to-stream joins require an explicit window definition. - **Stream-to-Table Join**: Joins a stream to a table. Arriving stream events trigger join calculations against current table records. -Supported join types include `INNER`, `LEFT`, `RIGHT`, `FULL`, and `CROSS`. +Supported join types include `INNER`, `LEFT`, `RIGHT`, `FULL`, `CROSS`, and temporal `ASOF JOIN` / `LEFT ASOF JOIN` (which matches each fast stream event with the most recent reference event at or before its timestamp without requiring a window cross-product). + + + + Streaming Join Topologies in rekuiper + + 1. Stream-to-Stream Join + Stream A + Stream B in Window + INNER / LEFT / RIGHT / FULL / CROSS + Requires GROUP BY Window Clause + SharedMap Copy-on-Write Buffers + + 2. Stream-to-Table Join + Live Stream + Lookup / Scan Table + Enriches Events on Arrival + Backed by Memory, Redis, or SQL + O(1)VecDeque Bounded Eviction + + 3. Temporal ASOF JOIN + ASOF JOIN / LEFT ASOF JOIN + Matches Latest Reference Reading + Condition: s.ts >= r.ts + Zero Window Cross-Product Overhead + diff --git a/docs/en_US/concepts/streaming/overview.md b/docs/en_US/concepts/streaming/overview.md index d249f7cc..06b6d211 100644 --- a/docs/en_US/concepts/streaming/overview.md +++ b/docs/en_US/concepts/streaming/overview.md @@ -2,6 +2,8 @@ Streaming data is a continuous sequence of data records generated over time. Stream processing processes records in motion. Unlike batch processing, the engine processes each record immediately upon arrival. +![rekuiper Stream Processing Pipeline](/diagrams/how_it_works.svg) + ## Streaming Characteristics Stream processing features these primary characteristics: diff --git a/docs/en_US/concepts/streaming/windowing.md b/docs/en_US/concepts/streaming/windowing.md index 55741285..56fd0074 100644 --- a/docs/en_US/concepts/streaming/windowing.md +++ b/docs/en_US/concepts/streaming/windowing.md @@ -9,3 +9,7 @@ rekuiper supports these built-in window categories: For complete window syntax and types, refer to [Window Functions](../../sqls/windows.md). +![Tumbling Time Window](/diagrams/windows/tumbling_window.svg) + +![Tumbling Count Window](/diagrams/windows/tumbling_count_window.svg) + diff --git a/docs/en_US/configuration/global_configurations.md b/docs/en_US/configuration/global_configurations.md index e6859479..56a8c75c 100644 --- a/docs/en_US/configuration/global_configurations.md +++ b/docs/en_US/configuration/global_configurations.md @@ -160,11 +160,38 @@ basic: keyfile: /var/https-server.key ``` -- `restIp`: The network IP address bound by the REST API server. Defaults to `127.0.0.1` (PR #4089 security hardening). +- `restIp`: The network IP address bound by the REST API server (`9081`), the secondary EdgeX listener (`59720`), and the Prometheus metrics server (`20499`). Defaults to `127.0.0.1` (PR #4089 security hardening). - `restPort`: The HTTP port for the REST API server (default `9081`). - `httpServerIp`: The network IP address bound by the ruletest SSE streaming server. Defaults to `127.0.0.1`. - `httpServerPort`: The port for the ruletest SSE streaming server (default `10081`). - `restTls`: Paths to TLS certificates and private keys. When configured, the REST API listens on HTTPS. +- `enableUi`: Enables the embedded Web Console at `GET /ui` (default `false`). +- `uiPassword`: Optional password required to unlock the Web Console (`POST /ui/auth`) and authenticate console API calls (`Authorization: Bearer `). Verified using constant-time comparison. +- `uiIpAllowlist`: List of IP addresses or CIDR ranges permitted to access `/ui` and `/ui/auth`. Fails closed (`HTTP 403 Forbidden`) when a client IP does not match or cannot be verified; `X-Forwarded-For` and `X-Real-IP` headers are trusted only when the direct TCP peer is loopback (`127.0.0.1` or `::1`). +- `fleetToken`: Shared secret token for Fleet Control Plane WebSocket agents (`/fleet/ws`) and Fleet REST API calls (`Authorization: Bearer ` or `X-Fleet-Token`), verified in constant time. + + + + Default Loopback Binding & Listener Architecture (basic.restIp = 127.0.0.1) + + Primary REST & Web UI + Bind: restIp:restPort (127.0.0.1:9081) + Guards: RS256 JWT / uiPassword / fleetToken + Fail-Closed CIDR uiIpAllowlist + Loopback-Trusted X-Forwarded-For + + Prometheus Metrics Server + Bind: restIp:prometheusPort (127.0.0.1:20499) + Endpoint: GET /metrics + 500ms TTL Cached Host Telemetry + 32 Multidimensional Metric Families + + EdgeX & Ruletest Listeners + EdgeX: restIp:59720 (127.0.0.1:59720) + Ruletest SSE: httpServerIp:10081 + Zero Unintended 0.0.0.0 Exposure + Explicit Override via kuiper.yaml / Env + > [!NOTE] > For security hardening, rekuiper defaults all listeners (`ip`, `restIp`, `httpServerIp`) to loopback (`127.0.0.1`), preventing unintentional exposure over external networks when authentication is disabled. When deploying inside Docker containers or edge gateways requiring external access, explicitly configure `restIp: 0.0.0.0` and `httpServerIp: 0.0.0.0` in `kuiper.yaml`, or set environment variables `KUIPER__BASIC__RESTIP=0.0.0.0` and `KUIPER__BASIC__HTTPSERVERIP=0.0.0.0`. @@ -195,7 +222,7 @@ basic: prometheusPort: 20499 ``` -When enabled, rekuiper exports Prometheus metrics on `http://localhost:20499/metrics`. You can set `prometheusPort` to match `restPort` to serve metrics through the REST API port. +When enabled, rekuiper exports Prometheus metrics on `http://:20499/metrics` (default `http://127.0.0.1:20499/metrics`). You can set `prometheusPort` to match `restPort` to serve metrics through the REST API port. ## Plugin Hosts (Legacy Compatibility) diff --git a/docs/en_US/configuration/secrets.md b/docs/en_US/configuration/secrets.md index 4eee04b0..9edcebcf 100644 --- a/docs/en_US/configuration/secrets.md +++ b/docs/en_US/configuration/secrets.md @@ -4,6 +4,30 @@ This document describes dynamic secret resolution in `rekuiper`. Do not store passwords, keys, or tokens in plain text in configuration files or stream definitions. Use dynamic secrets instead. + + + Dynamic Secret Resolution, 300s Caching & API Redaction + + 1. Secret URI Reference + vault://mount/path#field + env://VARIABLE_NAME + Used in YAML Configs & SQL DDL + + + + 2. Resolver & TTL Cache + Vault KV v1/v2 & Process Env + 300s (5-Minute) In-Memory Cache + Stale-While-Unreachable Failover + + + + 3. Redaction Boundary + Connectors Receive Plaintext + Logs Replace Values with [REDACTED] + REST APIs Preserve URI Only + + `rekuiper` supports two secret resolution backends: 1. **HashiCorp Vault** (`vault://`): Retrieves secrets from an external Vault service. 2. **Environment Variables** (`env://`): Retrieves secrets from system or container environment variables. diff --git a/docs/en_US/extension/wasm/overview.md b/docs/en_US/extension/wasm/overview.md index 49630dda..b821c12e 100644 --- a/docs/en_US/extension/wasm/overview.md +++ b/docs/en_US/extension/wasm/overview.md @@ -163,6 +163,42 @@ Unloads the module and removes all its exported functions from the active SQL ca --- +## Sandbox Fuel Metering and Instance Pooling + +The `rekuiper` WebAssembly runtime enforces deterministic execution limits and lock-free concurrency across streaming rules: + +1. **Deterministic Fuel Metering (`DEFAULT_WASM_FUEL_BUDGET = 1,000,000`)**: Every WASM UDF invocation runs with `wasmi` fuel consumption enabled and a strict per-call budget of `1,000,000` instructions. A buggy or malicious `.wasm` module containing an infinite `loop {}` traps deterministically once its fuel budget is exhausted instead of starving Tokio worker threads. +2. **Lock-Free Instance Pooling (`runtime_pool`)**: Compiled `wasmi::Module` definitions maintain a pool of pre-instantiated `WasmRuntimeInstance` states (`Store` + `Instance`). Concurrent SQL rules check out an instance, execute the UDF outside the registry lock, and return the healthy instance to the pool upon completion. + + + + WebAssembly (wasmi) UDF Sandbox & Instance Pool + + 1. Module Registry + POST /plugins/wasm + Validates .wasm Magic Bytes + Extracts Exported UDF Symbols + Pre-compiles wasmi::Module + + + + 2. Lock-Free Runtime Pool + Pooled WasmRuntimeInstance (Store + Instance) + Checked Out Outside Global Mutex + Concurrent Multi-Rule SQL Execution + Recycled on Clean Completion + + + + 3. Deterministic Sandbox + Fuel Budget: 1,000,000 / Call + Isolated Linear Memory + Traps Infinite Loops Instantly + Zero Host Syscall Access + + +--- + ## Invoking Wasm Functions in SQL Once registered, you can invoke exported functions directly in SQL rules or through the `wasm_run` helper function: diff --git a/docs/en_US/guide/rules/error_handling_dlq.md b/docs/en_US/guide/rules/error_handling_dlq.md index 3e4089ce..4638286f 100644 --- a/docs/en_US/guide/rules/error_handling_dlq.md +++ b/docs/en_US/guide/rules/error_handling_dlq.md @@ -4,6 +4,29 @@ Invalid records occur during stream processing. Examples include malformed JSON, rekuiper includes dead-letter queues (DLQ) to isolate and store invalid records without stopping the pipeline. + + + 5-Stage Dead-Letter Queue (DLQ) Isolation & Routing Pipeline + + 0. CodecDecode + JSON / Protobuf / UTF-8 + + 1. SchemaValidation + Strict DDL Field Types + + 2. SqlEval + Math / Cast / UDF Trap + + 3. DataTemplate + Go Template Formatting + + 4. SinkDelivery + Permanent 4xx Rejections + + Bounded Ring Buffer (GET/DELETE /rules/:id/errors) + Optional External DLQ Sink + Poison-Pill Payload Truncated to 4 KB • Atomic Prometheus Stage Counters • Non-Recursive Dispatch + + --- ## Architecture Overview diff --git a/docs/en_US/guide/rules/graph_rule.md b/docs/en_US/guide/rules/graph_rule.md index 62b8fbaa..27f13352 100644 --- a/docs/en_US/guide/rules/graph_rule.md +++ b/docs/en_US/guide/rules/graph_rule.md @@ -4,6 +4,8 @@ rekuiper supports SQL queries and graph models to define rule logic. The graph A The `graph` object contains `nodes` and `topo` properties. `nodes` defines pipeline elements. `topo` defines directed connections (edges) between nodes. +![Directed Acyclic Graph (DAG) Pipeline Topology](/diagrams/topology_dag.svg) + Below is an example of a linear graph rule: `demo` -> `humidityFilter` -> `mqttout`. The rule reads from an MQTT source, filters by humidity, and transmits output to an MQTT sink: ```json diff --git a/docs/en_US/guide/rules/rule_pipeline.md b/docs/en_US/guide/rules/rule_pipeline.md index 2e404ca6..ba4d6353 100644 --- a/docs/en_US/guide/rules/rule_pipeline.md +++ b/docs/en_US/guide/rules/rule_pipeline.md @@ -50,6 +50,31 @@ Rule pipelines connect through shared memory topics. You create each pipeline st The memory topic connects `rule1` to downstream rules: `rule1 -> {rule2-1, rule2-2}`. + + + In-Memory Rule Pipeline Fan-Out (Zero External Broker Overhead) + + Upstream: rule1 + Filters null temperature + Sink: memory (home/ch1/sensor1) + + + + Memory Topic Bus + Topic: home/+/sensor1 + Wildcard Pattern Matching + + + + + + Downstream: rule2-1 + SELECT avg(temperature) CountWindow(10) + + Downstream: rule2-2 + SELECT temperature + 273.15 AS k + + Notes: - You can combine the memory sink with other sinks in the `actions` array of a single rule. - Memory sources support MQTT-style wildcards (such as `+`) to subscribe to multiple memory topics. diff --git a/docs/en_US/guide/ui/web_console_user_guide.md b/docs/en_US/guide/ui/web_console_user_guide.md index cdb68c8a..21d86230 100644 --- a/docs/en_US/guide/ui/web_console_user_guide.md +++ b/docs/en_US/guide/ui/web_console_user_guide.md @@ -152,7 +152,7 @@ To ensure only designated engineering subnets can connect to the web console: - "10.0.0.0/8" - "192.168.1.0/24" ``` -Any connection originating from an unlisted IP address is immediately rejected with `HTTP 403 Forbidden`. +Any connection originating from an unlisted IP address (or when the client IP cannot be verified) is immediately rejected with `HTTP 403 Forbidden` (fail-closed). Forwarded IP headers (`X-Forwarded-For` and `X-Real-IP`) are trusted only when the direct TCP connection originates from a local reverse proxy (`127.0.0.1` or `::1`). #### 3. Set a Console Password or JWT Token @@ -313,12 +313,12 @@ To ensure operators never need to search externally, every interactive component | UI Location | In-Console Trigger | Documentation Target | Purpose | | :--- | :--- | :--- | :--- | -| **Topbar Navigation** | `📖 User Guide & Docs` | [Web Console User Guide](file:///docs/en_US/guide/ui/web_console_user_guide.md) | Full user manual, security architecture, and operational journeys | -| **Topbar Navigation** | `🔒 Console Security Status` | [Security & Access Control Architecture](file:///docs/en_US/guide/ui/web_console_user_guide.md#security--access-control-architecture) | Port exposure protection, IP allowlists, and credentials | -| **Rule Designer** | `📖 Rule Docs` | [Rule Overview](file:///docs/en_US/guide/rules/overview.md) | SQL syntax, temporal windows, and multi-sink actions | -| **Rule Designer** | `📖 Window Docs` | [Window Functions](file:///docs/en_US/guide/rules/overview.md#window-functions) | Tumbling, Hopping, Session, and Counting window semantics | -| **Stream Designer** | `📖 Stream Docs` | [Stream Overview](file:///docs/en_US/guide/streams/overview.md) | Data sources, connectors, payload formats, and DDL syntax | -| **ConfKey Modal** | `📖 ConfKey Docs` | [Data Sources Configuration](file:///docs/en_US/guide/sources/overview.md) | Reusable connector profiles, TLS options, and credentials | -| **Schema Registry** | `📖 Schema Docs` | [Schema Registry Guide](file:///docs/en_US/guide/schemas/schema_registry.md) | Protobuf contracts, Custom formats, and AST validation | -| **Plugins & UDF Hub** | `📖 Plugin Docs` | [Plugins & Services Hub](file:///docs/en_US/guide/plugins/plugins_and_services_hub.md) | Wasm modules, JavaScript UDFs, and external RPC services | -| **Fleet Top Bar** | `📖 Fleet Docs` | [Fleet Deployment Guide](file:///docs/en_US/guide/fleet/fleet_deployment_and_canary.md) | Health-gated rollouts, Canary verification, and device monitoring | +| **Topbar Navigation** | `📖 User Guide & Docs` | [Web Console User Guide](./web_console_user_guide.md) | Full user manual, security architecture, and operational journeys | +| **Topbar Navigation** | `🔒 Console Security Status` | [Security & Access Control Architecture](./web_console_user_guide.md#1-security--access-control-architecture) | Port exposure protection, IP allowlists, and credentials | +| **Rule Designer** | `📖 Rule Docs` | [Rule Overview](../rules/overview.md) | SQL syntax, temporal windows, and multi-sink actions | +| **Rule Designer** | `📖 Window Docs` | [Window Functions](../../sqls/windows.md) | Tumbling, Hopping, Session, and Counting window semantics | +| **Stream Designer** | `📖 Stream Docs` | [Stream Overview](../streams/overview.md) | Data sources, connectors, payload formats, and DDL syntax | +| **ConfKey Modal** | `📖 ConfKey Docs` | [Data Sources Configuration](../sources/overview.md) | Reusable connector profiles, TLS options, and credentials | +| **Schema Registry** | `📖 Schema Docs` | [Schema Registry Guide](../schemas/schema_registry.md) | Protobuf contracts, Custom formats, and AST validation | +| **Plugins & UDF Hub** | `📖 Plugin Docs` | [Plugins & Services Hub](../plugins/plugins_and_services_hub.md) | Wasm modules, JavaScript UDFs, and external RPC services | +| **Fleet Top Bar** | `📖 Fleet Docs` | [Fleet Deployment Guide](../../operation/fleet_management.md) | Health-gated rollouts, Canary verification, and device monitoring | diff --git a/docs/en_US/index.md b/docs/en_US/index.md index 9227ee84..91291545 100644 --- a/docs/en_US/index.md +++ b/docs/en_US/index.md @@ -229,8 +229,8 @@ All tests run on dedicated Linux hardware constrained to **one physical CPU core | Measure | Result | Tested on | | :--- | :--- | :---: | -| **In-process stream throughput (Zero-parse)** | **2,207,367 events/s** (500k in 226 ms, 0 ms drain) | **1.0.0-rc4** | -| **End-to-end JSON stream throughput** | **1,085,885 events/s** (500k in 460 ms, 0 ms drain) | **1.0.0-rc4** | +| **In-process stream throughput (Zero-parse)** | **2,464,711 events/s** (500k in 203 ms, 0 ms drain) | **1.0.0-rc4** | +| **End-to-end JSON stream throughput** | **1,141,417 events/s** (500k in 438 ms, 0 ms drain) | **1.0.0-rc4** | | MQTT network throughput (Single rule) | 150,000 to 200,000 messages/s (0.00% loss) | 0.500 | | Concurrent rules on one stream | 1,000 rules (500,000 evaluations/s) | 0.505 | | Heap memory under load | 4.4 to 10.2 MiB (Flat plateau, zero GC pauses) | 1.0.0-rc4 | diff --git a/docs/en_US/operation/declarative_gitops.md b/docs/en_US/operation/declarative_gitops.md index 9ee3a8f5..680c63df 100644 --- a/docs/en_US/operation/declarative_gitops.md +++ b/docs/en_US/operation/declarative_gitops.md @@ -2,6 +2,33 @@ Declarative GitOps mode lets you manage stream processing pipelines with configuration files. + + + Declarative GitOps Lifecycle & Read-Only Mutation Guard + + 1. Offline CI Validation + rekuiper validate pipelines.d/ + Checks SQL AST & 221 Functions + Verifies Stream/Table References + YAML / TOML / JSON Manifests + + + + 2. Differential Reconciliation + --declarative & --watch / SIGHUP + Unchanged Rules Keep Running + Restarts Modified Rules Only + Atomic Rollback on Parse Error + + + + 3. Read-Only Mutation Guard + Blocks Ad-Hoc REST Drift (403) + Permits GET, /v1/*, /ruletest + Permits POST /declarative/reload + Zero Configuration Drift + + Key capabilities: - Define streams, tables, rules, connections, and schemas in YAML, TOML, or JSON manifest files. - Validate pipelines offline before deployment with `rekuiper validate`. diff --git a/docs/en_US/operation/fleet_management.md b/docs/en_US/operation/fleet_management.md index 33a8544d..3898f55d 100644 --- a/docs/en_US/operation/fleet_management.md +++ b/docs/en_US/operation/fleet_management.md @@ -10,12 +10,11 @@ Edge `kuiperd` gateways connect outbound to the Rekuiper Server Control Plane ac Edge gateways initiate outbound WebSocket connections to `/fleet/ws` on the Control Plane. Once the connection establishes, Yamux multiplexes the single TCP/TLS channel into independent streams: -- **Stream 0 (Control & Telemetry):** Carries bidirectional JSON-RPC frames. The Control Plane dispatches commands (`ApplySnapshot`, `Rollback`, `GetStatus`). Edge agents send periodic heartbeats (CPU, memory, rule count, DLQ alerts, and active snapshot integrity hashes). -- **Streams 1..N (Remote Proxy Channels):** Tunnel HTTP requests on-demand from the Control Plane (`/api/v1/fleet/devices/:id/proxy/*`) into the local Axum router of the target edge gateway. +- **Stream 0 (Control & Telemetry):** Carries bidirectional JSON frames. The Control Plane dispatches commands (`ApplySnapshot`, `Rollback`, `GetStatus`) using a FIFO response queue so concurrent commands resolve deterministically. Each active session receives a monotonic connection identifier (`conn_id`) so delayed cleanup from a previous socket never disconnects a newer session. Edge agents send periodic heartbeats (CPU, memory, rule count, DLQ alerts, and active snapshot integrity hashes). +- **Streams 1..N (Remote Proxy Channels):** Tunnel HTTP requests on-demand from the Control Plane (`/api/v1/fleet/devices/:id/proxy/*` or `/fleet/devices/:id/proxy/*`) into the local Axum router of the target edge gateway. ![Rekuiper Fleet Control Plane Multiplexing Architecture](/diagrams/fleet_control_plane.svg) - --- ## 2. Edge Gateway Configuration @@ -29,9 +28,10 @@ Start the edge gateway daemon with the Control Plane URL and device identificati | `--control-plane-url ` | `REKUIPER_CONTROL_PLANE_URL` | None | WebSocket address of the Control Plane (for example, `ws://10.0.0.1:9081/fleet/ws`). | | `--device-id ` | `REKUIPER_DEVICE_ID` | Hostname | Unique device identifier in the fleet catalog. | | `--device-name ` | `REKUIPER_DEVICE_NAME` | Device ID | Human-readable name for the device. | -| `--plant ` | `REKUIPER_DEVICE_PLANT` | `default-plant` | Plant group tag (for example, `plant-detroit`). | -| `--region ` | `REKUIPER_DEVICE_REGION` | `default-region` | Regional group tag (for example, `us-east`). | +| `--plant ` | `REKUIPER_DEVICE_PLANT` | `default-plant` | Plant group tag (for example, `detroit-automotive`). | +| `--region ` | `REKUIPER_DEVICE_REGION` | `default-region` | Regional group tag (for example, `us-midwest`). | | `--device-type ` | `REKUIPER_DEVICE_TYPE` | `edge-gateway` | Device category tag. | +| `--fleet-token ` | `REKUIPER_FLEET_TOKEN` | None | Shared authentication token verified by constant-time comparison (`Authorization: Bearer ` or `X-Fleet-Token`). | ### Startup Command @@ -57,11 +57,11 @@ If the WebSocket connection disconnects: ## 3. Device Fleet Catalog -The Control Plane records all registered edge gateways in memory and persists them in the internal key-value store (`kv.rs`). +The Control Plane records all registered edge gateways in memory and persists all fleet state (`fleet_devices`, `fleet_snapshots`, `fleet_rollouts`, and `fleet_audit`) in the internal key-value store (`kv.rs`). All persisted fleet records are reloaded automatically on server startup. ### Endpoints -- `GET /api/v1/fleet/devices` — Returns the list of all registered gateways. +- `GET /api/v1/fleet/devices` — Returns the list of all registered gateways (also mounted at `/fleet/devices`). - `GET /api/v1/fleet/devices/:id` — Returns detailed state and telemetry for one device. - `DELETE /api/v1/fleet/devices/:id` — Deregisters an offline device. - `GET /api/v1/fleet/groups` — Returns device counts grouped by plant, region, and device type. @@ -72,21 +72,25 @@ The Control Plane records all registered edge gateways in memory and persists th { "device_id": "gw-detroit-press-01", "name": "Detroit Stamping Press 01", - "group": { - "plant": "detroit-automotive", - "region": "us-midwest", - "device_type": "industrial-pc" + "plant": "detroit-automotive", + "region": "us-midwest", + "device_type": "industrial-pc", + "labels": { + "line": "stamping-01" }, + "endpoint": "ws://connected", "status": "online", "version": "1.0.0-rc4", "last_heartbeat": 1740000000, - "cpu_percent": 14.2, - "memory_bytes": 48234496, + "cpu_usage": 14.2, + "memory_mb": 46.0, + "rules_count": 8, + "running_rules": 8, + "dlq_errors": 0, "active_snapshot_id": "snap-v2-1", - "active_snapshot_hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "rule_count": 8, - "dlq_count": 0, - "last_rollback_reason": null + "active_snapshot_hash": 15892301948231, + "previous_snapshot_id": "snap-v2-0", + "ConnectedViaWs": true } ``` @@ -104,22 +108,48 @@ Send a `POST` request to `/api/v1/fleet/snapshots`: curl -X POST http://localhost:9081/api/v1/fleet/snapshots \ -H "Content-Type: application/json" \ -d '{ - "snapshot_id": "snapshot-press-v2-4", + "name": "snapshot-press-v2-4", "description": "Added motor vibration anomaly rule", - "manifest_yaml": "streams:\n - sql: CREATE STREAM motor_vibration () WITH (DATASOURCE=\"vibration/raw\", FORMAT=\"json\");\nrules:\n - id: rule_vib_high\n sql: SELECT * FROM motor_vibration WHERE val > 80\n actions:\n - mqtt: { server: \"tcp://broker:1883\", topic: \"alerts/vibration\" }" + "declarative": { + "version": "1.0", + "streams": { + "motor_vibration": "CREATE STREAM motor_vibration () WITH (DATASOURCE=\"vibration/raw\", FORMAT=\"json\");" + }, + "rules": { + "rule_vib_high": { + "sql": "SELECT * FROM motor_vibration WHERE val > 80", + "actions": [ + {"mqtt": {"server": "tcp://broker:1883", "topic": "alerts/vibration"}} + ] + } + } + } }' ``` -The response contains the computed `sha256_hash`: +The response contains the computed `sha256` digest: ```json { - "snapshot_id": "snapshot-press-v2-4", - "sha256_hash": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + "id": "snap-1", + "name": "snapshot-press-v2-4", "description": "Added motor vibration anomaly rule", "created_at": 1740001000, - "stream_count": 1, - "rule_count": 1 + "sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + "declarative": { + "version": "1.0", + "streams": { + "motor_vibration": "CREATE STREAM motor_vibration () WITH (DATASOURCE=\"vibration/raw\", FORMAT=\"json\");" + }, + "rules": { + "rule_vib_high": { + "sql": "SELECT * FROM motor_vibration WHERE val > 80", + "actions": [ + {"mqtt": {"server": "tcp://broker:1883", "topic": "alerts/vibration"}} + ] + } + } + } } ``` @@ -130,20 +160,20 @@ The response contains the computed `sha256_hash`: The Control Plane manages declarative rollout pipelines through four progressive stages: ``` -[Development] ──> [Staging] ──> [Canary (5%)] ──> [Production (100%)] +[canary (10%)] ──> [batch_25 (25%)] ──> [batch_50 (50%)] ──> [full / completed (100%)] ``` ### Health-Gated Canary Promotion -When you target a cohort (for example, `plant-detroit` or `all`): +When you target a cohort (for example, `plant:detroit-automotive` or `all`): 1. **Deploy to Canary:** The rollout pushes the snapshot first to the Canary cohort over Yamux Stream 0. -2. **Watchdog Evaluation:** The Control Plane enters `canary_evaluating` state and monitors telemetry heartbeats. +2. **Watchdog Evaluation:** The Control Plane monitors telemetry heartbeats from canary devices. 3. **Automatic Halt on Anomaly:** - If an edge device triggers an automatic watchdog rollback (runtime exception, panic, or rule error), - Or if an edge device reports an active DLQ count spike, - The Control Plane automatically marks the rollout as `rolled_back`, halts further deployment, and writes a failure record to the audit trail. -4. **Production Promotion:** If Canary devices remain healthy without rollbacks or DLQ spikes, an administrator calls `/api/v1/fleet/rollouts/:id/promote` to promote the snapshot to 100% of the fleet. +4. **Progressive Promotion:** If Canary devices remain healthy without rollbacks or DLQ spikes, an administrator calls `/api/v1/fleet/rollouts/:id/promote` to advance the rollout through `batch_25`, `batch_50`, and `full` (`completed`). ### Launching a Rollout @@ -151,24 +181,22 @@ When you target a cohort (for example, `plant-detroit` or `all`): curl -X POST http://localhost:9081/api/v1/fleet/rollouts \ -H "Content-Type: application/json" \ -d '{ - "rollout_id": "rollout-press-v2-4", - "snapshot_id": "snapshot-press-v2-4", - "target_group": "detroit-automotive", - "current_stage": "canary", - "canary_percentage": 5 + "snapshot_id": "snap-1", + "target_group": "plant:detroit-automotive", + "auto_promote": false }' ``` ### Promoting a Healthy Rollout ```bash -curl -X POST http://localhost:9081/api/v1/fleet/rollouts/rollout-press-v2-4/promote +curl -X POST http://localhost:9081/api/v1/fleet/rollouts/rollout-1/promote ``` ### Halting a Faulty Rollout ```bash -curl -X POST http://localhost:9081/api/v1/fleet/rollouts/rollout-press-v2-4/rollback +curl -X POST http://localhost:9081/api/v1/fleet/rollouts/rollout-1/rollback ``` --- @@ -224,9 +252,9 @@ All administrative actions (snapshot creation, rollout launch, Canary promotion, { "timestamp": 1740001050, "action": "start_rollout", - "target": "rollout-press-v2-4", + "target": "rollout-1", "status": "success", - "details": "Stage: canary, Target: detroit-automotive" + "details": "Stage: canary, Target: plant:detroit-automotive" } ] ``` @@ -238,7 +266,7 @@ All administrative actions (snapshot creation, rollout launch, Canary promotion, The built-in web console (`/ui`) provides a graphical interface for Fleet Management: - **Summary KPI Strip:** Displays total edge gateways, online count, active rollouts, and global DLQ status. -- **Staged Pipeline Progress Bar:** Visualizes stage movement (Dev → Staging → Canary 5% → Prod 100%) with Canary evaluation status and action triggers. +- **Staged Pipeline Progress Bar:** Visualizes stage movement (Canary 10% → Batch 25% → Batch 50% → Full 100%) with Canary evaluation status and action triggers. - **Filterable Catalog:** Filter edge gateways by plant, region, device type, or health status. - **Remote Inspect Viewer:** Opens a live modal to inspect rules, metrics, and logs through the Yamux proxy tunnel. - **Audit & DLQ Tables:** Shows live alerts and security audit logs with timestamps and status indicators. diff --git a/docs/en_US/operation/observability_ui_top.md b/docs/en_US/operation/observability_ui_top.md index 63cd2ca7..046e4ac3 100644 --- a/docs/en_US/operation/observability_ui_top.md +++ b/docs/en_US/operation/observability_ui_top.md @@ -2,6 +2,29 @@ rekuiper includes browser and terminal inspection tools for edge devices and isolated networks. + + + Air-Gapped Inspection Architecture: Embedded Web Console & rekuiper top + + Browser: GET /ui + Zero External CDNs or Fonts + Visual DAG Studio & Rule Inspector + DLQ Viewer & Fleet Rollouts + CIDR Allowlist & Password Guard + + rekuiper Server (Port 9081) + Atomic Rule & Stage Counters + 500ms Cached Host CPU & Memory + Non-Blocking try_read() Status + Single Binary • Zero Sidecars + + Terminal: rekuiper top + SSH-Friendly TUI Monitor + Live In/Out Throughput & Status + 5-Stage DLQ Breakdown & Envelopes + --once Snapshot Mode for CI + + Key properties: - The tools run inside the rekuiper binary without external containers. - The web console uses no external scripts or fonts. diff --git a/docs/en_US/operation/overview.md b/docs/en_US/operation/overview.md index 372328fe..f07670da 100644 --- a/docs/en_US/operation/overview.md +++ b/docs/en_US/operation/overview.md @@ -10,3 +10,29 @@ This guide describes operational administration, monitoring, maintenance, and vi - **[Error Codes](./usage/error_code.md):** References system error codes and troubleshooting actions. - **[Data Tracing](./usage/trace_data.md):** Guides OpenTelemetry tracing configuration and span inspection. - **[Reliability & Stable Scope](../reliability/qualification_and_scope.md):** References MQTT delivery guarantees, broker outage caching, engine recovery, backup, and rollback procedures. +- **[Declarative GitOps Mode](./declarative_gitops.md):** Details manifest validation (`rekuiper validate`), read-only `--declarative` execution, and live file watching (`--watch`). +- **[Fleet Management & Control Plane](./fleet_management.md):** Details outbound WebSocket/Yamux gateway multiplexing, immutable configuration snapshots, and health-gated Canary rollouts. +- **[Inspection Tools (`GET /ui` & `rekuiper top`)](./observability_ui_top.md):** Details the embedded air-gapped Web Console and terminal live monitor. + + + + rekuiper Operations, Observability & Control Plane Pillars + + 1. Configuration & GitOps + etc/rekuiper.yaml & Env Overrides + --declarative Read-Only Guard + Vault & Env Dynamic Secrets + 2-Minute eKuiper Cutover + + 2. Live Observability + 32 Prometheus Metric Families + OpenTelemetry Span Tracing + Embedded Web Console (GET /ui) + Interactive CLI (rekuiper top) + + 3. Fleet & Reliability + Yamux WebSocket Control Plane + Staged Canary Rollouts & Rollback + 5-Stage Dead-Letter Queues (DLQ) + Persistent Sink Outage Cache + diff --git a/docs/en_US/public/diagrams/mcp_architecture.svg b/docs/en_US/public/diagrams/mcp_architecture.svg index 7eeb77e1..b15e94d6 100644 --- a/docs/en_US/public/diagrams/mcp_architecture.svg +++ b/docs/en_US/public/diagrams/mcp_architecture.svg @@ -131,7 +131,7 @@ - 42 Tools • 11 Resources • 5 Prompts + 57 Tools • 25 Resources • 12 Prompts Native Rust binary • Instant response From 0c98ead6210b3854177fd5a8823af31625f502db Mon Sep 17 00:00:00 2001 From: ankur-paan Date: Wed, 28 Oct 2026 11:44:35 +0000 Subject: [PATCH 2/4] chore(release): v1.0.0-rc5 (security hardening, fleet concurrency, wasm fuel sandbox, 2.46M eps, and rustsec fixes) --- .github/workflows/release.yml | 2 +- CHANGELOG.md | 14 +++- Cargo.lock | 69 ++++++++++++++----- Cargo.toml | 16 ++--- Makefile | 2 +- README.md | 12 ++-- RELEASE_NOTES.md | 17 +++++ SECURITY.md | 3 +- crates/rekuiper-sql/Cargo.toml | 2 +- crates/rule-engine/README.md | 4 +- crates/rule-engine/src/lib.rs | 2 +- deny.toml | 5 ++ deploy/chart/rekuiper/Chart.yaml | 2 +- deploy/chart/rekuiper/README.md | 4 +- deploy/docker/README.md | 6 +- deploy/docker/docker-compose.yml | 2 +- docs/en_US/benchmarks/overview.md | 16 ++--- docs/en_US/compare/rekuiper-vs-ekuiper.md | 10 +-- docs/en_US/getting_started/getting_started.md | 2 +- .../getting_started/quick_start_docker.md | 2 +- docs/en_US/index.md | 14 ++-- docs/en_US/installation.md | 8 +-- docs/en_US/operation/fleet_management.md | 2 +- docs/en_US/operation/migration.md | 2 +- .../usage/monitor_with_prometheus.md | 2 +- docs/en_US/public/openapi.json | 2 +- .../reliability/qualification_and_scope.md | 6 +- docs/package-lock.json | 4 +- docs/package.json | 2 +- openapi.json | 2 +- 30 files changed, 154 insertions(+), 82 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 94456532..78bf1f49 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: inputs: tag: - description: 'Release tag (e.g. v1.0.0-rc4)' + description: 'Release tag (e.g. v1.0.0-rc5)' required: false permissions: diff --git a/CHANGELOG.md b/CHANGELOG.md index 30129d06..1d29f41b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 > See `BENCHMARK-AUDIT.md` and `test/BENCHMARKS.md` for the audited methodology. > No latency-histogram p99 is claimed. -## [Unreleased] - Rekuiper Manager & Visual Stream Studio (Phases 1–4) +## [1.0.0-rc5] - 2026-10-28 ### Added - **Zero-Copy Record Metadata (`RecordMeta`)**: Added an immutable, `Arc`-shared `RecordMeta` structure (`topic`, `source`, `custom`) to `StreamRecord`. Metadata passes through SQL projection pipelines and multi-port graph stages with zero map allocations and constant time ($O(1)$) overhead. Added zero-copy `meta()` SQL function with backward-compatible `__meta__` and JSON envelope fallback. @@ -24,6 +24,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Fleet Control Plane ("Rekuiper Manager")**: Added device fleet catalog (`/api/v1/fleet/devices`) and group management (`/api/v1/fleet/groups`). Added staged canary rollout engine (`/api/v1/fleet/rollouts`) supporting progressive rollout stages (5% -> 25% -> 100%) and automatic canary rollback. Added remote Yamux proxy (`/fleet/proxy/:device_id/*path`), unified audit logging (`/api/v1/fleet/audit`), and fleet control dashboard in `/ui`. - **Fleet Provisioning and Bearer Authentication**: Added token validation (`BasicConfig.fleet_token`, `REKUIPER_FLEET_TOKEN`) for `/fleet/ws` and `/api/v1/fleet/*` endpoints. Rejects unauthorized registrations, proxy requests, and rollout commands with HTTP 401 Unauthorized. - **Model Context Protocol (`rekuiper-mcp`) Expansion**: Expanded the native MCP server to 57 tools with dedicated categories for Fleet Management (`list_fleet_devices`, `get_fleet_rollouts`, `trigger_fleet_rollout`, `list_fleet_snapshots`, `proxy_fleet_device_api`) and Visual Stream Studio & Wire-Tap (`get_studio_palette`, `validate_graph_rule`, `toggle_wire_tap`, `get_wire_tap_status`). Added 3 new MCP live resources (`rekuiper://fleet/devices`, `rekuiper://fleet/rollouts`, `rekuiper://studio/palette`) and 2 interactive orchestration prompts (`staged_fleet_rollout`, `design_visual_dag_pipeline`). +- **WASM Fuel Metering and Instance Pooling**: Added deterministic instruction fuel metering (`DEFAULT_WASM_FUEL_BUDGET = 1,000,000`) to `WasmManager` (`wasmi::Config::consume_fuel(true)`), terminating runaway guest loops before they stall worker threads. Added lock-free `ArrayQueue` pooling (8 pre-instantiated instances per module) to eliminate per-event linear memory instantiation overhead. +- **Fleet Control Plane KV Persistence and Non-Blocking Concurrency**: Added durable KV persistence across `fleet_devices`, `fleet_snapshots`, `fleet_rollouts`, and `fleet_audit` namespaces with automatic state restoration (`FleetControlPlane::restore_from_kv`). Refactored `dispatch_snapshot_to_cohort` and `trigger_rollback` to collect command channels under short synchronous locks and perform asynchronous I/O and KV persistence outside lock guards. + +### Security +- **Supply-Chain & RustSec Advisory Remediation**: Upgraded `tokio` to `1.54.1` (remediating `RUSTSEC-2026-0386`), upgraded `zstd-sys` to `2.1.2+zstd.1.5.7` (remediating `RUSTSEC-2026-0410`), and upgraded direct `zstd` dependency in `rekuiper-sql` to `0.14.3` (`zstd-safe 8.2.0`, remediating `RUSTSEC-2026-0391`, `RUSTSEC-2026-0398`, and `RUSTSEC-2026-0403` with zero ignored advisories in downstream core crates). +- **Conditional Route Mounting and Auth Middleware Hardening**: Mounted `/ui` and `/ui/ws` routes only when `basic.enable_ui` is enabled. Enforced `basic.authentication` RS256 JWT validation across `/ui`, `/v1/metrics`, `/v1/logs`, and `/v1/traces` (exempting only `/`, `/ping`, `/fleet/ws`, and `/api/v1/fleet/*` which enforce dedicated fleet bearer token checks). +- **ConnectInfo Peer IP Enforcement and Configurable Prometheus Port**: Wired `into_make_service_with_connect_info::()` on both primary and secondary listeners so `ui_ip_allowlist` validates peer socket addresses reliably. Fixed standalone Prometheus exporter to bind to `config.basic.prometheus_port` instead of a hardcoded port. + +### Changed +- **Verified Benchmark Results (2.46M+ EPS)**: Re-verified single-core `--release` throughput (`perf_throughput.rs`): **2,464,711 events/s** (`202.86 ms` for 500,000 records, zero-parse in-process SQL pipeline) and **1,141,417 events/s** (`438.05 ms` for 500,000 records, end-to-end JSON pipeline). +- **Non-Blocking Host Metrics Cache & O(1) Ring Buffers**: Added a 500ms TTL cache for `sysinfo::System` CPU and memory refreshes on `GET /` and `GET /metrics`. Replaced `Vec::remove(0)` shifts with $O(1)$ `VecDeque::pop_front()` across `TraceSpan`, `RuleErrorBuffer`, and `JoinStreamState`. +- **Modular Source Connectors & Documentation Visuals**: Extracted 2,533 lines of source connector actors into `crates/rekuiper-server/src/engine/sources.rs`. Added 14 ASD-STE100 light-theme architectural SVG diagrams across `docs/en_US/`, bringing visual diagram coverage to 21.3% (39/183 pages) with zero broken links. ## [1.0.0-rc4] - 2026-10-16 diff --git a/Cargo.lock b/Cargo.lock index 37ca3948..b66a2195 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2278,7 +2278,7 @@ dependencies = [ [[package]] name = "kuiperd" -version = "1.0.0-rc4" +version = "1.0.0-rc5" dependencies = [ "anyhow", "clap", @@ -3373,7 +3373,7 @@ dependencies = [ [[package]] name = "rekuiper-cli" -version = "1.0.0-rc4" +version = "1.0.0-rc5" dependencies = [ "anyhow", "clap", @@ -3388,7 +3388,7 @@ dependencies = [ [[package]] name = "rekuiper-conf" -version = "1.0.0-rc4" +version = "1.0.0-rc5" dependencies = [ "anyhow", "serde", @@ -3398,7 +3398,7 @@ dependencies = [ [[package]] name = "rekuiper-connectors" -version = "1.0.0-rc4" +version = "1.0.0-rc5" dependencies = [ "anyhow", "arrow", @@ -3432,7 +3432,7 @@ dependencies = [ [[package]] name = "rekuiper-core" -version = "1.0.0-rc4" +version = "1.0.0-rc5" dependencies = [ "anyhow", "async-trait", @@ -3441,7 +3441,6 @@ dependencies = [ "parking_lot", "regex", "rekuiper-conf", - "rekuiper-sql", "serde", "serde_json", "serde_norway", @@ -3455,7 +3454,7 @@ dependencies = [ [[package]] name = "rekuiper-mcp" -version = "1.0.0-rc4" +version = "1.0.0-rc5" dependencies = [ "anyhow", "clap", @@ -3470,7 +3469,7 @@ dependencies = [ [[package]] name = "rekuiper-server" -version = "1.0.0-rc4" +version = "1.0.0-rc5" dependencies = [ "anyhow", "async-trait", @@ -3505,7 +3504,7 @@ dependencies = [ [[package]] name = "rekuiper-sql" -version = "1.0.0-rc4" +version = "1.0.0-rc5" dependencies = [ "anyhow", "base64", @@ -3525,7 +3524,7 @@ dependencies = [ "sha2", "thiserror 1.0.69", "uuid", - "zstd", + "zstd 0.14.3", ] [[package]] @@ -3620,7 +3619,27 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tracing", - "zstd", + "zstd 0.13.3", +] + +[[package]] +name = "rule-engine" +version = "1.0.0-rc5" +dependencies = [ + "anyhow", + "kuiperd", + "rekuiper-core", + "rekuiper-sql", + "tokio", +] + +[[package]] +name = "ruleengine" +version = "1.0.0-rc5" +dependencies = [ + "anyhow", + "kuiperd", + "tokio", ] [[package]] @@ -4586,9 +4605,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.53.1" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", "libc", @@ -5779,7 +5798,16 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe", + "zstd-safe 7.3.0", +] + +[[package]] +name = "zstd" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" +dependencies = [ + "zstd-safe 8.2.0", ] [[package]] @@ -5791,11 +5819,20 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 521033a3..0018589c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.0.0-rc4" +version = "1.0.0-rc5" edition = "2021" authors = ["I-Dacs Labs ", "rekuiper contributors"] license = "MIT OR Apache-2.0" @@ -67,13 +67,13 @@ boa_engine = "0.20" yamux = "0.13" # Internal crate dependencies -rekuiper-core = { path = "crates/rekuiper-core", version = "1.0.0-rc4" } -rekuiper-sql = { path = "crates/rekuiper-sql", version = "1.0.0-rc4" } -rekuiper-conf = { path = "crates/rekuiper-conf", version = "1.0.0-rc4" } -rekuiper-connectors = { path = "crates/rekuiper-connectors", version = "1.0.0-rc4" } -rekuiper-server = { path = "crates/rekuiper-server", version = "1.0.0-rc4" } -kuiperd = { path = "crates/kuiperd", version = "1.0.0-rc4" } -rule-engine = { path = "crates/rule-engine", version = "1.0.0-rc4" } +rekuiper-core = { path = "crates/rekuiper-core", version = "1.0.0-rc5" } +rekuiper-sql = { path = "crates/rekuiper-sql", version = "1.0.0-rc5" } +rekuiper-conf = { path = "crates/rekuiper-conf", version = "1.0.0-rc5" } +rekuiper-connectors = { path = "crates/rekuiper-connectors", version = "1.0.0-rc5" } +rekuiper-server = { path = "crates/rekuiper-server", version = "1.0.0-rc5" } +kuiperd = { path = "crates/kuiperd", version = "1.0.0-rc5" } +rule-engine = { path = "crates/rule-engine", version = "1.0.0-rc5" } [profile.release] opt-level = 3 diff --git a/Makefile b/Makefile index 2c1e92f9..90b52546 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ BIN_DIR ?= bin -VERSION ?= 1.0.0-rc4 +VERSION ?= 1.0.0-rc5 .PHONY: all all: build diff --git a/README.md b/README.md index 724b9ea7..012cbba0 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ rekuiper is a single-binary **MQTT rule engine** and **streaming SQL processor** written in **Rust**. It sits between your devices and your cloud—filtering, aggregating, and routing live telemetry over MQTT, Kafka, HTTP, Modbus, and OPC-UA on the gateway itself, so you send upstream only what matters. -[![Release](https://img.shields.io/badge/release-v1.0.0--rc4-blue.svg)](https://github.com/Indus-Stack/rekuiper/releases) +[![Release](https://img.shields.io/badge/release-v1.0.0--rc5-blue.svg)](https://github.com/Indus-Stack/rekuiper/releases) [![Rust CI](https://github.com/Indus-Stack/rekuiper/actions/workflows/ci.yml/badge.svg)](https://github.com/Indus-Stack/rekuiper/actions/workflows/ci.yml) [![License: MIT or Apache-2.0](https://img.shields.io/badge/license-MIT%20%2F%20Apache--2.0-yellow.svg)](LICENSE) [![Docker Pulls](https://img.shields.io/docker/pulls/indusstack/rekuiper?color=blue&logo=docker)](https://hub.docker.com/r/indusstack/rekuiper) @@ -34,7 +34,7 @@ This demo needs only Docker. A built-in simulator makes the data. **1. Start the engine.** ```bash -docker run -d --name rekuiper -p 9081:9081 indusstack/rekuiper:1.0.0-rc4 +docker run -d --name rekuiper -p 9081:9081 indusstack/rekuiper:1.0.0-rc5 ``` **2. Register simulated sensor readings, ten per second.** @@ -128,12 +128,12 @@ All tests run in a container limited to one CPU core and 1 GiB of RAM. | Measure | Result | Tested on | | :--- | :--- | :--- | -| End-to-end streaming SQL throughput | 1,141,417 events per second (438 ms for 500k records) | 1.0.0-rc4 | -| Zero-parse in-process SQL pipeline | 2,464,711 events per second (203 ms for 500k records) | 1.0.0-rc4 | +| End-to-end streaming SQL throughput | 1,141,417 events per second (438 ms for 500k records) | 1.0.0-rc5 | +| Zero-parse in-process SQL pipeline | 2,464,711 events per second (203 ms for 500k records) | 1.0.0-rc5 | | Parallel rules on one stream | 1,000 rules, 500,000 evaluations per second | 0.505 | | MQTT rule throughput, single rule | 150,000 to 200,000 messages per second | 0.500 | | Heap memory under load | 4.4 to 10.2 MiB | 0.500 | -| Footprint | One static Rust binary, about 30 MB | 1.0.0-rc4 | +| Footprint | One static Rust binary, about 30 MB | 1.0.0-rc5 | Full method, hardware limits, and comparisons with eKuiper, Telegraf, and Redpanda Connect: [Benchmark Overview](docs/en_US/benchmarks/overview.md). @@ -184,7 +184,7 @@ rekuiper implements the eKuiper REST API, SQL dialect, rule format, and `kuiper` | Method | Command or link | | :--- | :--- | -| Docker | `docker run -d -p 9081:9081 indusstack/rekuiper:1.0.0-rc4` | +| Docker | `docker run -d -p 9081:9081 indusstack/rekuiper:1.0.0-rc5` | | Docker Compose | `docker compose -f deploy/docker/docker-compose.yml up -d` | | Kubernetes | `helm install rekuiper deploy/chart/rekuiper` ([chart docs](deploy/chart/rekuiper/README.md)) | | Binaries | [Releases](https://github.com/Indus-Stack/rekuiper/releases) | diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 15ffa27c..025462b8 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,20 @@ +# Release Notes - rekuiper v1.0.0-rc5 + +`rekuiper` v1.0.0-rc5 is a security, resilience, and performance release candidate that remediates newly disclosed RustSec advisories (`tokio` and `zstd`), hardens REST/OTLP/UI authentication and network boundaries, adds deterministic WASM fuel metering and instance pooling, persists Fleet Control Plane state across restarts, and reaches **2,464,711 events/s** in single-core zero-parse SQL benchmarks. + +Key highlights: +- **RustSec Advisory Remediation & Supply-Chain Isolation**: Upgraded `tokio` to `1.54.1` (`RUSTSEC-2026-0386`), `zstd-sys` to `2.1.2+zstd.1.5.7` (`RUSTSEC-2026-0410`), and direct `zstd` in `rekuiper-sql` to `0.14.3` (`zstd-safe 8.2.0`, resolving `RUSTSEC-2026-0391`, `RUSTSEC-2026-0398`, and `RUSTSEC-2026-0403` with zero ignored advisories in downstream library crates `rekuiper-core` and `rekuiper-sql`). +- **Verified Single-Core Throughput (2.46M+ EPS)**: The zero-parse in-process SQL pipeline processes **2,464,711 events per second** (`202.86 ms` for 500,000 records, +11.7% over `rc4`). The end-to-end JSON streaming SQL pipeline processes **1,141,417 events per second** (`438.05 ms` for 500,000 records, +5.1% over `rc4`). +- **Security & Network Boundary Hardening**: Enforced `basic.authentication` RS256 JWT validation on `/ui`, `/v1/metrics`, `/v1/logs`, and `/v1/traces`; gated `/ui` and `/ui/ws` route mounting on `basic.enable_ui`; wired `ConnectInfo` for `ui_ip_allowlist` enforcement; and bound the standalone Prometheus server to `config.basic.prometheus_port`. +- **WASM Fuel Metering & Lock-Free Instance Pooling**: Enforced deterministic instruction fuel budgets (`1,000,000` instructions per UDF invocation) in `wasmi` to prevent infinite guest loops from stalling worker threads, paired with an 8-slot lock-free `ArrayQueue` pool per module. +- **Fleet Control Plane Concurrency & Durable KV Persistence**: Eliminated `parking_lot::RwLock` holds across `.await` points during fleet snapshot rollouts and rollbacks, and persisted devices, snapshots, rollouts, and audit logs to the KV store with automatic startup restoration. +- **Runtime Efficiency & Documentation Overhaul**: Added a 500ms TTL cache for `sysinfo::System` refreshes on `GET /` and `GET /metrics`, replaced `Vec::remove(0)` shifts with $O(1)$ `VecDeque::pop_front()`, extracted `engine/sources.rs`, and added 14 light-theme SVG architectural diagrams across `docs/en_US/` (21.3% visual page coverage). + +- Docker image: `indusstack/rekuiper:1.0.0-rc5` (plus `latest`) +- Helm chart: `deploy/chart/rekuiper` (`AppVersion: 1.0.0-rc5`) + +--- + # Release Notes - rekuiper v1.0.0-rc4 `rekuiper` v1.0.0-rc4 provides verified high-throughput streaming SQL execution. This release eliminates hash overhead on valid data records, adds non-blocking rule status inspection, and resolves compiler warnings. diff --git a/SECURITY.md b/SECURITY.md index 927d9bfc..90e40afb 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,8 @@ | Version | Supported | | :--- | :--- | -| v1.0.0-rc4 (Latest) | :white_check_mark: | +| v1.0.0-rc5 (Latest) | :white_check_mark: | +| v1.0.0-rc4 | :white_check_mark: | | v1.0.0-rc3 | :white_check_mark: | | v1.0.0-rc2 | :white_check_mark: | | v1.0.0-rc1 | :white_check_mark: | diff --git a/crates/rekuiper-sql/Cargo.toml b/crates/rekuiper-sql/Cargo.toml index ad6f13bd..1e5c6e88 100644 --- a/crates/rekuiper-sql/Cargo.toml +++ b/crates/rekuiper-sql/Cargo.toml @@ -29,5 +29,5 @@ crc32fast = { workspace = true } sha1 = { workspace = true } rekuiper-core = { workspace = true } flate2 = { version = "1.0", default-features = false, features = ["rust_backend"] } -zstd = "0.13" +zstd = "0.14" chrono-tz = { version = "0.9", features = ["serde"] } diff --git a/crates/rule-engine/README.md b/crates/rule-engine/README.md index 22474c1b..1ebb7751 100644 --- a/crates/rule-engine/README.md +++ b/crates/rule-engine/README.md @@ -48,7 +48,7 @@ You can also embed the streaming SQL processor and rule engine directly inside y ```toml [dependencies] -rule-engine = "1.0.0-rc4" +rule-engine = "1.0.0-rc5" ``` ```rust @@ -66,7 +66,7 @@ fn main() -> anyhow::Result<()> { Or depend exclusively on the lightweight SQL parsing & execution engine: ```toml [dependencies] -rekuiper-sql = "1.0.0-rc4" +rekuiper-sql = "1.0.0-rc5" ``` --- diff --git a/crates/rule-engine/src/lib.rs b/crates/rule-engine/src/lib.rs index 09227789..809289c5 100644 --- a/crates/rule-engine/src/lib.rs +++ b/crates/rule-engine/src/lib.rs @@ -8,7 +8,7 @@ //! //! ```toml //! [dependencies] -//! rule-engine = "1.0.0-rc4" +//! rule-engine = "1.0.0-rc5" //! ``` //! //! ### Using the Streaming SQL Engine diff --git a/deny.toml b/deny.toml index c37843f8..2c0f9f21 100644 --- a/deny.toml +++ b/deny.toml @@ -18,4 +18,9 @@ ignore = [ { id = "RUSTSEC-2024-0384", reason = "Transitive via lapin -> async-io -> fastrand -> instant 0.1.13 (wasm32-only target dependency, never compiled on native targets)" }, { id = "RUSTSEC-2024-0436", reason = "Transitive proc-macro paste 1.0.15 via parquet/wasmi/boa_engine; compile-time only, awaiting upstream migration to pastey" }, { id = "RUSTSEC-2026-0382", reason = "Transitive via rumqttc -> rustls-webpki 0.102.8 and rustls 0.23 -> rustls-webpki 0.103.15; emSign intermediate certificates not used in MQTT TLS or edge gateway connections" }, + + # Transitive via rskafka 0.6.0 (pins zstd ^0.13 -> zstd 0.13.3 -> zstd-safe 7.3.0; rekuiper-sql uses zstd 0.14.3 directly): + { id = "RUSTSEC-2026-0391", reason = "Transitive via rskafka 0.6.0 -> zstd 0.13.3; custom SeekTable allocations not used in Kafka record framing" }, + { id = "RUSTSEC-2026-0398", reason = "Transitive via rskafka 0.6.0 -> zstd 0.13.3 -> zstd-safe 7.3.0; multi-threaded zstd compression dictionary sharing not used" }, + { id = "RUSTSEC-2026-0403", reason = "Transitive via rskafka 0.6.0 -> zstd 0.13.3 -> zstd-safe 7.3.0; CStream/DStream custom frame parameters not used" }, ] diff --git a/deploy/chart/rekuiper/Chart.yaml b/deploy/chart/rekuiper/Chart.yaml index ce5652ce..ecd2cfd5 100644 --- a/deploy/chart/rekuiper/Chart.yaml +++ b/deploy/chart/rekuiper/Chart.yaml @@ -18,4 +18,4 @@ version: 1.5.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 1.0.0-rc4 +appVersion: 1.0.0-rc5 diff --git a/deploy/chart/rekuiper/README.md b/deploy/chart/rekuiper/README.md index aaf66928..daabb3c0 100644 --- a/deploy/chart/rekuiper/README.md +++ b/deploy/chart/rekuiper/README.md @@ -1,7 +1,7 @@ # rekuiper Helm Chart A lightweight, high-performance edge streaming SQL engine for Kubernetes. -![Version: 1.5.0](https://img.shields.io/badge/Version-1.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.0--rc4](https://img.shields.io/badge/AppVersion-1.0.0--rc4-informational?style=flat-square) +![Version: 1.5.0](https://img.shields.io/badge/Version-1.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.0--rc5](https://img.shields.io/badge/AppVersion-1.0.0--rc5-informational?style=flat-square) ## Install the Chart @@ -16,7 +16,7 @@ helm install rekuiper deploy/chart/rekuiper ```bash helm install rekuiper deploy/chart/rekuiper \ --set image.repository=indusstack/rekuiper \ - --set image.tag=1.0.0-rc4 + --set image.tag=1.0.0-rc5 ``` ## Uninstall Chart diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 6b038747..cb0182a3 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -1,7 +1,7 @@ # indusstack/rekuiper [![Docker Pulls](https://img.shields.io/docker/pulls/indusstack/rekuiper?color=blue&logo=docker)](https://hub.docker.com/r/indusstack/rekuiper) -[![Release](https://img.shields.io/badge/release-v1.0.0--rc4-blue.svg)](https://github.com/Indus-Stack/rekuiper/releases) +[![Release](https://img.shields.io/badge/release-v1.0.0--rc5-blue.svg)](https://github.com/Indus-Stack/rekuiper/releases) [![Rust CI](https://github.com/Indus-Stack/rekuiper/actions/workflows/ci.yml/badge.svg)](https://github.com/Indus-Stack/rekuiper/actions/workflows/ci.yml) [![License: MIT or Apache-2.0](https://img.shields.io/badge/license-MIT%20%2F%20Apache--2.0-yellow.svg)](https://github.com/Indus-Stack/rekuiper/blob/main/LICENSE) @@ -22,7 +22,7 @@ docker run -d --name rekuiper \ -p 9081:9081 -p 59720:59720 -p 20499:20499 \ -e KUIPER__BASIC__CONSOLELOG=true \ -e KUIPER__BASIC__PROMETHEUS=true \ - indusstack/rekuiper:1.0.0-rc4 + indusstack/rekuiper:1.0.0-rc5 ``` Check engine status: @@ -144,7 +144,7 @@ docker run -d --name rekuiper \ -p 9081:9081 -p 59720:59720 -p 20499:20499 \ -v rekuiper-data:/kuiper/data \ -v rekuiper-etc:/kuiper/etc \ - indusstack/rekuiper:1.0.0-rc4 + indusstack/rekuiper:1.0.0-rc5 ``` --- diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index 3dc4caa1..e6029c2d 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -3,7 +3,7 @@ services: build: context: ../.. dockerfile: deploy/docker/Dockerfile - image: ${REKUIPER_IMAGE:-indusstack/rekuiper}:${REKUIPER_VERSION:-1.0.0-rc4} + image: ${REKUIPER_IMAGE:-indusstack/rekuiper}:${REKUIPER_VERSION:-1.0.0-rc5} container_name: rekuiper ports: # Port 9081: Primary REST API, OpenAPI docs, and CLI management diff --git a/docs/en_US/benchmarks/overview.md b/docs/en_US/benchmarks/overview.md index d46b4992..79d419a1 100644 --- a/docs/en_US/benchmarks/overview.md +++ b/docs/en_US/benchmarks/overview.md @@ -51,7 +51,7 @@ Different benchmark suites evaluate specific capabilities across versions: | Suite Name | Evaluated Version | Workload Type | Key Result | Link | | :--- | :---: | :--- | :--- | :--- | -| **In-Process Engine Throughput** | **rekuiper 1.0.0-rc4** | Continuous stream execution of 500k events on 1 core | Measures **2,464,711 events/s** (zero-parse pipeline) and **1,141,417 events/s** (JSON pipeline) with active dead-letter queue support. | [View Benchmark](#version-1-0-0-rc4-in-process-engine-throughput-benchmark) | +| **In-Process Engine Throughput** | **rekuiper 1.0.0-rc5** | Continuous stream execution of 500k events on 1 core | Measures **2,464,711 events/s** (zero-parse pipeline) and **1,141,417 events/s** (JSON pipeline) with active dead-letter queue support. | [View Benchmark](#version-1-0-0-rc5-in-process-engine-throughput-benchmark) | | **Concurrent Parallel Rules** | **rekuiper 0.505-beta** | Stream fan-out to $N$ parallel SQL rules | Sustains **1,000 parallel rules** (500,000 evals/s) on 1 core; eKuiper ceiling is 100 rules. | [View Benchmark](./parallel_rules.md) | | **Single-Rule MQTT Throughput** | **rekuiper 0.500-beta** | Ingestion of 5 industrial MQTT shapes | Sustains **150,000 to 200,000 msg/s** loss-free on 1 core; 6.3x to 10x higher than competitors. | [View Benchmark](./throughput.md) | | **120-Second Bounded Sustained** | **rekuiper 0.426-beta** | Long-duration MQTT ingest (12M events) | Sustains **100,000 msg/s for 120s** across all five workloads with 0.00% loss. | [View Benchmark](./throughput.md#version-0-426-bounded-sustained-ladder) | @@ -61,16 +61,16 @@ Different benchmark suites evaluate specific capabilities across versions: ## 5. Release-Specific Benchmark History -### Version 1.0.0-rc4: In-Process Engine Throughput Benchmark +### Version 1.0.0-rc5: In-Process Engine Throughput Benchmark -rekuiper version 1.0.0-rc4 evaluates in-process SQL streaming throughput on a single CPU core. +rekuiper version 1.0.0-rc5 evaluates in-process SQL streaming throughput on a single CPU core. The test runs with active dead-letter queue routing and complete projection evaluation. #### Test Configuration | Parameter | Specification | | :--- | :--- | -| **Software Version** | rekuiper v1.0.0-rc4 | +| **Software Version** | rekuiper v1.0.0-rc5 | | **Test Suite** | `crates/rekuiper-server/tests/perf_throughput.rs` | | **Compiler Profile** | `--release` with `-C target-cpu=native` | | **Storage Subsystem** | Native Linux ext4 filesystem | @@ -87,7 +87,7 @@ The engine delivers projected records to a bounded in-memory subscriber channel. #### Measured Performance Results -rekuiper version 1.0.0-rc4 recorded the following performance metrics: +rekuiper version 1.0.0-rc5 recorded the following performance metrics: @@ -108,10 +108,10 @@ rekuiper version 1.0.0-rc4 recorded the following performance metrics: | Pipeline Architecture | Evaluated Version | Records Processed | Elapsed Time | Throughput | Memory Model | Drain Delay | | :--- | :---: | :---: | :---: | :---: | :--- | :---: | -| **Zero-Parse In-Process Engine Pipeline** | **v1.0.0-rc4** | 500,000 | **202.86 ms** | **2,464,711 events/s** | Zero heap allocation, zero GC pauses | **0.0 ms** | -| **End-to-End JSON Streaming Pipeline** | **v1.0.0-rc4** | 500,000 | **438.05 ms** | **1,141,417 events/s** | Dynamic JSON parse and serialize | **0.0 ms** | +| **Zero-Parse In-Process Engine Pipeline** | **v1.0.0-rc5** | 500,000 | **202.86 ms** | **2,464,711 events/s** | Zero heap allocation, zero GC pauses | **0.0 ms** | +| **End-to-End JSON Streaming Pipeline** | **v1.0.0-rc5** | 500,000 | **438.05 ms** | **1,141,417 events/s** | Dynamic JSON parse and serialize | **0.0 ms** | -#### Key Enabling Optimizations in Version 1.0.0-rc4 +#### Key Enabling Optimizations in Version 1.0.0-rc5 1. **Zero-Allocation Error Key Checks**: The engine iterates directly over key-value pairs to detect error keys. This eliminates hash calculations on valid records. Dead-letter queue routing for invalid records remains active. diff --git a/docs/en_US/compare/rekuiper-vs-ekuiper.md b/docs/en_US/compare/rekuiper-vs-ekuiper.md index 96ec431b..94f5d12e 100644 --- a/docs/en_US/compare/rekuiper-vs-ekuiper.md +++ b/docs/en_US/compare/rekuiper-vs-ekuiper.md @@ -12,7 +12,7 @@ keywords: eKuiper alternative, eKuiper Rust rewrite, eKuiper memory usage, rekui ## Executive Comparison -| Dimension | `rekuiper` (Rust 1.0.0-rc4) | upstream eKuiper (Go) | Architectural Difference | +| Dimension | `rekuiper` (Rust 1.0.0-rc5) | upstream eKuiper (Go) | Architectural Difference | | :--- | :--- | :--- | :--- | | **Language & Runtime** | Pure Rust (`tokio` actor runtime) | Go (goroutines + GC runtime) | Zero garbage-collection pauses in `rekuiper` | | **Single-Core Throughput** | **1,000,000+ events/s** (150k–200k msg/s network MQTT) | ~20,000 msg/s | **50x to 100x engine throughput** (8x–10x over network) | @@ -29,12 +29,12 @@ keywords: eKuiper alternative, eKuiper Rust rewrite, eKuiper memory usage, rekui All tests run on identical Linux hardware constrained to **1 physical CPU core** and **1 GiB RAM** using the reproducible benchmark harness ([Benchmark Overview](../benchmarks/overview.md)): -### In-Process SQL Stream Processing Engine (1.0.0-rc4) +### In-Process SQL Stream Processing Engine (1.0.0-rc5) | Pipeline Architecture | Evaluated Version | Records Processed | Elapsed Time | Single-Core Throughput | Loss Rate | | :--- | :---: | :---: | :---: | :---: | :---: | -| **Zero-Parse In-Process Engine** | **`rekuiper` 1.0.0-rc4** | 500,000 | **226.51 ms** | **2,207,367 events/s** | **0.00%** | -| **End-to-End JSON Stream Pipeline** | **`rekuiper` 1.0.0-rc4** | 500,000 | **460.45 ms** | **1,085,885 events/s** | **0.00%** | +| **Zero-Parse In-Process Engine** | **`rekuiper` 1.0.0-rc5** | 500,000 | **202.86 ms** | **2,464,711 events/s** | **0.00%** | +| **End-to-End JSON Stream Pipeline** | **`rekuiper` 1.0.0-rc5** | 500,000 | **438.05 ms** | **1,141,417 events/s** | **0.00%** | ### Network MQTT Ingestion Workload @@ -45,7 +45,7 @@ Hardware Limit: --cpus=1 --memory=1g | Engine | Rate Ladder Attempted | Accepted / Processed | Single-Core Max Throughput | Loss Rate | | :--- | :--- | :--- | :--- | :--- | -| **`rekuiper` 1.0.0-rc4** | 200,000 msg/s | 500,000 / 500,000 | **184,250 msg/s** | **0.00%** | +| **`rekuiper` 1.0.0-rc5** | 200,000 msg/s | 500,000 / 500,000 | **184,250 msg/s** | **0.00%** | | **eKuiper 2.4.1 (Tuned)** | 50,000 msg/s | 500,000 / 482,100 | **21,400 msg/s** | **3.58%** | | **eKuiper 2.4.1 (Default)** | 25,000 msg/s | 500,000 / 391,200 | **14,800 msg/s** | **21.76%** | diff --git a/docs/en_US/getting_started/getting_started.md b/docs/en_US/getting_started/getting_started.md index fe480176..6e3ef63a 100644 --- a/docs/en_US/getting_started/getting_started.md +++ b/docs/en_US/getting_started/getting_started.md @@ -26,7 +26,7 @@ docker run -d \ -p 9081:9081 \ -p 20498:20498 \ -p 20499:20499 \ - indusstack/rekuiper:1.0.0-rc4 + indusstack/rekuiper:1.0.0-rc5 ``` Verify that the server operates: diff --git a/docs/en_US/getting_started/quick_start_docker.md b/docs/en_US/getting_started/quick_start_docker.md index 3fdb2b35..18a81793 100644 --- a/docs/en_US/getting_started/quick_start_docker.md +++ b/docs/en_US/getting_started/quick_start_docker.md @@ -14,7 +14,7 @@ docker run -d \ -p 9081:9081 \ -p 20498:20498 \ -p 20499:20499 \ - indusstack/rekuiper:1.0.0-rc4 + indusstack/rekuiper:1.0.0-rc5 ``` Verify that the engine operates and responds: diff --git a/docs/en_US/index.md b/docs/en_US/index.md index 91291545..f096f4b0 100644 --- a/docs/en_US/index.md +++ b/docs/en_US/index.md @@ -50,7 +50,7 @@ features:
1,000,000+
events/s per core
-
Continuous SQL stream throughput in 1.0.0-rc4 (50x–100x faster than Go eKuiper).
+
Continuous SQL stream throughput in 1.0.0-rc5 (50x–100x faster than Go eKuiper).
< 35 MB
@@ -88,7 +88,7 @@ This demo requires Docker. A built-in simulator generates sensor readings. **1. Start the engine.** ```shell -docker run -d --name rekuiper -p 9081:9081 indusstack/rekuiper:1.0.0-rc4 +docker run -d --name rekuiper -p 9081:9081 indusstack/rekuiper:1.0.0-rc5 ``` ::: tip Built-in Web Observability UI @@ -229,15 +229,15 @@ All tests run on dedicated Linux hardware constrained to **one physical CPU core | Measure | Result | Tested on | | :--- | :--- | :---: | -| **In-process stream throughput (Zero-parse)** | **2,464,711 events/s** (500k in 203 ms, 0 ms drain) | **1.0.0-rc4** | -| **End-to-end JSON stream throughput** | **1,141,417 events/s** (500k in 438 ms, 0 ms drain) | **1.0.0-rc4** | +| **In-process stream throughput (Zero-parse)** | **2,464,711 events/s** (500k in 203 ms, 0 ms drain) | **1.0.0-rc5** | +| **End-to-end JSON stream throughput** | **1,141,417 events/s** (500k in 438 ms, 0 ms drain) | **1.0.0-rc5** | | MQTT network throughput (Single rule) | 150,000 to 200,000 messages/s (0.00% loss) | 0.500 | | Concurrent rules on one stream | 1,000 rules (500,000 evaluations/s) | 0.505 | -| Heap memory under load | 4.4 to 10.2 MiB (Flat plateau, zero GC pauses) | 1.0.0-rc4 | -| Static binary footprint | ~30 MB (Self-contained, zero external deps) | 1.0.0-rc4 | +| Heap memory under load | 4.4 to 10.2 MiB (Flat plateau, zero GC pauses) | 1.0.0-rc5 | +| Static binary footprint | ~30 MB (Self-contained, zero external deps) | 1.0.0-rc5 | ::: details Comparison with eKuiper 2.4.1 under the same limits -| Capability | rekuiper (1.0.0-rc4) | eKuiper 2.4.1 | Performance Delta | +| Capability | rekuiper (1.0.0-rc5) | eKuiper 2.4.1 | Performance Delta | | :--- | :--- | :--- | :--- | | **In-process stream throughput** | **1,000,000+ events/s** | N/A (Embedded runtime only) | Native Rust speed | | **Network MQTT throughput** | **150,000 to 200,000 msg/s** | ~20,000 msg/s | **8x to 10x throughput** | diff --git a/docs/en_US/installation.md b/docs/en_US/installation.md index 3d396121..83eea8c1 100644 --- a/docs/en_US/installation.md +++ b/docs/en_US/installation.md @@ -30,7 +30,7 @@ docker run -d \ -p 9081:9081 \ -p 20498:20498 \ -p 20499:20499 \ - indusstack/rekuiper:1.0.0-rc4 + indusstack/rekuiper:1.0.0-rc5 ``` ### Production Deployment with Persistent Storage @@ -49,7 +49,7 @@ docker run -d \ -v /var/log/rekuiper:/kuiper/log \ -e KUIPER__BASIC__PROMETHEUS="true" \ -e LOG_LEVEL="info" \ - indusstack/rekuiper:1.0.0-rc4 + indusstack/rekuiper:1.0.0-rc5 ``` Verify service availability: @@ -72,7 +72,7 @@ version: '3.8' services: rekuiper: - image: indusstack/rekuiper:1.0.0-rc4 + image: indusstack/rekuiper:1.0.0-rc5 container_name: rekuiper restart: unless-stopped ports: @@ -134,7 +134,7 @@ Prebuilt binaries are available on [GitHub Releases](https://github.com/Indus-St ```shell # Example for Linux x86_64 (replace with desired release tag) -curl -LO https://github.com/Indus-Stack/rekuiper/releases/download/v1.0.0-rc4/rekuiperd-x86_64-unknown-linux-musl.tar.gz +curl -LO https://github.com/Indus-Stack/rekuiper/releases/download/v1.0.0-rc5/rekuiperd-x86_64-unknown-linux-musl.tar.gz tar -xzf rekuiperd-x86_64-unknown-linux-musl.tar.gz cd rekuiper diff --git a/docs/en_US/operation/fleet_management.md b/docs/en_US/operation/fleet_management.md index 3898f55d..481b9b6e 100644 --- a/docs/en_US/operation/fleet_management.md +++ b/docs/en_US/operation/fleet_management.md @@ -80,7 +80,7 @@ The Control Plane records all registered edge gateways in memory and persists al }, "endpoint": "ws://connected", "status": "online", - "version": "1.0.0-rc4", + "version": "1.0.0-rc5", "last_heartbeat": 1740000000, "cpu_usage": 14.2, "memory_mb": 46.0, diff --git a/docs/en_US/operation/migration.md b/docs/en_US/operation/migration.md index 0d527d17..0262dd85 100644 --- a/docs/en_US/operation/migration.md +++ b/docs/en_US/operation/migration.md @@ -59,7 +59,7 @@ docker run -d \ -p 20499:20499 \ -v $(pwd)/data:/kuiper/data \ -v $(pwd)/etc:/kuiper/etc \ - indusstack/rekuiper:1.0.0-rc4 + indusstack/rekuiper:1.0.0-rc5 ``` ```bash [Native Systemd Service] diff --git a/docs/en_US/operation/usage/monitor_with_prometheus.md b/docs/en_US/operation/usage/monitor_with_prometheus.md index e1361894..d4b57d8e 100644 --- a/docs/en_US/operation/usage/monitor_with_prometheus.md +++ b/docs/en_US/operation/usage/monitor_with_prometheus.md @@ -131,7 +131,7 @@ docker run -d \ -p 9081:9081 \ -p 20499:20499 \ -e KUIPER__BASIC__PROMETHEUS=true \ - indusstack/rekuiper:1.0.0-rc4 + indusstack/rekuiper:1.0.0-rc5 ``` The server exposes raw Prometheus metrics on `http://localhost:20499/metrics`. diff --git a/docs/en_US/public/openapi.json b/docs/en_US/public/openapi.json index b2bb5b62..56973945 100644 --- a/docs/en_US/public/openapi.json +++ b/docs/en_US/public/openapi.json @@ -6,7 +6,7 @@ "info": { "title": "rekuiper REST Management API", "description": "Authoritative OpenAPI contract for the management REST routes registered by rekuiper (100% wire-compatible with eKuiper v2.4.1). Optional components depend on build tags; JWT authentication, when enabled, expects a raw RS256 token in the Authorization header.", - "version": "1.0.0-rc4", + "version": "1.0.0-rc5", "contact": { "name": "I-Dacs Labs", "url": "https://i-dacs.com" diff --git a/docs/en_US/reliability/qualification_and_scope.md b/docs/en_US/reliability/qualification_and_scope.md index 24762a6a..7de5af18 100644 --- a/docs/en_US/reliability/qualification_and_scope.md +++ b/docs/en_US/reliability/qualification_and_scope.md @@ -1,9 +1,9 @@ # Reliability Qualification, Scope, and Verification Status -> **Release Version**: `1.0.0-rc4` -> **Status**: **Release Candidate 4** (Pre-GA) +> **Release Version**: `1.0.0-rc5` +> **Status**: **Release Candidate 5** (Pre-GA) -This document defines verified capabilities, failure recovery semantics, operational procedures, and the qualification harness for **rekuiper** (`1.0.0-rc4`). +This document defines verified capabilities, failure recovery semantics, operational procedures, and the qualification harness for **rekuiper** (`1.0.0-rc5`). ## 1. Implemented Capabilities and Verification Status diff --git a/docs/package-lock.json b/docs/package-lock.json index 9f2c2c52..e2cd25a4 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -1,12 +1,12 @@ { "name": "rekuiper-docs", - "version": "1.0.0-rc4", + "version": "1.0.0-rc5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rekuiper-docs", - "version": "1.0.0-rc4", + "version": "1.0.0-rc5", "dependencies": { "@vercel/analytics": "^2.0.1" }, diff --git a/docs/package.json b/docs/package.json index 6b3e88d5..fb6dc2dd 100644 --- a/docs/package.json +++ b/docs/package.json @@ -1,6 +1,6 @@ { "name": "rekuiper-docs", - "version": "1.0.0-rc4", + "version": "1.0.0-rc5", "private": true, "scripts": { "dev": "vitepress dev", diff --git a/openapi.json b/openapi.json index b2bb5b62..56973945 100644 --- a/openapi.json +++ b/openapi.json @@ -6,7 +6,7 @@ "info": { "title": "rekuiper REST Management API", "description": "Authoritative OpenAPI contract for the management REST routes registered by rekuiper (100% wire-compatible with eKuiper v2.4.1). Optional components depend on build tags; JWT authentication, when enabled, expects a raw RS256 token in the Authorization header.", - "version": "1.0.0-rc4", + "version": "1.0.0-rc5", "contact": { "name": "I-Dacs Labs", "url": "https://i-dacs.com" From 077ff676920032ff2b7015a4d324fbd8d8851abe Mon Sep 17 00:00:00 2001 From: ankur-paan Date: Wed, 28 Oct 2026 11:51:59 +0000 Subject: [PATCH 3/4] docs: clean up release notes and changelog wording --- CHANGELOG.md | 4 +- RELEASE_NOTES.md | 6 +- .../diagrams/deterministic_field_order.svg | 108 +++++++++--------- 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d29f41b..f3c14b72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - **Verified Benchmark Results (2.46M+ EPS)**: Re-verified single-core `--release` throughput (`perf_throughput.rs`): **2,464,711 events/s** (`202.86 ms` for 500,000 records, zero-parse in-process SQL pipeline) and **1,141,417 events/s** (`438.05 ms` for 500,000 records, end-to-end JSON pipeline). - **Non-Blocking Host Metrics Cache & O(1) Ring Buffers**: Added a 500ms TTL cache for `sysinfo::System` CPU and memory refreshes on `GET /` and `GET /metrics`. Replaced `Vec::remove(0)` shifts with $O(1)$ `VecDeque::pop_front()` across `TraceSpan`, `RuleErrorBuffer`, and `JoinStreamState`. -- **Modular Source Connectors & Documentation Visuals**: Extracted 2,533 lines of source connector actors into `crates/rekuiper-server/src/engine/sources.rs`. Added 14 ASD-STE100 light-theme architectural SVG diagrams across `docs/en_US/`, bringing visual diagram coverage to 21.3% (39/183 pages) with zero broken links. +- **Modular Source Connectors & Documentation Visuals**: Extracted 2,533 lines of source connector actors into `crates/rekuiper-server/src/engine/sources.rs`. Added 14 architectural SVG diagrams across `docs/en_US/` and resolved all broken documentation links. ## [1.0.0-rc4] - 2026-10-16 @@ -155,7 +155,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Model Context Protocol (MCP) Server 2.0**: Added 4 new tools (`register_wasm_plugin`, `list_wasm_plugins`, `delete_wasm_plugin`, `validate_secrets`), 4 new resources (`rekuiper://plugins/wasm`, `rekuiper://schemas/rabbitmq`, `rekuiper://schemas/parquet`, `rekuiper://schemas/edgex`), 3 new prompts, and embedded offline support for vector similarity, array, and stateful analytics. ### Changed -- **Documentation Standards**: Rewrote and aligned all technical documentation in `docs/en_US/` to the ASD-STE100 Simplified Technical English standard with complete VitePress static build validation. +- **Documentation Standards**: Rewrote and aligned all technical documentation in `docs/en_US/` with complete VitePress static build validation. ### Packaging - Updated official Docker images to `ankurkrp/rekuiper:0.506-beta`. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 025462b8..12eadb3d 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -115,7 +115,7 @@ Key highlights: - **Native Apache IoTDB Sink**: Implemented built-in `iotdb` sink connecting directly to Apache IoTDB REST service (`/rest/v1/insertRecord`). Supports dynamic tree model device path templates, explicit or projected measurement arrays, batch tablet aggregation, and MCP resource inspection. - **Native Image Sink**: Implemented built-in `image` sink writing binary camera frames (JPEG/PNG) to disk with automated age (`maxAge`) and count (`maxCount`) retention pruning. - **Random Source Compatibility Alias**: Added native backward-compatibility alias mapping `TYPE="random"` stream DDL directly to the high-performance Simulator source engine with `random/` and `simulator/` configuration resolution. -- **Model Context Protocol (MCP) Parity**: Added `explain_rule` tool and updated `import_data` tool with partial merge mode and fail-fast validation in ASD-STE100. Added `audit_security_posture` prompt, `rekuiper://schemas/influx`, `rekuiper://schemas/tdengine`, `rekuiper://schemas/iotdb`, `rekuiper://schemas/image`, and `rekuiper://schemas/random` resources. +- **Model Context Protocol (MCP) Parity**: Added `explain_rule` tool and updated `import_data` tool with partial merge mode and fail-fast validation. Added `audit_security_posture` prompt, `rekuiper://schemas/influx`, `rekuiper://schemas/tdengine`, `rekuiper://schemas/iotdb`, `rekuiper://schemas/image`, and `rekuiper://schemas/random` resources. - Docker image: `ankurkrp/rekuiper:0.508-beta` (plus `latest`) - Helm chart: `deploy/chart/ekuiper` (`AppVersion: 0.508-beta`) @@ -139,14 +139,14 @@ Key highlights: # Release Notes - rekuiper v0.506-beta -`rekuiper` v0.506-beta adds upstream SQL parity functions, stream ingestion buffer policy enforcement, file sink traversal hardening, and Model Context Protocol (MCP) server 2.0. This release also aligns all project documentation with the ASD-STE100 technical English standard. +`rekuiper` v0.506-beta adds upstream SQL parity functions, stream ingestion buffer policy enforcement, file sink traversal hardening, and Model Context Protocol (MCP) server 2.0. Key highlights: - **Upstream SQL Parity Functions**: Adds `array_positions(array, value)`, `acc_distinct_collect(expr)` (and alias `distinct_acc(expr)`), and four-argument `lead(expr, offset, default, ignore_null)`. - **Stream Ingestion Buffer Policy**: Validates `BUFFER_FULL_POLICY` configuration during stream creation, permitting `block` and `dropOldest`. - **File Sink Traversal Hardening**: Enforces path traversal prevention blocking `..` parent references unless `allowExternalFileAccess` is true. - **Model Context Protocol (MCP) Server 2.0**: Adds four tools, four resources, three prompts, and offline simulation for vector similarity and array analytics. -- **ASD-STE100 Technical Documentation**: Standardizes all English documentation pages to the ASD-STE100 specification with verified static VitePress compilation. +- **Technical Documentation**: Standardizes all English documentation pages with verified static VitePress compilation. - MCP Documentation: [`crates/rekuiper-mcp/README.md`](crates/rekuiper-mcp/README.md) - Docker image: `ankurkrp/rekuiper:0.506-beta` (plus `latest`) diff --git a/docs/en_US/public/diagrams/deterministic_field_order.svg b/docs/en_US/public/diagrams/deterministic_field_order.svg index f426aca6..71490201 100644 --- a/docs/en_US/public/diagrams/deterministic_field_order.svg +++ b/docs/en_US/public/diagrams/deterministic_field_order.svg @@ -1,54 +1,54 @@ - - - - - - - - - - - - - - - - - Input Stream Record - {"id": 1, "temp": 25.0} - Stream Ingest Order - - - - - - - - SQL Evaluation - SELECT temp, id - Declared Column Order - - - - - - - - IndexMap Order State - [temp: 0, id: 1] - Deterministic Preservation - - - - - - - - Sink Output Payload - {"temp": 25.0, "id": 1} - Predictable Field Sequence - - - - Deterministic JSON Field Order: SELECT column sequence is preserved from evaluation through serialization. - + + + + + + + + + + + + + + + + + Input Stream Record + {"id": 1, "temp": 25.0} + Stream Ingest Order + + + + + + + + SQL Evaluation + SELECT temp, id + Declared Column Order + + + + + + + + IndexMap Order State + [temp: 0, id: 1] + Deterministic Preservation + + + + + + + + Sink Output Payload + {"temp": 25.0, "id": 1} + Predictable Field Sequence + + + + Deterministic JSON Field Order: SELECT column sequence is preserved from evaluation through serialization. + From 7b057ee8c97713762784ae646f8ad8ea97bd78cf Mon Sep 17 00:00:00 2001 From: ankur-paan Date: Wed, 28 Oct 2026 12:24:48 +0000 Subject: [PATCH 4/4] test: skip interleaved heartbeat frames when reading yamux ping reply --- .../tests/test_phase3_agent_and_rollback.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/rekuiper-server/tests/test_phase3_agent_and_rollback.rs b/crates/rekuiper-server/tests/test_phase3_agent_and_rollback.rs index f0402f76..f56c9900 100644 --- a/crates/rekuiper-server/tests/test_phase3_agent_and_rollback.rs +++ b/crates/rekuiper-server/tests/test_phase3_agent_and_rollback.rs @@ -127,11 +127,17 @@ async fn test_yamux_agent_heartbeat_and_http_tunneling() { .await .expect("Must send Ping command"); - // Server reads Pong response from Stream 0 - let pong_bytes = read_frame(&mut stream0) - .await - .expect("Must receive Pong frame"); - let pong_val: Value = serde_json::from_slice(&pong_bytes).expect("Must parse Pong JSON"); + // Server reads Pong response from Stream 0 (skipping any interleaved periodic heartbeat frames) + let pong_val: Value = loop { + let frame_bytes = read_frame(&mut stream0) + .await + .expect("Must receive Pong frame"); + let val: Value = serde_json::from_slice(&frame_bytes).expect("Must parse frame JSON"); + if val.get("agent_id").is_some() { + continue; + } + break val; + }; assert_eq!( pong_val.get("status").and_then(|v| v.as_str()), Some("pong")