From 607d9a5abdcc588470fcd8ace0197c6bca55984e Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Fri, 3 Jul 2026 10:07:37 +0200 Subject: [PATCH 01/21] Scaffold Opc.Ua.PubSub.Kafka library (#3942) Add the Apache Kafka PubSub transport library skeleton per Part 14 Annex B.2: - Opc.Ua.PubSub.Kafka.csproj (LibxTargetFrameworks; deliberately no IsAotCompatible because Confluent.Kafka/native librdkafka is not NativeAOT/trim-safe). - KafkaProfiles (pubsub-kafka-uadp / pubsub-kafka-json), kept library-local like EthProfiles. - Confluent.Kafka 2.15.0 (Apache-2.0) pinned in Directory.Packages.props. - NugetREADME + CLSCompliant assembly attribute; wired into UA.slnx. Builds clean on net10 and net48 (net462 Confluent asset resolves under the existing net4x SuppressTfmSupportBuildWarnings). Transport implementation follows. --- Directory.Packages.props | 8 +++ .../Opc.Ua.PubSub.Kafka/KafkaProfiles.cs | 57 +++++++++++++++++++ Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md | 38 +++++++++++++ .../Opc.Ua.PubSub.Kafka.csproj | 37 ++++++++++++ .../Properties/AssemblyInfo.cs | 32 +++++++++++ UA.slnx | 2 + 6 files changed, 174 insertions(+) create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaProfiles.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/Opc.Ua.PubSub.Kafka.csproj create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/Properties/AssemblyInfo.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 8dcf49f2d1..6587048a98 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -130,4 +130,12 @@ + + + + diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaProfiles.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaProfiles.cs new file mode 100644 index 0000000000..1d52618643 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaProfiles.cs @@ -0,0 +1,57 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// PubSub transport profile identifiers defined by the Apache Kafka + /// transport, kept with the transport implementation rather than in + /// the core profile constants (mirrors EthProfiles). + /// + public static class KafkaProfiles + { + /// + /// Uri for the "PubSub Kafka UADP" Profile. This PubSub transport + /// Facet combines the OPC UA Part 14 Annex B.2 Apache Kafka + /// transport protocol mapping with UADP message mapping and is used + /// for broker-based messaging. + /// + public const string PubSubKafkaUadpTransport + = "http://opcfoundation.org/UA-Profile/Transport/pubsub-kafka-uadp"; + + /// + /// Uri for the "PubSub Kafka JSON" Profile. This PubSub transport + /// Facet combines the OPC UA Part 14 Annex B.2 Apache Kafka + /// transport protocol mapping with JSON message mapping and is used + /// for broker-based messaging. + /// + public const string PubSubKafkaJsonTransport + = "http://opcfoundation.org/UA-Profile/Transport/pubsub-kafka-json"; + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md b/Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md new file mode 100644 index 0000000000..c1cf3b3115 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md @@ -0,0 +1,38 @@ +# OPC UA .NET Standard — PubSub Apache Kafka transport + +`OPCFoundation.NetStandard.Opc.Ua.PubSub.Kafka` provides the Apache Kafka broker +transport (OPC UA Part 14 Annex B.2, with SASL/TLS security, configurable delivery +guarantees, and both the UADP and JSON message mappings) for the modern +`OPCFoundation.NetStandard.Opc.Ua.PubSub` stack. Kafka consumer groups and idempotent +producers back the high-availability publisher/subscriber deployments. + +## Getting started + +Register the transport on the PubSub builder: + +```csharp +using Microsoft.Extensions.DependencyInjection; + +builder.Services.AddOpcUa() + .AddPubSub(pubsub => pubsub + .AddSubscriber() + .AddKafkaTransport()); +``` + +Connection addresses use `kafka://host:9092` (plain/SASL) or `kafkas://host:9093` (TLS). + +## Target frameworks + +`net472`, `net48`, `netstandard2.1`, `net8.0`, `net9.0`, `net10.0`. + +## NativeAOT + +This transport wraps the native `librdkafka` client (via Confluent.Kafka) and is therefore +**not** NativeAOT/trimming compatible. Use a JIT-compiled host for Kafka deployments; the +other PubSub transports (UDP, Ethernet, MQTT) remain AOT-compatible. + +## Additional documentation + +See the [PubSub Kafka documentation](https://github.com/OPCFoundation/UA-.NETStandard/blob/master/Docs/PubSubKafka.md) +and the [PubSub documentation](https://github.com/OPCFoundation/UA-.NETStandard/blob/master/Docs/PubSub.md) +for transports, encodings, security, high availability, and the fluent / DI API. diff --git a/Libraries/Opc.Ua.PubSub.Kafka/Opc.Ua.PubSub.Kafka.csproj b/Libraries/Opc.Ua.PubSub.Kafka/Opc.Ua.PubSub.Kafka.csproj new file mode 100644 index 0000000000..fe0425b444 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/Opc.Ua.PubSub.Kafka.csproj @@ -0,0 +1,37 @@ + + + $(AssemblyPrefix).PubSub.Kafka + $(LibxTargetFrameworks) + $(PackagePrefix).Opc.Ua.PubSub.Kafka + Opc.Ua.PubSub.Kafka + OPC UA PubSub Apache Kafka transport (Part 14 Annex B.2) class library. + true + NugetREADME.md + true + enable + $(NoWarn);CS1591 + true + + + + + + + + + + $(PackageId).Debug + + + + + + + + + + diff --git a/Libraries/Opc.Ua.PubSub.Kafka/Properties/AssemblyInfo.cs b/Libraries/Opc.Ua.PubSub.Kafka/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..7798c9bd57 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +[assembly: CLSCompliant(false)] diff --git a/UA.slnx b/UA.slnx index 9f6e0359e0..09b5783aea 100644 --- a/UA.slnx +++ b/UA.slnx @@ -72,6 +72,7 @@ + @@ -223,6 +224,7 @@ + From 7cfcb39ea275a31d76d0446aa8ab9bc39e2b6ab7 Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Fri, 3 Jul 2026 11:49:20 +0200 Subject: [PATCH 02/21] Implement Kafka PubSub transport, dual-source by TFM (#3942) Apache Kafka broker transport (Part 14 Annex B.2) mirroring the MQTT transport: value types (options, endpoint kafka://kafkas://, content-type encoding, QoS mapping, message envelope), IKafkaClientAdapter abstraction, KafkaBrokerTransport (IPubSubTransport + IPubSubTopicProvider), KafkaPubSubTransportFactory (json+uadp profiles, Broker*TransportDataType QueueName/MetaDataQueueName/auth, ISecretRegistry password), and AddKafkaTransport DI. Dual-source client by TFM via the adapter seam: - net10.0 -> Dekaf (pure-managed, no native dep) -> IsAotCompatible=true. - net472/net48/netstandard2.1/net8.0/net9.0 -> Confluent.Kafka (native librdkafka). DI registers the default IKafkaClientFactory available per TFM (#if NET10_0_OR_GREATER). Content-type record header ('application/opcua+uadp' / 'application/json') is normative per Annex B.2. QoS BrokerTransportQualityOfService -> acks/idempotence. Auth reuses the MQTT AuthenticationProfileUri/ResourceUri + secret-store pattern. Bumps the Microsoft.Extensions.*/System.* 10.0.8 family to 10.0.9 (required by the Dekaf net10 package). Adds Testcontainers 4.13.0 for the upcoming Docker Kafka integration test. Builds clean (0 warnings) on net10.0 (Dekaf/AOT), net8.0 and net48 (Confluent). --- Directory.Packages.props | 59 +- ...fkaTransportServiceCollectionExtensions.cs | 165 ++++ .../Internal/ConfluentKafkaClientAdapter.cs | 638 ++++++++++++++ .../Internal/ConfluentKafkaClientFactory.cs | 62 ++ .../Internal/DekafKafkaClientAdapter.cs | 731 ++++++++++++++++ .../Internal/DekafKafkaClientFactory.cs | 62 ++ .../Internal/IKafkaClientAdapter.cs | 113 +++ .../Internal/IKafkaClientFactory.cs | 57 ++ .../KafkaBrokerTransport.cs | 804 ++++++++++++++++++ .../KafkaConnectionOptions.cs | 292 +++++++ .../KafkaConnectionStateChangedEventArgs.cs | 76 ++ .../Opc.Ua.PubSub.Kafka/KafkaEncoding.cs | 109 +++ .../Opc.Ua.PubSub.Kafka/KafkaEndpoint.cs | 54 ++ .../KafkaEndpointParser.cs | 221 +++++ .../KafkaIncomingMessageEventArgs.cs | 71 ++ Libraries/Opc.Ua.PubSub.Kafka/KafkaMessage.cs | 71 ++ .../KafkaPubSubTransportFactory.cs | 412 +++++++++ .../KafkaQualityOfService.cs | 184 ++++ .../Opc.Ua.PubSub.Kafka/KafkaTlsOptions.cs | 89 ++ .../Opc.Ua.PubSub.Kafka/KafkaTopicOptions.cs | 56 ++ .../Opc.Ua.PubSub.Kafka.csproj | 14 +- 21 files changed, 4310 insertions(+), 30 deletions(-) create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/DependencyInjection/KafkaTransportServiceCollectionExtensions.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/Internal/ConfluentKafkaClientAdapter.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/Internal/ConfluentKafkaClientFactory.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/Internal/DekafKafkaClientAdapter.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/Internal/DekafKafkaClientFactory.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/Internal/IKafkaClientAdapter.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/Internal/IKafkaClientFactory.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaBrokerTransport.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaConnectionOptions.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaConnectionStateChangedEventArgs.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaEncoding.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaEndpoint.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaEndpointParser.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaIncomingMessageEventArgs.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaMessage.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaPubSubTransportFactory.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaQualityOfService.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaTlsOptions.cs create mode 100644 Libraries/Opc.Ua.PubSub.Kafka/KafkaTopicOptions.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 6587048a98..3dd0c17cb0 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -30,27 +30,27 @@ - - + + - - - - - - - - - + + + + + + + + + - - - - + + + + @@ -88,6 +88,8 @@ + + @@ -95,11 +97,11 @@ - + - + - + - + - + - + + [netstandard2.0, net462, net8.0, net10.0], covering the non-net10 LibxTargetFrameworks + entries through NuGet compatibility fallback. It is NOT NativeAOT/trim-safe, so it is used + by Opc.Ua.PubSub.Kafka only on the older TFMs. Apache-2.0 licensed. --> - + + + + + \ No newline at end of file diff --git a/Libraries/Opc.Ua.PubSub.Kafka/DependencyInjection/KafkaTransportServiceCollectionExtensions.cs b/Libraries/Opc.Ua.PubSub.Kafka/DependencyInjection/KafkaTransportServiceCollectionExtensions.cs new file mode 100644 index 0000000000..799d054544 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/DependencyInjection/KafkaTransportServiceCollectionExtensions.cs @@ -0,0 +1,165 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using Opc.Ua; +using Opc.Ua.PubSub.Diagnostics; +using Opc.Ua.PubSub.Kafka; +using Opc.Ua.PubSub.Kafka.Internal; +using Opc.Ua.PubSub.Transports; + +namespace Microsoft.Extensions.DependencyInjection +{ + /// + /// extensions that register the + /// Apache Kafka PubSub transport with the OPC UA PubSub DI surface. + /// + /// + /// Registers two + /// instances — one for the JSON profile and one for the UADP + /// profile — so that the runtime can match a + /// by its + /// TransportProfileUri. The supported surface hangs off + /// (returned by + /// AddPubSub(pubsub => ...)) because a transport only makes + /// sense together with the PubSub feature. Implements + /// + /// Part 14 Annex B.2 Apache Kafka transport. + /// + public static class KafkaTransportServiceCollectionExtensions + { + /// + /// Default configuration section name read by + /// . + /// + public const string DefaultConfigurationSection = "OpcUa:PubSub:Kafka"; + + /// + /// Registers both Kafka factories (JSON + UADP) and binds + /// via the optional + /// callback. + /// + /// PubSub builder. + /// Optional options callback. + public static IPubSubBuilder AddKafkaTransport( + this IPubSubBuilder builder, + Action? configure = null) + { + if (builder is null) + { + throw new ArgumentNullException(nameof(builder)); + } + if (configure is null) + { + builder.Services.AddOptions(); + } + else + { + builder.Services.AddOptions().Configure(configure); + } + RegisterShared(builder.Services); + return builder; + } + + /// + /// Registers both Kafka factories (JSON + UADP) and binds + /// from the supplied root + /// under + /// . + /// + /// PubSub builder. + /// Root configuration. + public static IPubSubBuilder AddKafkaTransport( + this IPubSubBuilder builder, + IConfiguration configuration) + { + if (builder is null) + { + throw new ArgumentNullException(nameof(builder)); + } + if (configuration is null) + { + throw new ArgumentNullException(nameof(configuration)); + } + return builder.AddKafkaTransport(configuration.GetSection(DefaultConfigurationSection)); + } + + /// + /// Registers both Kafka factories (JSON + UADP) and binds + /// from the supplied + /// section. + /// + /// PubSub builder. + /// Configuration section. + public static IPubSubBuilder AddKafkaTransport( + this IPubSubBuilder builder, + IConfigurationSection section) + { + if (builder is null) + { + throw new ArgumentNullException(nameof(builder)); + } + if (section is null) + { + throw new ArgumentNullException(nameof(section)); + } + builder.Services.AddOptions().Bind(section); + RegisterShared(builder.Services); + return builder; + } + + private static void RegisterShared(IServiceCollection services) + { +#if NET10_0_OR_GREATER + services.TryAddSingleton(); +#else + services.TryAddSingleton(); +#endif + services.Add( + ServiceDescriptor.Singleton(sp => + new KafkaPubSubTransportFactory( + KafkaProfiles.PubSubKafkaJsonTransport, + sp.GetRequiredService(), + sp.GetRequiredService>(), + sp.GetService(), + sp.GetService()))); + services.Add( + ServiceDescriptor.Singleton(sp => + new KafkaPubSubTransportFactory( + KafkaProfiles.PubSubKafkaUadpTransport, + sp.GetRequiredService(), + sp.GetRequiredService>(), + sp.GetService(), + sp.GetService()))); + } + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/Internal/ConfluentKafkaClientAdapter.cs b/Libraries/Opc.Ua.PubSub.Kafka/Internal/ConfluentKafkaClientAdapter.cs new file mode 100644 index 0000000000..f01530ec40 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/Internal/ConfluentKafkaClientAdapter.cs @@ -0,0 +1,638 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#if !NET10_0_OR_GREATER +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Confluent.Kafka; +using Microsoft.Extensions.Logging; + +namespace Opc.Ua.PubSub.Kafka.Internal +{ + /// + /// Confluent.Kafka-backed implementation of + /// . + /// + /// + /// The adapter owns a lazily created + /// (built on first produce) and a lazily created + /// plus a background consume + /// loop (built on first subscribe), so a send-only or receive-only + /// connection never instantiates the unused half. Confluent.Kafka + /// wraps native librdkafka via P/Invoke and is therefore not + /// NativeAOT/trim-safe; this transport library is intentionally + /// excluded from AOT publishing (see the project file). + /// + internal sealed class ConfluentKafkaClientAdapter : IKafkaClientAdapter + { + private const string ContentTypeHeader = "content-type"; + + private readonly ILogger m_logger; + private readonly TimeProvider m_timeProvider; + private readonly System.Threading.Lock m_sync = new(); + private readonly List m_subscribedTopics = new(); + + private KafkaConnectionOptions? m_options; + private IProducer? m_producer; + private IConsumer? m_consumer; + private CancellationTokenSource? m_loopCts; + private Task? m_consumeTask; + private bool m_connected; + private bool m_disposed; + + /// + /// Initializes a new . + /// + /// Telemetry context. + /// Clock for receive-time stamps. + public ConfluentKafkaClientAdapter(ITelemetryContext telemetry, TimeProvider timeProvider) + { + if (telemetry is null) + { + throw new ArgumentNullException(nameof(telemetry)); + } + if (timeProvider is null) + { + throw new ArgumentNullException(nameof(timeProvider)); + } + m_logger = telemetry.CreateLogger(); + m_timeProvider = timeProvider; + } + + /// + public bool IsConnected + { + get + { + lock (m_sync) + { + return m_connected; + } + } + } + + /// + public event EventHandler? IncomingMessage; + + /// + public event EventHandler? ConnectionStateChanged; + + /// + public ValueTask ConnectAsync(KafkaConnectionOptions options, CancellationToken ct) + { + if (options is null) + { + throw new ArgumentNullException(nameof(options)); + } + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ValidateCredentialTransport(options); + + bool raiseConnected; + lock (m_sync) + { + m_options = options; + m_loopCts ??= new CancellationTokenSource(); + raiseConnected = !m_connected; + m_connected = true; + } + m_logger.LogDebug( + "Kafka adapter connected to {BootstrapServers} (protocol={Protocol}).", + string.IsNullOrEmpty(options.BootstrapServers) ? options.Endpoint : options.BootstrapServers, + options.SecurityProtocol); + if (raiseConnected) + { + ConnectionStateChanged?.Invoke( + this, + new KafkaConnectionStateChangedEventArgs(isConnected: true, reason: null)); + } + return default; + } + + /// + public async ValueTask DisconnectAsync(CancellationToken ct) + { + await ShutdownAsync(raiseEvent: true).ConfigureAwait(false); + } + + /// + public ValueTask SubscribeAsync(IReadOnlyList topics, CancellationToken ct) + { + if (topics is null) + { + throw new ArgumentNullException(nameof(topics)); + } + ThrowIfDisposed(); + if (topics.Count == 0) + { + return default; + } + + lock (m_sync) + { + IConsumer consumer = EnsureConsumer(); + bool changed = false; + foreach (string topic in topics) + { + if (!m_subscribedTopics.Contains(topic)) + { + m_subscribedTopics.Add(topic); + changed = true; + } + } + if (changed) + { + consumer.Subscribe(m_subscribedTopics); + } + m_consumeTask ??= Task.Factory.StartNew( + () => ConsumeLoop(consumer, m_options!.EnableAutoCommit, m_loopCts!.Token), + m_loopCts!.Token, + TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, + TaskScheduler.Default); + } + m_logger.LogDebug("Kafka subscribed to {Count} topic(s).", topics.Count); + return default; + } + + /// + public ValueTask UnsubscribeAsync(IReadOnlyList topics, CancellationToken ct) + { + if (topics is null) + { + throw new ArgumentNullException(nameof(topics)); + } + ThrowIfDisposed(); + if (topics.Count == 0) + { + return default; + } + + lock (m_sync) + { + if (m_consumer is null) + { + return default; + } + bool changed = false; + foreach (string topic in topics) + { + if (m_subscribedTopics.Remove(topic)) + { + changed = true; + } + } + if (changed) + { + if (m_subscribedTopics.Count == 0) + { + m_consumer.Unsubscribe(); + } + else + { + m_consumer.Subscribe(m_subscribedTopics); + } + } + } + return default; + } + + /// + public async ValueTask ProduceAsync(KafkaMessage message, CancellationToken ct) + { + if (string.IsNullOrEmpty(message.Topic)) + { + throw new ArgumentException("Kafka produce requires a topic.", nameof(message)); + } + ThrowIfDisposed(); + + IProducer producer; + lock (m_sync) + { + producer = EnsureProducer(); + } + + var headers = new Headers(); + if (!string.IsNullOrEmpty(message.ContentType)) + { + headers.Add(ContentTypeHeader, System.Text.Encoding.UTF8.GetBytes(message.ContentType)); + } + if (message.Headers is not null) + { + foreach (KeyValuePair header in message.Headers) + { + headers.Add(header.Key, System.Text.Encoding.UTF8.GetBytes(header.Value)); + } + } + + var record = new Message + { + Key = message.Key.IsEmpty ? null! : message.Key.ToArray(), + Value = message.Value.IsEmpty ? Array.Empty() : message.Value.ToArray(), + Headers = headers + }; + await producer.ProduceAsync(message.Topic, record, ct).ConfigureAwait(false); + } + + /// + public async ValueTask DisposeAsync() + { + lock (m_sync) + { + if (m_disposed) + { + return; + } + m_disposed = true; + } + await ShutdownAsync(raiseEvent: false).ConfigureAwait(false); + } + + private async ValueTask ShutdownAsync(bool raiseEvent) + { + CancellationTokenSource? loopCts; + Task? consumeTask; + IConsumer? consumer; + IProducer? producer; + bool wasConnected; + lock (m_sync) + { + loopCts = m_loopCts; + m_loopCts = null; + consumeTask = m_consumeTask; + m_consumeTask = null; + consumer = m_consumer; + m_consumer = null; + producer = m_producer; + m_producer = null; + wasConnected = m_connected; + m_connected = false; + m_subscribedTopics.Clear(); + } + + if (loopCts is not null) + { + try + { + loopCts.Cancel(); + } + catch (Exception ex) + { + m_logger.LogDebug(ex, "Kafka consume loop cancellation raised an exception."); + } + } + if (consumeTask is not null) + { + try + { + await consumeTask.ConfigureAwait(false); + } + catch (Exception ex) + { + m_logger.LogDebug(ex, "Kafka consume loop terminated with an exception."); + } + } + if (consumer is not null) + { + try + { + consumer.Close(); + } + catch (Exception ex) + { + m_logger.LogDebug(ex, "Kafka consumer close raised an exception."); + } + consumer.Dispose(); + } + if (producer is not null) + { + try + { + producer.Flush(TimeSpan.FromSeconds(5)); + } + catch (Exception ex) + { + m_logger.LogDebug(ex, "Kafka producer flush raised an exception."); + } + producer.Dispose(); + } + loopCts?.Dispose(); + + if (raiseEvent && wasConnected) + { + ConnectionStateChanged?.Invoke( + this, + new KafkaConnectionStateChangedEventArgs( + isConnected: false, + reason: "Kafka adapter disconnected.")); + } + } + + private IProducer EnsureProducer() + { + if (m_producer is not null) + { + return m_producer; + } + ProducerConfig config = CreateProducerConfig(m_options!); + m_producer = new ProducerBuilder(config) + .SetLogHandler((_, message) => OnLog(message)) + .SetErrorHandler((_, error) => OnError(error)) + .Build(); + return m_producer; + } + + private IConsumer EnsureConsumer() + { + if (m_consumer is not null) + { + return m_consumer; + } + ConsumerConfig config = CreateConsumerConfig(m_options!); + m_consumer = new ConsumerBuilder(config) + .SetLogHandler((_, message) => OnLog(message)) + .SetErrorHandler((_, error) => OnError(error)) + .Build(); + return m_consumer; + } + + private void ConsumeLoop( + IConsumer consumer, + bool enableAutoCommit, + CancellationToken ct) + { + try + { + while (!ct.IsCancellationRequested) + { + ConsumeResult result; + try + { + result = consumer.Consume(ct); + } + catch (OperationCanceledException) + { + break; + } + catch (ConsumeException ex) + { + m_logger.LogWarning( + ex, + "Kafka consume error {Code}: {Reason}", + ex.Error.Code, + ex.Error.Reason); + continue; + } + if (result?.Message is null || result.IsPartitionEOF) + { + continue; + } + try + { + DispatchRecord(result); + if (!enableAutoCommit) + { + consumer.Commit(result); + } + } + catch (Exception ex) + { + m_logger.LogWarning(ex, "Failed to dispatch inbound Kafka record."); + } + } + } + catch (Exception ex) + { + m_logger.LogError(ex, "Kafka consume loop terminated unexpectedly."); + } + } + + private void DispatchRecord(ConsumeResult result) + { + string? contentType = null; + Dictionary? extraHeaders = null; + Headers headers = result.Message.Headers; + if (headers is not null && headers.Count > 0) + { + foreach (IHeader header in headers) + { + string value = System.Text.Encoding.UTF8.GetString(header.GetValueBytes()); + if (string.Equals(header.Key, ContentTypeHeader, StringComparison.OrdinalIgnoreCase)) + { + contentType = value; + continue; + } + extraHeaders ??= new Dictionary(StringComparer.Ordinal); + extraHeaders[header.Key] = value; + } + } + byte[] key = result.Message.Key ?? Array.Empty(); + byte[] value2 = result.Message.Value ?? Array.Empty(); + var message = new KafkaMessage(result.Topic, key, value2, contentType, extraHeaders); + var args = new KafkaIncomingMessageEventArgs( + message, + DateTimeUtc.From(m_timeProvider.GetUtcNow())); + IncomingMessage?.Invoke(this, args); + } + + private void OnLog(LogMessage message) + { + m_logger.LogTrace( + "librdkafka [{Facility}] {Message}", + message.Facility, + message.Message); + } + + private void OnError(Error error) + { + if (error.IsFatal) + { + m_logger.LogError("Kafka fatal error {Code}: {Reason}", error.Code, error.Reason); + } + else + { + m_logger.LogWarning("Kafka error {Code}: {Reason}", error.Code, error.Reason); + } + } + + private void ThrowIfDisposed() + { + if (m_disposed) + { + throw new ObjectDisposedException(nameof(ConfluentKafkaClientAdapter)); + } + } + + private static ProducerConfig CreateProducerConfig(KafkaConnectionOptions options) + { + var config = new ProducerConfig(); + ApplyCommonConfig(config, options); + KafkaDeliveryGuarantee guarantee = options.DeliveryGuarantee.ToDeliveryGuarantee(); + config.Acks = MapAcks(guarantee.Acks); + config.EnableIdempotence = guarantee.EnableIdempotence; + config.MessageTimeoutMs = (int)options.MessageTimeout.TotalMilliseconds; + config.MessageMaxBytes = options.MaxMessageSize; + return config; + } + + private static ConsumerConfig CreateConsumerConfig(KafkaConnectionOptions options) + { + var config = new ConsumerConfig(); + ApplyCommonConfig(config, options); + config.GroupId = ResolveGroupId(options); + config.AutoOffsetReset = MapAutoOffsetReset(options.AutoOffsetReset); + config.EnableAutoCommit = options.EnableAutoCommit; + return config; + } + + private static void ApplyCommonConfig(ClientConfig config, KafkaConnectionOptions options) + { + string bootstrap = options.BootstrapServers; + if (string.IsNullOrEmpty(bootstrap) && !string.IsNullOrEmpty(options.Endpoint)) + { + bootstrap = KafkaEndpointParser.Parse(options.Endpoint).BootstrapServers; + } + config.BootstrapServers = bootstrap; + if (!string.IsNullOrEmpty(options.ClientId)) + { + config.ClientId = options.ClientId; + } + config.SecurityProtocol = MapSecurityProtocol(options.SecurityProtocol); + if (options.SaslMechanism != KafkaSaslMechanism.None) + { + config.SaslMechanism = MapSaslMechanism(options.SaslMechanism); + if (!string.IsNullOrEmpty(options.UserName)) + { + config.SaslUsername = options.UserName; + } + if (options.PasswordBytes is { Length: > 0 } passwordBytes) + { + config.SaslPassword = System.Text.Encoding.UTF8.GetString(passwordBytes); + } + } + KafkaTlsOptions? tls = options.Tls; + if (tls is not null) + { + if (!string.IsNullOrEmpty(tls.CaCertificatePath)) + { + config.SslCaLocation = tls.CaCertificatePath; + } + if (!string.IsNullOrEmpty(tls.ClientCertificatePath)) + { + config.SslCertificateLocation = tls.ClientCertificatePath; + } + if (!string.IsNullOrEmpty(tls.ClientKeyPath)) + { + config.SslKeyLocation = tls.ClientKeyPath; + } + config.EnableSslCertificateVerification = tls.ValidateServerCertificate; + } + } + + private static void ValidateCredentialTransport(KafkaConnectionOptions options) + { + if (options.SaslMechanism == KafkaSaslMechanism.None + || string.IsNullOrEmpty(options.UserName)) + { + return; + } + bool useTls = options.SecurityProtocol + is KafkaSecurityProtocol.Ssl + or KafkaSecurityProtocol.SaslSsl + || (options.Tls?.UseTls ?? false); + if (!useTls && !options.AllowCredentialsOverPlaintext) + { + throw new InvalidOperationException( + "Kafka SASL credentials require TLS unless " + + "KafkaConnectionOptions.AllowCredentialsOverPlaintext is set."); + } + } + + private static string ResolveGroupId(KafkaConnectionOptions options) + { + if (!string.IsNullOrEmpty(options.GroupId)) + { + return options.GroupId; + } + string prefix = string.IsNullOrEmpty(options.ClientId) ? "opcua-pubsub" : options.ClientId; + return string.Concat( + prefix, + "-", + Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture)); + } + + private static Acks MapAcks(KafkaAcks acks) + { + return acks switch + { + KafkaAcks.None => Confluent.Kafka.Acks.None, + KafkaAcks.Leader => Confluent.Kafka.Acks.Leader, + KafkaAcks.All => Confluent.Kafka.Acks.All, + _ => Confluent.Kafka.Acks.All + }; + } + + private static SecurityProtocol MapSecurityProtocol(KafkaSecurityProtocol protocol) + { + return protocol switch + { + KafkaSecurityProtocol.Plaintext => SecurityProtocol.Plaintext, + KafkaSecurityProtocol.Ssl => SecurityProtocol.Ssl, + KafkaSecurityProtocol.SaslPlaintext => SecurityProtocol.SaslPlaintext, + KafkaSecurityProtocol.SaslSsl => SecurityProtocol.SaslSsl, + _ => SecurityProtocol.Plaintext + }; + } + + private static SaslMechanism MapSaslMechanism(KafkaSaslMechanism mechanism) + { + return mechanism switch + { + KafkaSaslMechanism.Plain => SaslMechanism.Plain, + KafkaSaslMechanism.ScramSha256 => SaslMechanism.ScramSha256, + KafkaSaslMechanism.ScramSha512 => SaslMechanism.ScramSha512, + KafkaSaslMechanism.OAuthBearer => SaslMechanism.OAuthBearer, + _ => SaslMechanism.Plain + }; + } + + private static AutoOffsetReset MapAutoOffsetReset(KafkaAutoOffsetReset autoOffsetReset) + { + return autoOffsetReset switch + { + KafkaAutoOffsetReset.Earliest => AutoOffsetReset.Earliest, + KafkaAutoOffsetReset.Latest => AutoOffsetReset.Latest, + _ => AutoOffsetReset.Latest + }; + } + } +} +#endif diff --git a/Libraries/Opc.Ua.PubSub.Kafka/Internal/ConfluentKafkaClientFactory.cs b/Libraries/Opc.Ua.PubSub.Kafka/Internal/ConfluentKafkaClientFactory.cs new file mode 100644 index 0000000000..871361ca38 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/Internal/ConfluentKafkaClientFactory.cs @@ -0,0 +1,62 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#if !NET10_0_OR_GREATER +using System; + +namespace Opc.Ua.PubSub.Kafka.Internal +{ + /// + /// Default implementation backed by + /// Confluent.Kafka (native librdkafka). + /// + /// + /// Wired into the DI container by the PubSub transport composition; + /// tests may instantiate it directly or substitute a fake factory to + /// avoid an actual broker connection. + /// + internal sealed class ConfluentKafkaClientFactory : IKafkaClientFactory + { + /// + public IKafkaClientAdapter Create(ITelemetryContext telemetry, TimeProvider timeProvider) + { + if (telemetry is null) + { + throw new ArgumentNullException(nameof(telemetry)); + } + if (timeProvider is null) + { + throw new ArgumentNullException(nameof(timeProvider)); + } + return new ConfluentKafkaClientAdapter(telemetry, timeProvider); + } + } +} +#endif diff --git a/Libraries/Opc.Ua.PubSub.Kafka/Internal/DekafKafkaClientAdapter.cs b/Libraries/Opc.Ua.PubSub.Kafka/Internal/DekafKafkaClientAdapter.cs new file mode 100644 index 0000000000..57edaa1010 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/Internal/DekafKafkaClientAdapter.cs @@ -0,0 +1,731 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#if NET10_0_OR_GREATER +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Dekaf; +using Dekaf.Consumer; +using Dekaf.Producer; +using Dekaf.Security; +using Dekaf.Serialization; +using Microsoft.Extensions.Logging; + +namespace Opc.Ua.PubSub.Kafka.Internal +{ + /// + /// Dekaf-backed implementation of . + /// + /// + /// The adapter owns a lazily created + /// (built on first produce) and a lazily created + /// plus a background consume + /// loop (built on first subscribe), so a send-only or receive-only + /// connection never instantiates the unused half. Dekaf is a pure-managed + /// .NET 10 Kafka client and is therefore usable by NativeAOT publishers. + /// + internal sealed class DekafKafkaClientAdapter : IKafkaClientAdapter + { + private const string ContentTypeHeader = "content-type"; + + private readonly ILogger m_logger; + private readonly TimeProvider m_timeProvider; + private readonly System.Threading.Lock m_sync = new(); + private readonly SemaphoreSlim m_clientGate = new(1, 1); + private readonly List m_subscribedTopics = new(); + + private KafkaConnectionOptions? m_options; + private IKafkaProducer? m_producer; + private IKafkaConsumer? m_consumer; + private CancellationTokenSource? m_loopCts; + private Task? m_consumeTask; + private bool m_connected; + private bool m_disposed; + + /// + /// Initializes a new . + /// + /// Telemetry context. + /// Clock for receive-time stamps. + public DekafKafkaClientAdapter(ITelemetryContext telemetry, TimeProvider timeProvider) + { + if (telemetry is null) + { + throw new ArgumentNullException(nameof(telemetry)); + } + if (timeProvider is null) + { + throw new ArgumentNullException(nameof(timeProvider)); + } + m_logger = telemetry.CreateLogger(); + m_timeProvider = timeProvider; + } + + /// + public bool IsConnected + { + get + { + lock (m_sync) + { + return m_connected; + } + } + } + + /// + public event EventHandler? IncomingMessage; + + /// + public event EventHandler? ConnectionStateChanged; + + /// + public ValueTask ConnectAsync(KafkaConnectionOptions options, CancellationToken ct) + { + if (options is null) + { + throw new ArgumentNullException(nameof(options)); + } + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ValidateCredentialTransport(options); + ValidateDekafSupport(options); + + bool raiseConnected; + lock (m_sync) + { + m_options = options; + m_loopCts ??= new CancellationTokenSource(); + raiseConnected = !m_connected; + m_connected = true; + } + m_logger.LogDebug( + "Kafka adapter connected to {BootstrapServers} (protocol={Protocol}).", + string.IsNullOrEmpty(options.BootstrapServers) ? options.Endpoint : options.BootstrapServers, + options.SecurityProtocol); + if (raiseConnected) + { + ConnectionStateChanged?.Invoke( + this, + new KafkaConnectionStateChangedEventArgs(isConnected: true, reason: null)); + } + return default; + } + + /// + public async ValueTask DisconnectAsync(CancellationToken ct) + { + await ShutdownAsync(raiseEvent: true).ConfigureAwait(false); + } + + /// + public async ValueTask SubscribeAsync(IReadOnlyList topics, CancellationToken ct) + { + if (topics is null) + { + throw new ArgumentNullException(nameof(topics)); + } + ThrowIfDisposed(); + if (topics.Count == 0) + { + return; + } + + await m_clientGate.WaitAsync(ct).ConfigureAwait(false); + try + { + ThrowIfDisposed(); + IKafkaConsumer consumer = await EnsureConsumerAsync(ct).ConfigureAwait(false); + bool changed = false; + foreach (string topic in topics) + { + if (!m_subscribedTopics.Contains(topic)) + { + m_subscribedTopics.Add(topic); + changed = true; + } + } + if (changed) + { + consumer.Subscribe(m_subscribedTopics.ToArray()); + } + if (m_consumeTask is null) + { + CancellationToken loopToken = m_loopCts!.Token; + bool enableAutoCommit = m_options!.EnableAutoCommit; + m_consumeTask = Task.Factory.StartNew( + () => ConsumeLoopAsync(consumer, enableAutoCommit, loopToken), + loopToken, + TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, + TaskScheduler.Default).Unwrap(); + } + } + finally + { + m_clientGate.Release(); + } + m_logger.LogDebug("Kafka subscribed to {Count} topic(s).", topics.Count); + } + + /// + public async ValueTask UnsubscribeAsync(IReadOnlyList topics, CancellationToken ct) + { + if (topics is null) + { + throw new ArgumentNullException(nameof(topics)); + } + ThrowIfDisposed(); + if (topics.Count == 0) + { + return; + } + + await m_clientGate.WaitAsync(ct).ConfigureAwait(false); + try + { + if (m_consumer is null) + { + return; + } + bool changed = false; + foreach (string topic in topics) + { + if (m_subscribedTopics.Remove(topic)) + { + changed = true; + } + } + if (changed) + { + if (m_subscribedTopics.Count == 0) + { + m_consumer.Unsubscribe(); + } + else + { + m_consumer.Subscribe(m_subscribedTopics.ToArray()); + } + } + } + finally + { + m_clientGate.Release(); + } + } + + /// + public async ValueTask ProduceAsync(KafkaMessage message, CancellationToken ct) + { + if (string.IsNullOrEmpty(message.Topic)) + { + throw new ArgumentException("Kafka produce requires a topic.", nameof(message)); + } + ThrowIfDisposed(); + + Headers headers = CreateHeaders(message); + byte[] key = message.Key.IsEmpty ? null! : message.Key.ToArray(); + byte[] value = message.Value.IsEmpty ? Array.Empty() : message.Value.ToArray(); + ProducerMessage record = ProducerMessage.Create( + message.Topic, + key, + value, + headers); + + await m_clientGate.WaitAsync(ct).ConfigureAwait(false); + try + { + ThrowIfDisposed(); + IKafkaProducer producer = await EnsureProducerAsync(ct).ConfigureAwait(false); + await producer.ProduceAsync(record, ct).ConfigureAwait(false); + } + finally + { + m_clientGate.Release(); + } + } + + /// + public async ValueTask DisposeAsync() + { + lock (m_sync) + { + if (m_disposed) + { + return; + } + m_disposed = true; + } + await ShutdownAsync(raiseEvent: false).ConfigureAwait(false); + m_clientGate.Dispose(); + } + + private async ValueTask ShutdownAsync(bool raiseEvent) + { + await m_clientGate.WaitAsync(CancellationToken.None).ConfigureAwait(false); + try + { + CancellationTokenSource? loopCts; + Task? consumeTask; + IKafkaConsumer? consumer; + IKafkaProducer? producer; + bool wasConnected; + lock (m_sync) + { + loopCts = m_loopCts; + m_loopCts = null; + consumeTask = m_consumeTask; + m_consumeTask = null; + consumer = m_consumer; + m_consumer = null; + producer = m_producer; + m_producer = null; + wasConnected = m_connected; + m_connected = false; + m_subscribedTopics.Clear(); + } + + if (loopCts is not null) + { + try + { + loopCts.Cancel(); + consumer?.Wakeup(); + } + catch (Exception ex) + { + m_logger.LogDebug(ex, "Kafka consume loop cancellation raised an exception."); + } + } + if (consumeTask is not null) + { + try + { + await consumeTask.ConfigureAwait(false); + } + catch (Exception ex) + { + m_logger.LogDebug(ex, "Kafka consume loop terminated with an exception."); + } + } + if (consumer is not null) + { + try + { + await consumer.CloseAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (Exception ex) + { + m_logger.LogDebug(ex, "Kafka consumer close raised an exception."); + } + await consumer.DisposeAsync().ConfigureAwait(false); + } + if (producer is not null) + { + try + { + using var flushCts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await producer.FlushAsync(flushCts.Token).ConfigureAwait(false); + } + catch (Exception ex) + { + m_logger.LogDebug(ex, "Kafka producer flush raised an exception."); + } + await producer.DisposeAsync().ConfigureAwait(false); + } + loopCts?.Dispose(); + + if (raiseEvent && wasConnected) + { + ConnectionStateChanged?.Invoke( + this, + new KafkaConnectionStateChangedEventArgs( + isConnected: false, + reason: "Kafka adapter disconnected.")); + } + } + finally + { + m_clientGate.Release(); + } + } + + private async ValueTask> EnsureProducerAsync(CancellationToken ct) + { + if (m_producer is not null) + { + return m_producer; + } + ProducerBuilder builder = global::Dekaf.Kafka.CreateProducer(); + ApplyProducerConfig(builder, m_options!); + m_producer = await builder.BuildAsync(ct).ConfigureAwait(false); + return m_producer; + } + + private async ValueTask> EnsureConsumerAsync(CancellationToken ct) + { + if (m_consumer is not null) + { + return m_consumer; + } + ConsumerBuilder builder = global::Dekaf.Kafka.CreateConsumer(); + ApplyConsumerConfig(builder, m_options!); + m_consumer = await builder.BuildAsync(ct).ConfigureAwait(false); + return m_consumer; + } + + private async Task ConsumeLoopAsync( + IKafkaConsumer consumer, + bool enableAutoCommit, + CancellationToken ct) + { + try + { + await foreach (ConsumeResult result in consumer.ConsumeAsync(ct).ConfigureAwait(false)) + { + if (result.IsPartitionEof) + { + continue; + } + try + { + DispatchRecord(result); + if (!enableAutoCommit) + { + await consumer.CommitAsync(ct).ConfigureAwait(false); + } + } + catch (Exception ex) + { + m_logger.LogWarning(ex, "Failed to dispatch inbound Kafka record."); + } + } + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + } + catch (Exception ex) + { + m_logger.LogError(ex, "Kafka consume loop terminated unexpectedly."); + } + } + + private void DispatchRecord(ConsumeResult result) + { + string? contentType = null; + Dictionary? extraHeaders = null; + IReadOnlyList
? headers = result.Headers; + if (headers is not null && headers.Count > 0) + { + foreach (Header header in headers) + { + string headerValue = header.IsValueNull + ? string.Empty + : System.Text.Encoding.UTF8.GetString(header.Value.Span); + if (string.Equals(header.Key, ContentTypeHeader, StringComparison.OrdinalIgnoreCase)) + { + contentType = headerValue; + continue; + } + extraHeaders ??= new Dictionary(StringComparer.Ordinal); + extraHeaders[header.Key] = headerValue; + } + } + byte[]? keyOrNull = result.Key; + byte[]? valueOrNull = result.Value; + byte[] key = keyOrNull ?? Array.Empty(); + byte[] value = valueOrNull ?? Array.Empty(); + var message = new KafkaMessage(result.Topic, key, value, contentType, extraHeaders); + var args = new KafkaIncomingMessageEventArgs( + message, + DateTimeUtc.From(m_timeProvider.GetUtcNow())); + IncomingMessage?.Invoke(this, args); + } + + private void ThrowIfDisposed() + { + if (m_disposed) + { + throw new ObjectDisposedException(nameof(DekafKafkaClientAdapter)); + } + } + + private static Headers CreateHeaders(KafkaMessage message) + { + Headers headers = Headers.Create(); + if (!string.IsNullOrEmpty(message.ContentType)) + { + headers.Add(ContentTypeHeader, message.ContentType); + } + if (message.Headers is not null) + { + foreach (KeyValuePair header in message.Headers) + { + headers.Add(header.Key, header.Value); + } + } + return headers; + } + + private static void ApplyProducerConfig( + ProducerBuilder builder, + KafkaConnectionOptions options) + { + ApplyProducerCommonConfig(builder, options); + KafkaDeliveryGuarantee guarantee = options.DeliveryGuarantee.ToDeliveryGuarantee(); + builder.WithAcks(MapAcks(guarantee.Acks)); + builder.WithIdempotence(guarantee.EnableIdempotence); + builder.WithDeliveryTimeout(options.MessageTimeout); + } + + private static void ApplyConsumerConfig( + ConsumerBuilder builder, + KafkaConnectionOptions options) + { + ApplyConsumerCommonConfig(builder, options); + builder.WithGroupId(ResolveGroupId(options)); + builder.WithAutoOffsetReset(MapAutoOffsetReset(options.AutoOffsetReset)); + builder.WithOffsetCommitMode(options.EnableAutoCommit ? OffsetCommitMode.Auto : OffsetCommitMode.Manual); + } + + private static void ApplyProducerCommonConfig( + ProducerBuilder builder, + KafkaConnectionOptions options) + { + builder.WithBootstrapServers(ResolveBootstrapServers(options)); + if (!string.IsNullOrEmpty(options.ClientId)) + { + builder.WithClientId(options.ClientId); + } + ApplyProducerSecurity(builder, options); + } + + private static void ApplyConsumerCommonConfig( + ConsumerBuilder builder, + KafkaConnectionOptions options) + { + builder.WithBootstrapServers(ResolveBootstrapServers(options)); + if (!string.IsNullOrEmpty(options.ClientId)) + { + builder.WithClientId(options.ClientId); + } + ApplyConsumerSecurity(builder, options); + } + + private static void ApplyProducerSecurity( + ProducerBuilder builder, + KafkaConnectionOptions options) + { + if (UseTls(options)) + { + builder.UseTls(CreateTlsConfig(options)); + } + switch (options.SaslMechanism) + { + case KafkaSaslMechanism.None: + break; + case KafkaSaslMechanism.Plain: + builder.WithSaslPlain(RequireUserName(options), ResolvePassword(options)); + break; + case KafkaSaslMechanism.ScramSha256: + builder.WithSaslScramSha256(RequireUserName(options), ResolvePassword(options)); + break; + case KafkaSaslMechanism.ScramSha512: + builder.WithSaslScramSha512(RequireUserName(options), ResolvePassword(options)); + break; + case KafkaSaslMechanism.OAuthBearer: + throw CreateUnsupportedSaslMechanismException(options.SaslMechanism); + default: + throw CreateUnsupportedSaslMechanismException(options.SaslMechanism); + } + } + + private static void ApplyConsumerSecurity( + ConsumerBuilder builder, + KafkaConnectionOptions options) + { + if (UseTls(options)) + { + builder.UseTls(CreateTlsConfig(options)); + } + switch (options.SaslMechanism) + { + case KafkaSaslMechanism.None: + break; + case KafkaSaslMechanism.Plain: + builder.WithSaslPlain(RequireUserName(options), ResolvePassword(options)); + break; + case KafkaSaslMechanism.ScramSha256: + builder.WithSaslScramSha256(RequireUserName(options), ResolvePassword(options)); + break; + case KafkaSaslMechanism.ScramSha512: + builder.WithSaslScramSha512(RequireUserName(options), ResolvePassword(options)); + break; + case KafkaSaslMechanism.OAuthBearer: + throw CreateUnsupportedSaslMechanismException(options.SaslMechanism); + default: + throw CreateUnsupportedSaslMechanismException(options.SaslMechanism); + } + } + + private static TlsConfig CreateTlsConfig(KafkaConnectionOptions options) + { + KafkaTlsOptions? tls = options.Tls; + if (tls is null) + { + return new TlsConfig(); + } + if (string.IsNullOrEmpty(tls.ClientCertificatePath) != string.IsNullOrEmpty(tls.ClientKeyPath)) + { + throw new NotSupportedException( + "Dekaf mutual TLS requires both KafkaTlsOptions.ClientCertificatePath and " + + "KafkaTlsOptions.ClientKeyPath."); + } + return new TlsConfig + { + CaCertificatePath = tls.CaCertificatePath, + ClientCertificatePath = tls.ClientCertificatePath, + ClientKeyPath = tls.ClientKeyPath, + ValidateServerCertificate = tls.ValidateServerCertificate + }; + } + + private static void ValidateDekafSupport(KafkaConnectionOptions options) + { + if (options.SaslMechanism == KafkaSaslMechanism.OAuthBearer) + { + throw CreateUnsupportedSaslMechanismException(options.SaslMechanism); + } + _ = CreateTlsConfig(options); + } + + private static void ValidateCredentialTransport(KafkaConnectionOptions options) + { + if (options.SaslMechanism == KafkaSaslMechanism.None + || string.IsNullOrEmpty(options.UserName)) + { + return; + } + bool useTls = options.SecurityProtocol + is KafkaSecurityProtocol.Ssl + or KafkaSecurityProtocol.SaslSsl + || (options.Tls?.UseTls ?? false); + if (!useTls && !options.AllowCredentialsOverPlaintext) + { + throw new InvalidOperationException( + "Kafka SASL credentials require TLS unless " + + "KafkaConnectionOptions.AllowCredentialsOverPlaintext is set."); + } + } + + private static string ResolveBootstrapServers(KafkaConnectionOptions options) + { + string bootstrap = options.BootstrapServers; + if (string.IsNullOrEmpty(bootstrap) && !string.IsNullOrEmpty(options.Endpoint)) + { + bootstrap = KafkaEndpointParser.Parse(options.Endpoint).BootstrapServers; + } + return bootstrap; + } + + private static bool UseTls(KafkaConnectionOptions options) + { + return options.SecurityProtocol + is KafkaSecurityProtocol.Ssl + or KafkaSecurityProtocol.SaslSsl + || (options.Tls?.UseTls ?? false); + } + + private static string RequireUserName(KafkaConnectionOptions options) + { + if (string.IsNullOrEmpty(options.UserName)) + { + throw new InvalidOperationException( + "Kafka SASL authentication requires KafkaConnectionOptions.UserName."); + } + return options.UserName; + } + + private static string ResolvePassword(KafkaConnectionOptions options) + { + if (options.PasswordBytes is not { Length: > 0 } passwordBytes) + { + throw new InvalidOperationException( + "Kafka SASL authentication requires KafkaConnectionOptions.PasswordBytes."); + } + return System.Text.Encoding.UTF8.GetString(passwordBytes); + } + + private static string ResolveGroupId(KafkaConnectionOptions options) + { + if (!string.IsNullOrEmpty(options.GroupId)) + { + return options.GroupId; + } + string prefix = string.IsNullOrEmpty(options.ClientId) ? "opcua-pubsub" : options.ClientId; + return string.Concat( + prefix, + "-", + Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture)); + } + + private static Acks MapAcks(KafkaAcks acks) + { + return acks switch + { + KafkaAcks.None => Acks.None, + KafkaAcks.Leader => Acks.Leader, + KafkaAcks.All => Acks.All, + _ => Acks.All + }; + } + + private static AutoOffsetReset MapAutoOffsetReset(KafkaAutoOffsetReset autoOffsetReset) + { + return autoOffsetReset switch + { + KafkaAutoOffsetReset.Earliest => AutoOffsetReset.Earliest, + KafkaAutoOffsetReset.Latest => AutoOffsetReset.Latest, + _ => AutoOffsetReset.Latest + }; + } + + private static NotSupportedException CreateUnsupportedSaslMechanismException(KafkaSaslMechanism mechanism) + { + return new NotSupportedException( + string.Format( + CultureInfo.InvariantCulture, + "Dekaf Kafka adapter does not support SASL mechanism '{0}'.", + mechanism)); + } + } +} +#endif diff --git a/Libraries/Opc.Ua.PubSub.Kafka/Internal/DekafKafkaClientFactory.cs b/Libraries/Opc.Ua.PubSub.Kafka/Internal/DekafKafkaClientFactory.cs new file mode 100644 index 0000000000..729c237dcd --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/Internal/DekafKafkaClientFactory.cs @@ -0,0 +1,62 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#if NET10_0_OR_GREATER +using System; + +namespace Opc.Ua.PubSub.Kafka.Internal +{ + /// + /// Default implementation backed by + /// Dekaf. + /// + /// + /// Wired into the DI container by the PubSub transport composition; + /// tests may instantiate it directly or substitute a fake factory to + /// avoid an actual broker connection. + /// + internal sealed class DekafKafkaClientFactory : IKafkaClientFactory + { + /// + public IKafkaClientAdapter Create(ITelemetryContext telemetry, TimeProvider timeProvider) + { + if (telemetry is null) + { + throw new ArgumentNullException(nameof(telemetry)); + } + if (timeProvider is null) + { + throw new ArgumentNullException(nameof(timeProvider)); + } + return new DekafKafkaClientAdapter(telemetry, timeProvider); + } + } +} +#endif diff --git a/Libraries/Opc.Ua.PubSub.Kafka/Internal/IKafkaClientAdapter.cs b/Libraries/Opc.Ua.PubSub.Kafka/Internal/IKafkaClientAdapter.cs new file mode 100644 index 0000000000..fc3ce9be51 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/Internal/IKafkaClientAdapter.cs @@ -0,0 +1,113 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.PubSub.Kafka.Internal +{ + /// + /// Internal abstraction shielding the rest of the library from the + /// concrete Kafka client surface. The default implementation wraps a + /// producer / consumer pair; unit tests can inject a + /// fake to drive the transport without an actual broker. + /// + /// + /// The adapter wraps a single Kafka client session — connect / + /// produce / subscribe / close. It does not own retry semantics; the + /// owning is responsible for + /// reconnect orchestration. Implementations must be safe to call + /// concurrently with an + /// in-flight . + /// + internal interface IKafkaClientAdapter : IAsyncDisposable + { + /// + /// Whether the adapter believes it is connected. + /// + bool IsConnected { get; } + + /// + /// Connects to the brokers using . + /// Idempotent — calling on an already-connected adapter returns + /// immediately. + /// + /// Connection options. + /// Cancellation token. + ValueTask ConnectAsync( + KafkaConnectionOptions options, + CancellationToken cancellationToken); + + /// + /// Disconnects cleanly from the brokers. Idempotent. + /// + /// Cancellation token. + ValueTask DisconnectAsync(CancellationToken cancellationToken); + + /// + /// Subscribes to the supplied topics and starts the background + /// consume loop. + /// + /// Topics to consume. + /// Cancellation token. + ValueTask SubscribeAsync( + IReadOnlyList topics, + CancellationToken cancellationToken); + + /// + /// Removes the supplied topics from the consumer subscription. + /// + /// Topics to remove. + /// Cancellation token. + ValueTask UnsubscribeAsync( + IReadOnlyList topics, + CancellationToken cancellationToken); + + /// + /// Produces to its topic. + /// + /// Record envelope. + /// Cancellation token. + ValueTask ProduceAsync( + KafkaMessage message, + CancellationToken cancellationToken); + + /// + /// Raised whenever a broker-delivered record arrives. + /// + event EventHandler? IncomingMessage; + + /// + /// Raised whenever the broker connection state changes. + /// + event EventHandler? ConnectionStateChanged; + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/Internal/IKafkaClientFactory.cs b/Libraries/Opc.Ua.PubSub.Kafka/Internal/IKafkaClientFactory.cs new file mode 100644 index 0000000000..b9558b1516 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/Internal/IKafkaClientFactory.cs @@ -0,0 +1,57 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.PubSub.Kafka.Internal +{ + /// + /// Provider-model factory for the Kafka client adapter used by + /// . Test code can swap in a fake + /// to drive the transport without an actual broker; the default + /// implementation creates a TFM-specific Kafka-backed adapter. + /// + /// + /// Provides the adapter seam used by the Kafka broker transport per + /// + /// Part 14 Annex B.2 Apache Kafka transport. + /// + public interface IKafkaClientFactory + { + /// + /// Creates a fresh adapter instance. + /// + /// Telemetry context. + /// Clock for the adapter. + /// The new adapter. + internal IKafkaClientAdapter Create( + ITelemetryContext telemetry, + TimeProvider timeProvider); + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaBrokerTransport.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaBrokerTransport.cs new file mode 100644 index 0000000000..6b7311f9c0 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaBrokerTransport.cs @@ -0,0 +1,804 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Opc.Ua.PubSub.Diagnostics; +using Opc.Ua.PubSub.Encoding; +using Opc.Ua.PubSub.Kafka.Internal; +using Opc.Ua.PubSub.Transports; + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// implementation for the Apache Kafka + /// broker profiles + /// ( and + /// ). One instance + /// represents one bound to a + /// kafka:// or kafkas:// broker endpoint. + /// + /// + /// + /// Implements the broker mapping defined in + /// + /// Part 14 Annex B.2 Apache Kafka transport, including the + /// delivery-guarantee mapping realised through the producer + /// acks / enable.idempotence settings. Payload encoding + /// is opaque to the transport; the encoding profile is chosen by the + /// writer-group MessageSettings on the connection + /// ( → + /// , + /// → + /// ). + /// + /// + /// The transport delegates to an so + /// the Confluent.Kafka client surface is invisible to higher layers, + /// and so unit tests can inject a fake adapter to exercise the state + /// machine without an actual broker. Each produced record carries a + /// partition key derived from the PublisherId so records from a + /// publisher preserve ordering within a partition. + /// + /// + public sealed class KafkaBrokerTransport : IPubSubTransport, IPubSubTopicProvider + { + private readonly PubSubConnectionDataType m_connection; + private readonly KafkaEndpoint m_endpoint; + private readonly PubSubTransportDirection m_direction; + private readonly KafkaConnectionOptions m_options; + private readonly IKafkaClientFactory m_clientFactory; + private readonly ITelemetryContext m_telemetry; + private readonly TimeProvider m_timeProvider; + private readonly IPubSubDiagnostics? m_diagnostics; + private readonly ILogger m_logger; + private readonly System.Threading.Lock m_sync = new(); + private readonly string m_transportProfileUri; + private readonly byte[] m_partitionKey; + + private IKafkaClientAdapter? m_adapter; + private Channel? m_channel; + private bool m_isConnected; + private bool m_disposed; + + /// + /// Initializes a new . + /// + /// + /// PubSubConnection configuration the transport is bound to. + /// + /// + /// Parsed broker endpoint from + /// . + /// + /// + /// Direction the transport services. + /// + /// + /// Resolved connection options (credentials already populated by + /// the factory). + /// + /// + /// Factory used to create the underlying Kafka client adapter. + /// + /// + /// Telemetry context for per-instance logger creation. + /// + /// + /// Clock used for receive-time stamps. + /// + /// + /// Optional diagnostics sink. Counters are incremented per inbound + /// / outbound frame when non-. + /// + public KafkaBrokerTransport( + PubSubConnectionDataType connection, + KafkaEndpoint endpoint, + PubSubTransportDirection direction, + KafkaConnectionOptions options, + IKafkaClientFactory clientFactory, + ITelemetryContext telemetry, + TimeProvider timeProvider, + IPubSubDiagnostics? diagnostics = null) + { + if (connection is null) + { + throw new ArgumentNullException(nameof(connection)); + } + if (options is null) + { + throw new ArgumentNullException(nameof(options)); + } + if (clientFactory is null) + { + throw new ArgumentNullException(nameof(clientFactory)); + } + if (telemetry is null) + { + throw new ArgumentNullException(nameof(telemetry)); + } + if (timeProvider is null) + { + throw new ArgumentNullException(nameof(timeProvider)); + } + + m_connection = connection; + m_endpoint = endpoint; + m_direction = direction; + m_options = options; + m_clientFactory = clientFactory; + m_telemetry = telemetry; + m_timeProvider = timeProvider; + m_diagnostics = diagnostics; + m_logger = telemetry.CreateLogger(); + m_transportProfileUri = DetermineTransportProfileUri(connection); + m_partitionKey = BuildPartitionKey(connection); + Subscriptions = new List(); + AddDefaultSubscriptions(); + } + + /// + public string TransportProfileUri => m_transportProfileUri; + + /// + public PubSubTransportDirection Direction => m_direction; + + /// + public bool IsConnected + { + get + { + lock (m_sync) + { + return m_isConnected; + } + } + } + + /// + /// Parsed endpoint the transport is bound to. Exposed so + /// integration tests can confirm bootstrap server selection + /// without re-parsing the URL. + /// + public KafkaEndpoint Endpoint => m_endpoint; + + /// + /// Resolved connection options. Exposed for diagnostics and + /// tests; the password bytes are never serialized. + /// + public KafkaConnectionOptions Options => m_options; + + /// + /// Kafka topics the subscriber consumes from. Populated from the + /// connection's reader-group broker transport settings + /// (QueueName / MetaDataQueueName); callers may append additional + /// topics before . + /// + public IList Subscriptions { get; } + + /// + public event EventHandler? StateChanged; + + /// + public async ValueTask OpenAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + IKafkaClientAdapter adapter; + Channel? channel = null; + lock (m_sync) + { + if (m_disposed) + { + throw new ObjectDisposedException(nameof(KafkaBrokerTransport)); + } + if (m_adapter is not null) + { + return; + } + adapter = m_clientFactory.Create(m_telemetry, m_timeProvider); + if (HasReceiveDirection) + { + channel = Channel.CreateBounded( + new BoundedChannelOptions(GetReceiveQueueCapacity()) + { + FullMode = BoundedChannelFullMode.DropOldest, + SingleReader = false, + SingleWriter = true + }); + m_channel = channel; + } + m_adapter = adapter; + } + + adapter.IncomingMessage += OnIncomingMessage; + adapter.ConnectionStateChanged += OnConnectionStateChanged; + + try + { + await adapter.ConnectAsync(m_options, cancellationToken).ConfigureAwait(false); + if (HasReceiveDirection && Subscriptions.Count > 0) + { + var topicList = new List(Subscriptions); + foreach (string topic in topicList) + { + ValidateTopic(topic); + } + await adapter.SubscribeAsync(topicList, cancellationToken).ConfigureAwait(false); + } + } + catch + { + adapter.IncomingMessage -= OnIncomingMessage; + adapter.ConnectionStateChanged -= OnConnectionStateChanged; + lock (m_sync) + { + m_adapter = null; + m_channel = null; + } + channel?.Writer.TryComplete(); + await adapter.DisposeAsync().ConfigureAwait(false); + throw; + } + + lock (m_sync) + { + m_isConnected = true; + } + m_logger.LogInformation( + "Kafka transport opened: connection='{Connection}' bootstrap={Bootstrap} direction={Direction} profile={Profile}", + m_connection.Name, + m_endpoint.BootstrapServers, + m_direction, + m_transportProfileUri); + RaiseStateChanged(true, StatusCodes.Good, null); + } + + /// + public async ValueTask CloseAsync(CancellationToken cancellationToken = default) + { + IKafkaClientAdapter? adapter; + Channel? channel; + bool wasConnected; + lock (m_sync) + { + adapter = m_adapter; + channel = m_channel; + wasConnected = m_isConnected; + m_adapter = null; + m_channel = null; + m_isConnected = false; + } + + if (adapter is not null) + { + adapter.IncomingMessage -= OnIncomingMessage; + adapter.ConnectionStateChanged -= OnConnectionStateChanged; + try + { + await adapter.DisconnectAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + m_logger.LogDebug( + ex, + "Kafka disconnect for connection '{Connection}' raised an exception.", + m_connection.Name); + } + await adapter.DisposeAsync().ConfigureAwait(false); + } + channel?.Writer.TryComplete(); + if (wasConnected) + { + RaiseStateChanged(false, StatusCodes.Good, "Transport closed."); + } + } + + /// + public async ValueTask SendAsync( + ReadOnlyMemory payload, + string? topic = null, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (string.IsNullOrEmpty(topic)) + { + throw new ArgumentException( + "Kafka broker transport requires a topic for every Send.", + nameof(topic)); + } + ValidateTopic(topic); + + IKafkaClientAdapter? adapter; + lock (m_sync) + { + if (m_disposed) + { + throw new ObjectDisposedException(nameof(KafkaBrokerTransport)); + } + adapter = m_adapter; + } + if (adapter is null) + { + throw new InvalidOperationException( + "Kafka transport must be opened before sending."); + } + + string? contentType = MapContentType(m_transportProfileUri); + var message = new KafkaMessage( + topic, + m_partitionKey, + payload, + contentType, + Headers: null); + + await adapter.ProduceAsync(message, cancellationToken).ConfigureAwait(false); + m_diagnostics?.Increment(PubSubDiagnosticsCounterKind.SentNetworkMessages, 1); + } + + /// + public async IAsyncEnumerable ReceiveAsync( + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Channel? channel; + lock (m_sync) + { + channel = m_channel; + } + if (channel is null) + { + yield break; + } + ChannelReader reader = channel.Reader; + while (await reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) + { + while (reader.TryRead(out PubSubTransportFrame frame)) + { + yield return frame; + } + } + } + + /// + public async ValueTask DisposeAsync() + { + bool alreadyDisposed; + lock (m_sync) + { + alreadyDisposed = m_disposed; + m_disposed = true; + } + if (alreadyDisposed) + { + return; + } + await CloseAsync().ConfigureAwait(false); + } + + /// + public string BuildMetaDataTopic( + PublisherId publisherId, + ushort writerGroupId, + ushort dataSetWriterId) + { + if (TryFindWriter(writerGroupId, dataSetWriterId, out DataSetWriterDataType? writer) + && writer is not null + && TryReadBrokerWriterSettings( + writer.TransportSettings, out _, out string? metadataQueue) + && !string.IsNullOrEmpty(metadataQueue)) + { + return metadataQueue; + } + return BuildDefaultTopic( + ResolveEncoding(m_transportProfileUri), + "metadata", + publisherId.ToString(), + writerGroupId.ToString(System.Globalization.CultureInfo.InvariantCulture), + dataSetWriterId.ToString(System.Globalization.CultureInfo.InvariantCulture)); + } + + /// + public string BuildDataTopic( + PublisherId publisherId, + WriterGroupDataType writerGroup, + ushort? dataSetWriterId) + { + if (writerGroup is null) + { + throw new ArgumentNullException(nameof(writerGroup)); + } + if (dataSetWriterId.HasValue + && TryFindWriter(writerGroup.WriterGroupId, dataSetWriterId.Value, out DataSetWriterDataType? writer) + && writer is not null + && TryReadBrokerWriterSettings(writer.TransportSettings, out string? queue, out _) + && !string.IsNullOrEmpty(queue)) + { + return queue; + } + if (TryReadBrokerGroupSettings(writerGroup.TransportSettings, out string? groupQueue) + && !string.IsNullOrEmpty(groupQueue)) + { + return groupQueue; + } + string? writerSegment = dataSetWriterId.HasValue + ? dataSetWriterId.Value.ToString(System.Globalization.CultureInfo.InvariantCulture) + : null; + return BuildDefaultTopic( + ResolveEncoding(m_transportProfileUri), + "data", + publisherId.ToString(), + writerGroup.WriterGroupId.ToString(System.Globalization.CultureInfo.InvariantCulture), + writerSegment); + } + + /// + public string BuildDiscoveryTopic(PublisherId publisherId, string messageTypeSegment) + { + return BuildDefaultTopic( + ResolveEncoding(m_transportProfileUri), + messageTypeSegment, + publisherId.ToString(), + additional1: null, + additional2: null); + } + + private bool HasReceiveDirection => + (m_direction & PubSubTransportDirection.Receive) != 0; + + private int GetReceiveQueueCapacity() + { + int subscriptions = Subscriptions.Count; + if (subscriptions <= 0) + { + return 256; + } + int capacity = subscriptions * 16; + return capacity < 256 ? 256 : capacity; + } + + private void OnIncomingMessage(object? sender, KafkaIncomingMessageEventArgs e) + { + Channel? channel; + lock (m_sync) + { + channel = m_channel; + } + if (channel is null) + { + return; + } + var frame = new PubSubTransportFrame( + e.Message.Value, + e.Message.Topic, + e.ReceivedAt); + if (!channel.Writer.TryWrite(frame)) + { + m_logger.LogWarning( + "Dropped inbound Kafka frame for connection '{Connection}': receive queue full.", + m_connection.Name); + return; + } + m_diagnostics?.Increment(PubSubDiagnosticsCounterKind.ReceivedNetworkMessages, 1); + } + + private void OnConnectionStateChanged( + object? sender, + KafkaConnectionStateChangedEventArgs e) + { + lock (m_sync) + { + m_isConnected = e.IsConnected; + } + StatusCode status = e.IsConnected + ? StatusCodes.Good + : StatusCodes.BadConnectionClosed; + RaiseStateChanged(e.IsConnected, status, e.Reason); + } + + private void RaiseStateChanged(bool isConnected, StatusCode status, string? reason) + { + EventHandler? handler = StateChanged; + handler?.Invoke( + this, + new PubSubTransportStateChangedEventArgs(isConnected, status, reason)); + } + + private void AddDefaultSubscriptions() + { + if (!HasReceiveDirection || m_connection.ReaderGroups.IsNull) + { + return; + } + foreach (ReaderGroupDataType group in m_connection.ReaderGroups) + { + if (group.DataSetReaders.IsNull) + { + continue; + } + foreach (DataSetReaderDataType reader in group.DataSetReaders) + { + if (TryReadBrokerReaderSettings( + reader.TransportSettings, + out string? queue, + out string? metadataQueue)) + { + AddSubscription(queue); + AddSubscription(metadataQueue); + } + } + } + } + + private void AddSubscription(string? topic) + { + if (string.IsNullOrEmpty(topic)) + { + return; + } + foreach (string existing in Subscriptions) + { + if (string.Equals(existing, topic, StringComparison.Ordinal)) + { + return; + } + } + Subscriptions.Add(topic); + } + + private bool TryFindWriter( + ushort writerGroupId, + ushort dataSetWriterId, + out DataSetWriterDataType? writer) + { + writer = null; + if (m_connection.WriterGroups.IsNull) + { + return false; + } + foreach (WriterGroupDataType group in m_connection.WriterGroups) + { + if (group.WriterGroupId != writerGroupId || group.DataSetWriters.IsNull) + { + continue; + } + foreach (DataSetWriterDataType candidate in group.DataSetWriters) + { + if (candidate.DataSetWriterId == dataSetWriterId) + { + writer = candidate; + return true; + } + } + } + return false; + } + + private static bool TryReadBrokerGroupSettings( + ExtensionObject settings, + out string? queueName) + { + queueName = null; + if (!settings.TryGetValue(out BrokerWriterGroupTransportDataType? broker) || broker is null) + { + return false; + } + queueName = broker.QueueName; + return true; + } + + private static bool TryReadBrokerWriterSettings( + ExtensionObject settings, + out string? queueName, + out string? metaDataQueueName) + { + queueName = null; + metaDataQueueName = null; + if (!settings.TryGetValue(out BrokerDataSetWriterTransportDataType? broker) || broker is null) + { + return false; + } + queueName = broker.QueueName; + metaDataQueueName = broker.MetaDataQueueName; + return true; + } + + private static bool TryReadBrokerReaderSettings( + ExtensionObject settings, + out string? queueName, + out string? metaDataQueueName) + { + queueName = null; + metaDataQueueName = null; + if (!settings.TryGetValue(out BrokerDataSetReaderTransportDataType? broker) || broker is null) + { + return false; + } + queueName = broker.QueueName; + metaDataQueueName = broker.MetaDataQueueName; + return true; + } + + private string BuildDefaultTopic( + KafkaEncoding encoding, + string messageType, + string publisherId, + string? additional1, + string? additional2) + { + var builder = new StringBuilder(); + AppendSegment(builder, m_options.Topics.Prefix); + AppendSegment(builder, encoding.ToTopicSegment()); + AppendSegment(builder, messageType); + AppendSegment(builder, publisherId); + AppendSegment(builder, additional1); + AppendSegment(builder, additional2); + return builder.ToString(); + } + + private static void AppendSegment(StringBuilder builder, string? segment) + { + if (string.IsNullOrEmpty(segment)) + { + return; + } + if (builder.Length > 0) + { + builder.Append('.'); + } + builder.Append(SanitizeSegment(segment)); + } + + private static string SanitizeSegment(string segment) + { + var builder = new StringBuilder(segment.Length); + foreach (char c in segment) + { + bool allowed = (c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || c is '_' or '-'; + builder.Append(allowed ? c : '_'); + } + return builder.ToString(); + } + + private static byte[] BuildPartitionKey(PubSubConnectionDataType connection) + { + if (!connection.PublisherId.IsNull) + { + string publisherId = PublisherId.From(connection.PublisherId).ToString(); + if (!string.IsNullOrEmpty(publisherId)) + { + return System.Text.Encoding.UTF8.GetBytes(publisherId); + } + } + if (!string.IsNullOrEmpty(connection.Name)) + { + return System.Text.Encoding.UTF8.GetBytes(connection.Name); + } + return Array.Empty(); + } + + private static void ValidateTopic(string topic) + { + if (string.IsNullOrEmpty(topic)) + { + throw new ArgumentException("Topic must not be empty.", nameof(topic)); + } + if (topic.Contains('\0', StringComparison.Ordinal)) + { + throw new ArgumentException( + "Topic must not contain a null character.", + nameof(topic)); + } + } + + private static KafkaEncoding ResolveEncoding(string transportProfileUri) + { + return string.Equals( + transportProfileUri, + KafkaProfiles.PubSubKafkaUadpTransport, + StringComparison.Ordinal) + ? KafkaEncoding.Uadp + : KafkaEncoding.Json; + } + + private static string? MapContentType(string transportProfileUri) + { + if (string.Equals( + transportProfileUri, + KafkaProfiles.PubSubKafkaJsonTransport, + StringComparison.Ordinal)) + { + return "application/json"; + } + if (string.Equals( + transportProfileUri, + KafkaProfiles.PubSubKafkaUadpTransport, + StringComparison.Ordinal)) + { + return "application/opcua+uadp"; + } + return null; + } + + private static string DetermineTransportProfileUri(PubSubConnectionDataType connection) + { + if (!connection.WriterGroups.IsNull) + { + foreach (WriterGroupDataType group in connection.WriterGroups) + { + string? profile = InferProfileFromMessageSettings(group.MessageSettings); + if (profile is not null) + { + return profile; + } + } + } + if (!string.IsNullOrEmpty(connection.TransportProfileUri)) + { + if (string.Equals( + connection.TransportProfileUri, + KafkaProfiles.PubSubKafkaUadpTransport, + StringComparison.Ordinal)) + { + return KafkaProfiles.PubSubKafkaUadpTransport; + } + if (string.Equals( + connection.TransportProfileUri, + KafkaProfiles.PubSubKafkaJsonTransport, + StringComparison.Ordinal)) + { + return KafkaProfiles.PubSubKafkaJsonTransport; + } + } + return KafkaProfiles.PubSubKafkaJsonTransport; + } + + private static string? InferProfileFromMessageSettings(ExtensionObject settings) + { + if (settings.IsNull) + { + return null; + } + IEncodeable? decoded = ExtensionObject.ToEncodeable(settings); + return decoded switch + { + UadpWriterGroupMessageDataType => KafkaProfiles.PubSubKafkaUadpTransport, + JsonWriterGroupMessageDataType => KafkaProfiles.PubSubKafkaJsonTransport, + _ => null + }; + } + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaConnectionOptions.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaConnectionOptions.cs new file mode 100644 index 0000000000..c309f935c7 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaConnectionOptions.cs @@ -0,0 +1,292 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// Transport-level security protocol negotiated with the Kafka + /// brokers, mirroring the librdkafka security.protocol + /// property. + /// + /// + /// Implements the security profile selector of + /// + /// Part 14 Annex B.2 Apache Kafka transport. + /// + public enum KafkaSecurityProtocol + { + /// + /// Unauthenticated plaintext transport (PLAINTEXT). + /// + Plaintext, + + /// + /// TLS-protected transport without SASL (SSL). + /// + Ssl, + + /// + /// SASL authentication over plaintext transport + /// (SASL_PLAINTEXT). + /// + SaslPlaintext, + + /// + /// SASL authentication over TLS transport (SASL_SSL). + /// + SaslSsl + } + + /// + /// SASL authentication mechanism used when the negotiated + /// carries SASL, mirroring the + /// librdkafka sasl.mechanism property. + /// + /// + /// Implements the SASL mechanism selector of + /// + /// Part 14 Annex B.2 Apache Kafka transport. + /// + public enum KafkaSaslMechanism + { + /// + /// No SASL mechanism is configured. + /// + None, + + /// + /// SASL/PLAIN username and password mechanism. + /// + Plain, + + /// + /// SASL/SCRAM-SHA-256 challenge-response mechanism. + /// + ScramSha256, + + /// + /// SASL/SCRAM-SHA-512 challenge-response mechanism. + /// + ScramSha512, + + /// + /// SASL/OAUTHBEARER token mechanism. + /// + OAuthBearer + } + + /// + /// Offset reset policy applied when a consumer group has no + /// committed offset for a partition, mirroring the librdkafka + /// auto.offset.reset property. + /// + /// + /// Implements the subscriber start-offset policy of + /// + /// Part 14 Annex B.2 Apache Kafka transport. + /// + public enum KafkaAutoOffsetReset + { + /// + /// Start consuming from the most recent record + /// (latest). + /// + Latest, + + /// + /// Start consuming from the earliest retained record + /// (earliest). + /// + Earliest + } + + /// + /// Connection-level options for the Apache Kafka broker transport. + /// Bound from IConfiguration via + /// EnableConfigurationBindingGenerator, instantiated by the + /// fluent DI surface, or supplied directly to the + /// . + /// + /// + /// + /// Mirrors the connection property surface defined in + /// + /// Part 14 Annex B.2 Apache Kafka transport. Credentials are + /// looked up through the OPC UA secret store via + /// ; no plain-text password field is + /// ever exposed. + /// + /// + /// All properties accept ISO-8601 duration + /// strings when bound from IConfiguration. + /// + /// + public sealed class KafkaConnectionOptions + { + /// + /// Broker endpoint URL — kafka://host[:port][,host[:port]...] + /// for plaintext or kafkas://host[:port][,...] for TLS. + /// The default port is 9092. + /// + public string Endpoint { get; set; } = string.Empty; + + /// + /// Comma-separated host:port bootstrap server list passed + /// to librdkafka bootstrap.servers. Populated by the + /// transport factory from when not set + /// explicitly. + /// + public string BootstrapServers { get; set; } = string.Empty; + + /// + /// Optional Kafka client.id; when + /// the transport derives one from the PubSubConnection's + /// PublisherId. + /// + public string? ClientId { get; set; } + + /// + /// Consumer group.id used by subscriber connections so + /// partitions and committed offsets are shared across the group. + /// + public string? GroupId { get; set; } + + /// + /// Transport security protocol. Defaults to + /// ; the transport + /// factory upgrades it to an SSL variant when the endpoint scheme + /// is kafkas. + /// + public KafkaSecurityProtocol SecurityProtocol { get; set; } + = KafkaSecurityProtocol.Plaintext; + + /// + /// SASL authentication mechanism. Defaults to + /// (no SASL). + /// + public KafkaSaslMechanism SaslMechanism { get; set; } = KafkaSaslMechanism.None; + + /// + /// Optional SASL user name. + /// + public string? UserName { get; set; } + + /// + /// Identifier of the password secret in the application's + /// ISecretStore. The transport factory resolves the secret + /// at connect time so the configuration file never carries the + /// cleartext password. disables password + /// authentication. + /// + public string? PasswordSecretId { get; set; } + + /// + /// Authentication profile URI used to select SASL authentication + /// per Part 14 §7.3.4.3. + /// + public string? AuthenticationProfileUri { get; set; } + + /// + /// Resource URI associated with + /// . + /// + public string? ResourceUri { get; set; } + + /// + /// Allows SASL credentials to be sent over plaintext + /// kafka:// connections. Defaults to + /// . + /// + public bool AllowCredentialsOverPlaintext { get; set; } + + /// + /// TLS options. picks up scheme-derived + /// defaults (TLS off for kafka://, on for + /// kafkas://). + /// + public KafkaTlsOptions? Tls { get; set; } + + /// + /// Producer delivery guarantee mapped to the librdkafka + /// acks / enable.idempotence settings. Defaults to + /// . + /// + public KafkaQualityOfService DeliveryGuarantee { get; set; } + = KafkaQualityOfService.AtLeastOnce; + + /// + /// Consumer offset reset policy. Defaults to + /// . + /// + public KafkaAutoOffsetReset AutoOffsetReset { get; set; } = KafkaAutoOffsetReset.Latest; + + /// + /// Whether the consumer commits offsets automatically. When + /// the transport commits each record + /// after it has been queued for delivery. Defaults to + /// . + /// + public bool EnableAutoCommit { get; set; } = true; + + /// + /// Topic-level options (fallback topic prefix). + /// + public KafkaTopicOptions Topics { get; set; } = new KafkaTopicOptions(); + + /// + /// Timeout applied to the initial metadata / connection exchange. + /// + public TimeSpan ConnectTimeout { get; set; } = TimeSpan.FromSeconds(10); + + /// + /// Maximum time the producer waits for a record to be delivered + /// before failing it, mapped to librdkafka + /// message.timeout.ms. + /// + public TimeSpan MessageTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Maximum size (in bytes) of a single Kafka record, mapped to + /// librdkafka message.max.bytes. The default of 1048576 + /// matches the common broker default. + /// + public int MaxMessageSize { get; set; } = 1048576; + + /// + /// Resolved password bytes populated by the transport factory + /// after looking up in the secret + /// store. Not bound from configuration; never persisted or + /// serialized. Adapter implementations consume this value when + /// establishing the SASL credentials. + /// + internal byte[]? PasswordBytes { get; set; } + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaConnectionStateChangedEventArgs.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaConnectionStateChangedEventArgs.cs new file mode 100644 index 0000000000..d303131fb8 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaConnectionStateChangedEventArgs.cs @@ -0,0 +1,76 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// Event payload raised by an IKafkaClientAdapter whenever the + /// underlying broker connection state changes. + /// + /// + /// Implements the connection state notification surface used by the + /// Kafka broker transport defined in + /// + /// Part 14 Annex B.2 Apache Kafka transport. + /// + public sealed class KafkaConnectionStateChangedEventArgs : EventArgs + { + /// + /// Initializes a new + /// . + /// + /// + /// when the adapter just transitioned to + /// Connected; when it transitioned + /// to Disconnected. + /// + /// + /// Optional human-readable explanation. Must not contain + /// sensitive data such as cleartext credentials. + /// + public KafkaConnectionStateChangedEventArgs(bool isConnected, string? reason) + { + IsConnected = isConnected; + Reason = reason; + } + + /// + /// when the adapter just transitioned to + /// the connected state. + /// + public bool IsConnected { get; } + + /// + /// Optional human-readable description of the transition. + /// + public string? Reason { get; } + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaEncoding.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaEncoding.cs new file mode 100644 index 0000000000..14fd5d6a50 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaEncoding.cs @@ -0,0 +1,109 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// Encoding tier carried by the Kafka NetworkMessage body and + /// advertised through the record content-type header. + /// + /// + /// Implements the message body encoding selector defined in + /// + /// Part 14 Annex B.2 Apache Kafka transport. The wire segment + /// is the lowercase enum name; the content-type strings are shared + /// with the MQTT broker transport. + /// + public enum KafkaEncoding + { + /// + /// UADP binary NetworkMessage body + /// (). + /// + Uadp, + + /// + /// JSON NetworkMessage body + /// (). + /// + Json + } + + /// + /// Extension helpers for . + /// + public static class KafkaEncodingExtensions + { + /// + /// Returns the lowercase topic segment for the given encoding. + /// + /// Encoding value. + /// + /// "uadp" for , + /// "json" for . + /// + /// + /// is not a defined value. + /// + public static string ToTopicSegment(this KafkaEncoding encoding) + { + return encoding switch + { + KafkaEncoding.Uadp => "uadp", + KafkaEncoding.Json => "json", + _ => throw new ArgumentOutOfRangeException(nameof(encoding)) + }; + } + + /// + /// Returns the record content-type header value for the + /// given encoding per Part 14 Annex B.2. + /// + /// Encoding value. + /// + /// "application/json" for , + /// "application/opcua+uadp" for + /// . + /// + /// + /// is not a defined value. + /// + public static string ToContentType(this KafkaEncoding encoding) + { + return encoding switch + { + KafkaEncoding.Uadp => "application/opcua+uadp", + KafkaEncoding.Json => "application/json", + _ => throw new ArgumentOutOfRangeException(nameof(encoding)) + }; + } + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaEndpoint.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaEndpoint.cs new file mode 100644 index 0000000000..c96096be46 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaEndpoint.cs @@ -0,0 +1,54 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// Parsed Kafka endpoint produced by + /// . Carries the normalised + /// comma-separated bootstrap server list plus a flag selecting + /// plaintext vs TLS so transport call sites do not re-parse the URL. + /// + /// + /// Implements the addressing surface of + /// + /// Part 14 Annex B.2 Apache Kafka transport. Kafka clients + /// connect to a list of bootstrap brokers rather than a single host, + /// so the endpoint carries the full host:port list; the + /// kafka / kafkas scheme is the only scheme-derived + /// signal for TLS. + /// + /// + /// Normalised comma-separated host:port bootstrap server list. + /// + /// + /// when the URL scheme was kafkas. + /// + public readonly record struct KafkaEndpoint(string BootstrapServers, bool UseTls); +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaEndpointParser.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaEndpointParser.cs new file mode 100644 index 0000000000..34b08aac1d --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaEndpointParser.cs @@ -0,0 +1,221 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Globalization; +using System.Text; + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// Dedicated parser for kafka:// and kafkas:// URLs. + /// Used instead of directly because a Kafka + /// endpoint carries a comma-separated bootstrap server list, which is + /// not a valid single authority; the parser also + /// makes the scheme validation and default-port selection explicit + /// and rejects malformed inputs with a precise + /// message. + /// + /// + /// Implements the URI parsing surface of + /// + /// Part 14 Annex B.2 Apache Kafka transport. The default port + /// follows the Apache Kafka convention (9092). + /// + public static class KafkaEndpointParser + { + /// + /// Kafka scheme for plaintext transport. + /// + public const string KafkaScheme = "kafka"; + + /// + /// Kafka scheme for TLS-protected transport. + /// + public const string KafkasScheme = "kafkas"; + + /// + /// Default Kafka broker port. + /// + public const int DefaultPort = 9092; + + /// + /// Parses into a strongly-typed + /// . + /// + /// + /// URL to parse (kafka://host[:port][,host[:port]...] or + /// kafkas://...). + /// + /// The parsed endpoint. + /// + /// is . + /// + /// + /// is malformed or uses a scheme other + /// than kafka / kafkas. + /// + public static KafkaEndpoint Parse(string url) + { + if (url is null) + { + throw new ArgumentNullException(nameof(url)); + } + if (url.Length == 0) + { + throw new FormatException("Kafka endpoint URL cannot be empty."); + } + + int schemeEnd = url.IndexOf("://", StringComparison.Ordinal); + if (schemeEnd <= 0) + { + throw new FormatException( + "Kafka endpoint must be of the form kafka[s]://host[:port][,host[:port]...]."); + } + string scheme = url.Substring(0, schemeEnd); + bool useTls; + if (string.Equals(scheme, KafkaScheme, StringComparison.OrdinalIgnoreCase)) + { + useTls = false; + } + else if (string.Equals(scheme, KafkasScheme, StringComparison.OrdinalIgnoreCase)) + { + useTls = true; + } + else + { + throw new FormatException( + "Kafka endpoint scheme must be 'kafka' or 'kafkas'."); + } + + string authority = url.Substring(schemeEnd + 3); + int pathStart = authority.IndexOf('/', StringComparison.Ordinal); + if (pathStart >= 0) + { + authority = authority.Substring(0, pathStart); + } + if (authority.Length == 0) + { + throw new FormatException("Kafka endpoint is missing the bootstrap server list."); + } + + string[] entries = authority.Split(','); + var builder = new StringBuilder(); + foreach (string rawEntry in entries) + { + string entry = rawEntry.Trim(); + if (entry.Length == 0) + { + throw new FormatException("Kafka endpoint has an empty bootstrap server entry."); + } + (string host, int port) = ParseHostPort(entry); + if (builder.Length > 0) + { + builder.Append(','); + } + builder.Append(host); + builder.Append(':'); + builder.Append(port.ToString(CultureInfo.InvariantCulture)); + } + return new KafkaEndpoint(builder.ToString(), useTls); + } + + private static (string Host, int Port) ParseHostPort(string entry) + { + if (entry[0] == '[') + { + int hostEnd = entry.IndexOf(']', StringComparison.Ordinal); + if (hostEnd < 0) + { + throw new FormatException( + "Kafka endpoint has an unterminated IPv6 literal."); + } + string ipv6Host = entry.Substring(1, hostEnd - 1); + if (ipv6Host.Length == 0) + { + throw new FormatException("Kafka endpoint has an empty IPv6 literal."); + } + int ipv6Port; + if (hostEnd + 1 < entry.Length) + { + if (entry[hostEnd + 1] != ':') + { + throw new FormatException( + "Kafka endpoint has an unexpected character after the IPv6 literal."); + } + ipv6Port = ParsePort(entry.Substring(hostEnd + 2)); + } + else + { + ipv6Port = DefaultPort; + } + return (string.Concat("[", ipv6Host, "]"), ipv6Port); + } + + int colon = entry.LastIndexOf(':'); + string host; + int port; + if (colon >= 0) + { + host = entry.Substring(0, colon); + port = ParsePort(entry.Substring(colon + 1)); + } + else + { + host = entry; + port = DefaultPort; + } + if (host.Length == 0) + { + throw new FormatException("Kafka endpoint is missing the host component."); + } + return (host, port); + } + + private static int ParsePort(string text) + { + if (text.Length == 0) + { + throw new FormatException("Kafka endpoint has an empty port component."); + } + if (!int.TryParse( + text, + NumberStyles.Integer, + CultureInfo.InvariantCulture, + out int port) + || port <= 0 + || port > 65535) + { + throw new FormatException( + "Kafka endpoint has an invalid port component (must be 1..65535)."); + } + return port; + } + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaIncomingMessageEventArgs.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaIncomingMessageEventArgs.cs new file mode 100644 index 0000000000..2a547e1c0a --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaIncomingMessageEventArgs.cs @@ -0,0 +1,71 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// Event payload raised by an IKafkaClientAdapter whenever a + /// fresh record arrives from the broker. + /// + /// + /// Implements the receive-side notification surface used by the + /// Kafka broker transport defined in + /// + /// Part 14 Annex B.2 Apache Kafka transport. + /// + public sealed class KafkaIncomingMessageEventArgs : EventArgs + { + /// + /// Initializes a new . + /// + /// The incoming Kafka record. + /// + /// Receive-time stamp from the transport clock. + /// + public KafkaIncomingMessageEventArgs(KafkaMessage message, DateTimeUtc receivedAt) + { + Message = message; + ReceivedAt = receivedAt; + } + + /// + /// The Kafka record delivered to the adapter. + /// + public KafkaMessage Message { get; } + + /// + /// Receive-time stamp captured by the adapter when the record + /// was first observed (used downstream for chunk reassembly + /// timeouts and diagnostic clocks). + /// + public DateTimeUtc ReceivedAt { get; } + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaMessage.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaMessage.cs new file mode 100644 index 0000000000..7c62b60f9f --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaMessage.cs @@ -0,0 +1,71 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// One outbound or inbound Kafka record exchanged through the + /// adapter. Modelled as a readonly record struct so it can be + /// moved through bounded channels without per-message allocation. + /// + /// + /// Implements the per-message payload envelope used for the JSON and + /// UADP body mappings of + /// + /// Part 14 Annex B.2 Apache Kafka transport. + /// selects the target partition (records sharing a key preserve + /// ordering); is emitted as the + /// content-type record header. + /// + /// + /// Topic to produce to / topic the record was consumed from. + /// + /// + /// Partition key bytes. Empty selects round-robin / sticky + /// partitioning. + /// + /// Raw record value bytes (the encoder's output). + /// + /// Value emitted as the content-type record header (e.g. + /// application/json, application/opcua+uadp). + /// + /// + /// Optional additional record headers beyond , + /// or when none are present. + /// + public readonly record struct KafkaMessage( + string Topic, + ReadOnlyMemory Key, + ReadOnlyMemory Value, + string? ContentType, + IReadOnlyDictionary? Headers); +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaPubSubTransportFactory.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaPubSubTransportFactory.cs new file mode 100644 index 0000000000..146d94659b --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaPubSubTransportFactory.cs @@ -0,0 +1,412 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using Microsoft.Extensions.Options; +using Opc.Ua.PubSub.Diagnostics; +using Opc.Ua.PubSub.Kafka.Internal; +using Opc.Ua.PubSub.Transports; + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// for the Apache Kafka broker + /// transport profiles + /// ( and + /// ). + /// + /// + /// + /// Implements + /// + /// Part 14 Annex B.2 Apache Kafka transport from the factory + /// side. Two instances are registered with DI — one per encoding + /// profile — so the transport registry can pick the right factory + /// based on the connection's TransportProfileUri field. + /// + /// + /// The factory resolves the password configured under + /// through the + /// application's before handing the + /// resolved bytes to the transport. The cleartext password is never + /// serialized into configuration. + /// + /// + public sealed class KafkaPubSubTransportFactory : IPubSubTransportFactory + { + private const string DefaultSecretStoreType = "InMemory"; + + private readonly IKafkaClientFactory m_clientFactory; + private readonly KafkaConnectionOptions m_defaultOptions; + private readonly ISecretRegistry? m_secretRegistry; + private readonly IPubSubDiagnostics? m_diagnostics; + private readonly string m_transportProfileUri; + + /// + /// Initializes a new . + /// + /// + /// One of + /// or + /// . Required + /// so the transport registry can dispatch to the right factory + /// per connection profile. + /// + /// + /// used to create the + /// underlying client adapter. Wired by DI; tests inject a fake. + /// + /// + /// Default connection options applied to each transport. The + /// caller may override per-connection via the connection's + /// ConnectionProperties. + /// + /// + /// Optional used to resolve + /// . + /// + /// + /// Optional shared diagnostics sink. The DI container wires the + /// per-component diagnostics container. + /// + public KafkaPubSubTransportFactory( + string transportProfileUri, + IKafkaClientFactory clientFactory, + IOptions defaultOptions, + ISecretRegistry? secretRegistry = null, + IPubSubDiagnostics? diagnostics = null) + { + if (string.IsNullOrEmpty(transportProfileUri)) + { + throw new ArgumentException( + "transportProfileUri must be supplied.", + nameof(transportProfileUri)); + } + if (!string.Equals( + transportProfileUri, + KafkaProfiles.PubSubKafkaJsonTransport, + StringComparison.Ordinal) + && !string.Equals( + transportProfileUri, + KafkaProfiles.PubSubKafkaUadpTransport, + StringComparison.Ordinal)) + { + throw new ArgumentException( + $"transportProfileUri '{transportProfileUri}' is not a Kafka profile.", + nameof(transportProfileUri)); + } + if (clientFactory is null) + { + throw new ArgumentNullException(nameof(clientFactory)); + } + if (defaultOptions is null) + { + throw new ArgumentNullException(nameof(defaultOptions)); + } + m_transportProfileUri = transportProfileUri; + m_clientFactory = clientFactory; + m_defaultOptions = defaultOptions.Value ?? new KafkaConnectionOptions(); + m_secretRegistry = secretRegistry; + m_diagnostics = diagnostics; + } + + /// + public string TransportProfileUri => m_transportProfileUri; + + /// + public IPubSubTransport Create( + PubSubConnectionDataType connection, + ITelemetryContext telemetry, + TimeProvider timeProvider) + { + if (connection is null) + { + throw new ArgumentNullException(nameof(connection)); + } + if (telemetry is null) + { + throw new ArgumentNullException(nameof(telemetry)); + } + if (timeProvider is null) + { + throw new ArgumentNullException(nameof(timeProvider)); + } + if (connection.Address.IsNull) + { + throw new NotSupportedException( + "PubSubConnection.Address is required for Kafka transport."); + } + if (!connection.Address.TryGetValue(out NetworkAddressUrlDataType? networkAddress) + || networkAddress is null) + { + throw new NotSupportedException( + "Kafka transport requires a NetworkAddressUrlDataType address payload."); + } + string? url = networkAddress.Url; + if (string.IsNullOrEmpty(url)) + { + throw new NotSupportedException( + "NetworkAddressUrlDataType.Url is required for Kafka transport."); + } + + KafkaEndpoint endpoint = KafkaEndpointParser.Parse(url); + KafkaConnectionOptions options = CloneOptionsWithEndpoint(m_defaultOptions, url, endpoint); + ApplyAuthenticationSettings(connection, options); + ApplyDeliveryGuarantee(connection, options); + ResolvePassword(options); + + PubSubTransportDirection direction = DetermineDirection(connection); + return new KafkaBrokerTransport( + connection, + endpoint, + direction, + options, + m_clientFactory, + telemetry, + timeProvider, + m_diagnostics); + } + + private static KafkaConnectionOptions CloneOptionsWithEndpoint( + KafkaConnectionOptions source, + string endpointUrl, + KafkaEndpoint endpoint) + { + var clone = new KafkaConnectionOptions + { + Endpoint = endpointUrl, + BootstrapServers = endpoint.BootstrapServers, + ClientId = source.ClientId, + GroupId = source.GroupId, + SecurityProtocol = source.SecurityProtocol, + SaslMechanism = source.SaslMechanism, + UserName = source.UserName, + PasswordSecretId = source.PasswordSecretId, + AuthenticationProfileUri = source.AuthenticationProfileUri, + ResourceUri = source.ResourceUri, + AllowCredentialsOverPlaintext = source.AllowCredentialsOverPlaintext, + Tls = source.Tls, + DeliveryGuarantee = source.DeliveryGuarantee, + AutoOffsetReset = source.AutoOffsetReset, + EnableAutoCommit = source.EnableAutoCommit, + Topics = source.Topics, + ConnectTimeout = source.ConnectTimeout, + MessageTimeout = source.MessageTimeout, + MaxMessageSize = source.MaxMessageSize + }; + if (endpoint.UseTls) + { + ApplyTlsUpgrade(clone); + } + return clone; + } + + private static void ApplyTlsUpgrade(KafkaConnectionOptions options) + { + KafkaTlsOptions tls = options.Tls ?? new KafkaTlsOptions(); + tls.UseTls = true; + options.Tls = tls; + options.SecurityProtocol = options.SecurityProtocol switch + { + KafkaSecurityProtocol.Plaintext => KafkaSecurityProtocol.Ssl, + KafkaSecurityProtocol.SaslPlaintext => KafkaSecurityProtocol.SaslSsl, + _ => options.SecurityProtocol + }; + } + + private static void ApplyDeliveryGuarantee( + PubSubConnectionDataType connection, + KafkaConnectionOptions options) + { + if (connection.WriterGroups.IsNull) + { + return; + } + foreach (WriterGroupDataType group in connection.WriterGroups) + { + if (group.TransportSettings.TryGetValue( + out BrokerWriterGroupTransportDataType? broker) + && broker is not null + && broker.RequestedDeliveryGuarantee + != BrokerTransportQualityOfService.NotSpecified) + { + options.DeliveryGuarantee = + KafkaQualityOfServiceExtensions.FromBrokerGuarantee( + broker.RequestedDeliveryGuarantee, + options.DeliveryGuarantee); + return; + } + } + } + + private static void ApplyAuthenticationSettings( + PubSubConnectionDataType connection, + KafkaConnectionOptions options) + { + if (!string.IsNullOrEmpty(options.AuthenticationProfileUri)) + { + return; + } + if (TryApplyBrokerAuthentication(connection.TransportSettings, options)) + { + return; + } + if (!connection.WriterGroups.IsNull) + { + foreach (WriterGroupDataType group in connection.WriterGroups) + { + if (TryApplyBrokerAuthentication(group.TransportSettings, options)) + { + return; + } + if (group.DataSetWriters.IsNull) + { + continue; + } + foreach (DataSetWriterDataType writer in group.DataSetWriters) + { + if (TryApplyBrokerAuthentication(writer.TransportSettings, options)) + { + return; + } + } + } + } + if (!connection.ReaderGroups.IsNull) + { + foreach (ReaderGroupDataType group in connection.ReaderGroups) + { + if (group.DataSetReaders.IsNull) + { + continue; + } + foreach (DataSetReaderDataType reader in group.DataSetReaders) + { + if (TryApplyBrokerAuthentication(reader.TransportSettings, options)) + { + return; + } + } + } + } + } + + private static bool TryApplyBrokerAuthentication( + ExtensionObject settings, + KafkaConnectionOptions options) + { + if (settings.TryGetValue(out BrokerWriterGroupTransportDataType? group) && group is not null) + { + options.AuthenticationProfileUri = group.AuthenticationProfileUri; + options.ResourceUri = group.ResourceUri; + } + else if (settings.TryGetValue(out BrokerDataSetWriterTransportDataType? writer) && writer is not null) + { + options.AuthenticationProfileUri = writer.AuthenticationProfileUri; + options.ResourceUri = writer.ResourceUri; + } + else if (settings.TryGetValue(out BrokerDataSetReaderTransportDataType? reader) && reader is not null) + { + options.AuthenticationProfileUri = reader.AuthenticationProfileUri; + options.ResourceUri = reader.ResourceUri; + } + else + { + return false; + } + if (string.IsNullOrEmpty(options.UserName) && !string.IsNullOrEmpty(options.ResourceUri)) + { + options.UserName = options.ResourceUri; + } + return true; + } + + private void ResolvePassword(KafkaConnectionOptions options) + { + if (string.IsNullOrEmpty(options.PasswordSecretId)) + { + return; + } + if (m_secretRegistry is null) + { + throw new InvalidOperationException( + "KafkaConnectionOptions.PasswordSecretId is set but no " + + "ISecretRegistry was registered with the transport factory."); + } + SecretIdentifier id = ParseSecretIdentifier(options.PasswordSecretId); + ISecret? secret = m_secretRegistry.TryGet(id); + if (secret is null) + { + throw new InvalidOperationException( + $"Password secret '{options.PasswordSecretId}' could not be " + + "resolved from the registered secret stores."); + } + try + { + options.PasswordBytes = secret.Bytes.ToArray(); + } + finally + { + secret.Dispose(); + } + } + + private static SecretIdentifier ParseSecretIdentifier(string secretId) + { + int separator = secretId.IndexOf(':', StringComparison.Ordinal); + if (separator <= 0 || separator >= secretId.Length - 1) + { + return new SecretIdentifier(secretId, DefaultSecretStoreType); + } + string storeType = secretId.Substring(0, separator); + string name = secretId.Substring(separator + 1); + return new SecretIdentifier(name, storeType); + } + + private static PubSubTransportDirection DetermineDirection( + PubSubConnectionDataType connection) + { + PubSubTransportDirection direction = PubSubTransportDirection.None; + if (!connection.WriterGroups.IsNull && connection.WriterGroups.Count > 0) + { + direction |= PubSubTransportDirection.Send; + } + if (!connection.ReaderGroups.IsNull && connection.ReaderGroups.Count > 0) + { + direction |= PubSubTransportDirection.Receive; + } + if (direction == PubSubTransportDirection.None) + { + direction = PubSubTransportDirection.SendReceive; + } + return direction; + } + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaQualityOfService.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaQualityOfService.cs new file mode 100644 index 0000000000..1fdbbdd07e --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaQualityOfService.cs @@ -0,0 +1,184 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// Kafka producer delivery guarantee mapped onto the Part 14 + /// BrokerTransportQualityOfService enumeration. + /// + /// + /// Implements the delivery-guarantee mapping of + /// + /// Part 14 Annex B.2 Apache Kafka transport. Unlike MQTT, where + /// the guarantee is expressed as a per-message QoS level, Kafka + /// realises the guarantee through producer configuration + /// (acks and enable.idempotence); the mapping is + /// therefore applied once at producer creation. + /// + public enum KafkaQualityOfService + { + /// + /// Fire-and-forget. The producer does not wait for any broker + /// acknowledgement (acks=0). Maps to + /// BrokerTransportQualityOfService.BestEffort. + /// + BestEffort, + + /// + /// The producer waits for the partition leader to acknowledge + /// the write (acks=1) without idempotence. Maps to + /// BrokerTransportQualityOfService.AtMostOnce. + /// + AtMostOnce, + + /// + /// The producer waits for all in-sync replicas to acknowledge + /// the write (acks=all) with retries but without + /// idempotence, so duplicates are possible on retry. Maps to + /// BrokerTransportQualityOfService.AtLeastOnce. + /// + AtLeastOnce, + + /// + /// The producer enables the idempotent producer + /// (enable.idempotence=true, implying acks=all) so + /// retries do not create duplicate records. Maps to + /// BrokerTransportQualityOfService.ExactlyOnce. + /// + ExactlyOnce + } + + /// + /// Broker acknowledgement level requested from the Kafka producer. + /// Numeric values match the librdkafka acks wire encoding so + /// the adapter can map without an extra lookup. + /// + /// + /// Backs the acks selector described in + /// + /// Part 14 Annex B.2 Apache Kafka transport. + /// + public enum KafkaAcks + { + /// + /// No acknowledgement is requested from the broker + /// (acks=0). + /// + None = 0, + + /// + /// Only the partition leader must acknowledge the write + /// (acks=1). + /// + Leader = 1, + + /// + /// All in-sync replicas must acknowledge the write + /// (acks=all). + /// + All = -1 + } + + /// + /// Resolved Kafka producer delivery settings derived from a + /// value. + /// + /// + /// Carries the two producer knobs that realise the Part 14 Annex B.2 + /// delivery guarantee so the Confluent-backed adapter can apply them + /// without re-deriving the mapping. + /// + /// Broker acknowledgement level. + /// + /// to enable the idempotent producer for + /// exactly-once semantics on retry. + /// + public readonly record struct KafkaDeliveryGuarantee(KafkaAcks Acks, bool EnableIdempotence); + + /// + /// Mapping helpers for . + /// + public static class KafkaQualityOfServiceExtensions + { + /// + /// Resolves the producer acks / enable.idempotence + /// settings for the given delivery guarantee per Part 14 Annex + /// B.2. + /// + /// Delivery guarantee value. + /// The resolved producer settings. + /// + /// is not a defined value. + /// + public static KafkaDeliveryGuarantee ToDeliveryGuarantee( + this KafkaQualityOfService qualityOfService) + { + return qualityOfService switch + { + KafkaQualityOfService.BestEffort => + new KafkaDeliveryGuarantee(KafkaAcks.None, EnableIdempotence: false), + KafkaQualityOfService.AtMostOnce => + new KafkaDeliveryGuarantee(KafkaAcks.Leader, EnableIdempotence: false), + KafkaQualityOfService.AtLeastOnce => + new KafkaDeliveryGuarantee(KafkaAcks.All, EnableIdempotence: false), + KafkaQualityOfService.ExactlyOnce => + new KafkaDeliveryGuarantee(KafkaAcks.All, EnableIdempotence: true), + _ => throw new ArgumentOutOfRangeException(nameof(qualityOfService)) + }; + } + + /// + /// Maps a Part 14 + /// value to the equivalent . + /// + /// Requested delivery guarantee. + /// + /// Value returned when is + /// or + /// an unrecognised value. + /// + /// The mapped delivery guarantee. + public static KafkaQualityOfService FromBrokerGuarantee( + BrokerTransportQualityOfService guarantee, + KafkaQualityOfService fallback) + { + return guarantee switch + { + BrokerTransportQualityOfService.BestEffort => KafkaQualityOfService.BestEffort, + BrokerTransportQualityOfService.AtMostOnce => KafkaQualityOfService.AtMostOnce, + BrokerTransportQualityOfService.AtLeastOnce => KafkaQualityOfService.AtLeastOnce, + BrokerTransportQualityOfService.ExactlyOnce => KafkaQualityOfService.ExactlyOnce, + _ => fallback + }; + } + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaTlsOptions.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaTlsOptions.cs new file mode 100644 index 0000000000..d251407409 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaTlsOptions.cs @@ -0,0 +1,89 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// TLS configuration for an Apache Kafka connection. The + /// connection's + /// scheme (kafka vs kafkas) drives the default + /// value; callers may override afterwards. + /// + /// + /// Backs the TLS transport surface required by + /// + /// Part 14 Annex B.2 Apache Kafka transport. Unlike the MQTT + /// transport, the underlying librdkafka client consumes certificate + /// and key material through file-system paths rather than the OPC UA + /// certificate store, so this POCO references PEM file locations. No + /// private key material is embedded in configuration files. + /// + public sealed class KafkaTlsOptions + { + /// + /// Enables TLS on the underlying broker connection. Defaults to + /// ; the transport factory sets it to + /// automatically when the endpoint scheme + /// is kafkas. + /// + public bool UseTls { get; set; } + + /// + /// When the client verifies the broker + /// certificate against the configured trust anchors; when + /// certificate verification is disabled. + /// Disabling verification should only be used for local + /// development. + /// + public bool ValidateServerCertificate { get; set; } = true; + + /// + /// Path to a PEM file containing the certificate authority (CA) + /// certificates that form the trust chain used to validate the + /// broker certificate. Maps to the librdkafka + /// ssl.ca.location property. defers + /// to the platform / runtime default trust store. + /// + public string? CaCertificatePath { get; set; } + + /// + /// Path to a PEM file containing the client certificate presented + /// during the TLS handshake for mutual TLS. Maps to the + /// librdkafka ssl.certificate.location property. + /// + public string? ClientCertificatePath { get; set; } + + /// + /// Path to a PEM file containing the client private key that + /// matches . Maps to the + /// librdkafka ssl.key.location property. + /// + public string? ClientKeyPath { get; set; } + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaTopicOptions.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaTopicOptions.cs new file mode 100644 index 0000000000..efb4aa00f9 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaTopicOptions.cs @@ -0,0 +1,56 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.PubSub.Kafka +{ + /// + /// Topic-level options that apply to every publish / subscribe on + /// the connection. Individual data / metadata topics are taken from + /// the connection's broker transport settings (QueueName / + /// MetaDataQueueName); this bag only carries the fallback prefix used + /// when a writer / reader does not specify an explicit Kafka topic. + /// + /// + /// Implements the topic addressing surface of + /// + /// Part 14 Annex B.2 Apache Kafka transport. Apache Kafka topic + /// names are restricted to the character set + /// [a-zA-Z0-9._-], so the fallback naming scheme uses + /// . as the segment separator rather than the MQTT /. + /// + public sealed class KafkaTopicOptions + { + /// + /// Topic prefix used as the first segment of every fallback data + /// / metadata topic when the connection does not carry an + /// explicit Kafka topic (QueueName). Defaults to opcua. + /// + public string Prefix { get; set; } = "opcua"; + } +} diff --git a/Libraries/Opc.Ua.PubSub.Kafka/Opc.Ua.PubSub.Kafka.csproj b/Libraries/Opc.Ua.PubSub.Kafka/Opc.Ua.PubSub.Kafka.csproj index fe0425b444..b5309c0050 100644 --- a/Libraries/Opc.Ua.PubSub.Kafka/Opc.Ua.PubSub.Kafka.csproj +++ b/Libraries/Opc.Ua.PubSub.Kafka/Opc.Ua.PubSub.Kafka.csproj @@ -12,10 +12,12 @@ $(NoWarn);CS1591 true + true @@ -30,7 +32,11 @@ - + + + + + From a87fa392ff51a4eca17dbbb439c46c0c2c914a3e Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Fri, 3 Jul 2026 12:03:27 +0200 Subject: [PATCH 03/21] Add PubSub redundancy/HA coordination abstraction (#3942, #3924) Transport-agnostic HA foundation for Part 14 Sec 9.1.6, additive and behaviour-neutral: - PubSubRedundancyMode (None/Cold/Warm/Hot), PubSubComponentRole (Active/Standby), PubSubRoleChangedEventArgs. - IPubSubActivationCoordinator (per-component Active/Standby decision + RoleChanged) with the default AlwaysActiveCoordinator (no-op: every component active, so non-redundant deployments are unaffected). - IPubSubLeaseStore + PubSubLease (monotonic fencing token) for leader election, with a single-process InMemoryPubSubLeaseStore. The runtime wiring (pause standby / resume active in WriterGroup/ReaderGroup/PubSubConnection) and the lease-based coordinator + DI follow. Builds clean (0 warnings) on net10. --- .../Redundancy/AlwaysActiveCoordinator.cs | 80 ++++++++++ .../IPubSubActivationCoordinator.cs | 91 ++++++++++++ .../Redundancy/IPubSubLeaseStore.cs | 96 ++++++++++++ .../Redundancy/InMemoryPubSubLeaseStore.cs | 138 ++++++++++++++++++ .../Redundancy/PubSubComponentRole.cs | 50 +++++++ .../Opc.Ua.PubSub/Redundancy/PubSubLease.cs | 63 ++++++++ .../Redundancy/PubSubRedundancyMode.cs | 69 +++++++++ .../Redundancy/PubSubRoleChangedEventArgs.cs | 64 ++++++++ 8 files changed, 651 insertions(+) create mode 100644 Libraries/Opc.Ua.PubSub/Redundancy/AlwaysActiveCoordinator.cs create mode 100644 Libraries/Opc.Ua.PubSub/Redundancy/IPubSubActivationCoordinator.cs create mode 100644 Libraries/Opc.Ua.PubSub/Redundancy/IPubSubLeaseStore.cs create mode 100644 Libraries/Opc.Ua.PubSub/Redundancy/InMemoryPubSubLeaseStore.cs create mode 100644 Libraries/Opc.Ua.PubSub/Redundancy/PubSubComponentRole.cs create mode 100644 Libraries/Opc.Ua.PubSub/Redundancy/PubSubLease.cs create mode 100644 Libraries/Opc.Ua.PubSub/Redundancy/PubSubRedundancyMode.cs create mode 100644 Libraries/Opc.Ua.PubSub/Redundancy/PubSubRoleChangedEventArgs.cs diff --git a/Libraries/Opc.Ua.PubSub/Redundancy/AlwaysActiveCoordinator.cs b/Libraries/Opc.Ua.PubSub/Redundancy/AlwaysActiveCoordinator.cs new file mode 100644 index 0000000000..c1d8c99aee --- /dev/null +++ b/Libraries/Opc.Ua.PubSub/Redundancy/AlwaysActiveCoordinator.cs @@ -0,0 +1,80 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.PubSub.Redundancy +{ + /// + /// Default that reports every + /// component . + /// + /// + /// Registered by default so non-redundant deployments behave exactly as + /// before: no component is ever placed on standby and + /// never fires. Redundant deployments replace + /// this with a shared-store-backed coordinator. + /// + public sealed class AlwaysActiveCoordinator : IPubSubActivationCoordinator + { + /// + /// A shared stateless instance. + /// + public static AlwaysActiveCoordinator Instance { get; } = new(); + + /// + public event EventHandler? RoleChanged + { + add { } + remove { } + } + + /// + public ValueTask StartAsync(CancellationToken cancellationToken = default) + { + return default; + } + + /// + public ValueTask StopAsync(CancellationToken cancellationToken = default) + { + return default; + } + + /// + public ValueTask GetRoleAsync( + string componentId, + CancellationToken cancellationToken = default) + { + return new ValueTask(PubSubComponentRole.Active); + } + } +} diff --git a/Libraries/Opc.Ua.PubSub/Redundancy/IPubSubActivationCoordinator.cs b/Libraries/Opc.Ua.PubSub/Redundancy/IPubSubActivationCoordinator.cs new file mode 100644 index 0000000000..9a79ad1985 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub/Redundancy/IPubSubActivationCoordinator.cs @@ -0,0 +1,91 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.PubSub.Redundancy +{ + /// + /// Coordinates which PubSub components are active on this instance in a + /// redundant (high-availability) deployment, per OPC UA Part 14 §9.1.6. + /// + /// + /// + /// A coordinator answers, per component, whether this instance should be + /// (drive the transport) or + /// (wait to take over), and + /// raises when that decision changes so the + /// runtime can pause or resume the component. + /// + /// + /// The default registration is , + /// which reports every component active so non-redundant deployments are + /// unaffected. Redundant deployments register a shared-store-backed + /// coordinator (for example a lease-based one) via dependency injection. + /// Implementations are a provider extension point and must be injectable. + /// + /// + public interface IPubSubActivationCoordinator + { + /// + /// Starts coordination (for example begins acquiring and renewing + /// leadership leases). Idempotent. + /// + /// Cancellation token. + ValueTask StartAsync(CancellationToken cancellationToken = default); + + /// + /// Stops coordination and relinquishes any acquired leadership. + /// Idempotent. + /// + /// Cancellation token. + ValueTask StopAsync(CancellationToken cancellationToken = default); + + /// + /// Returns the current role of the identified component on this + /// instance. + /// + /// + /// Deterministic component identifier (for example + /// pubsub:writergroup:WriterGroup1). + /// + /// Cancellation token. + /// The component's current role. + ValueTask GetRoleAsync( + string componentId, + CancellationToken cancellationToken = default); + + /// + /// Raised whenever the role of a component changes on this instance. + /// + event EventHandler? RoleChanged; + } +} diff --git a/Libraries/Opc.Ua.PubSub/Redundancy/IPubSubLeaseStore.cs b/Libraries/Opc.Ua.PubSub/Redundancy/IPubSubLeaseStore.cs new file mode 100644 index 0000000000..1e7241da3b --- /dev/null +++ b/Libraries/Opc.Ua.PubSub/Redundancy/IPubSubLeaseStore.cs @@ -0,0 +1,96 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.PubSub.Redundancy +{ + /// + /// Shared store used to elect the active instance of a redundant PubSub + /// component through leadership leases. + /// + /// + /// Implements the externalized coordination state required for OPC UA + /// Part 14 §9.1.6 HA deployments. A distributed implementation (for + /// example backed by the shared runtime-state store, a database, or a + /// broker primitive) lets multiple instances agree on a single active + /// owner per lease key, with a monotonic fencing token on each ownership + /// change. The default is + /// single-process and intended for tests and single-instance use. + /// + public interface IPubSubLeaseStore + { + /// + /// Attempts to acquire the lease for on + /// behalf of . Succeeds when the lease is + /// unheld or expired, or already held by the same owner (renewal). + /// + /// Contended resource key. + /// Requesting instance identifier. + /// Requested lease duration. + /// Cancellation token. + /// + /// The acquired lease, or when the lease is + /// currently held by another live owner. + /// + ValueTask TryAcquireAsync( + string leaseKey, + string ownerId, + TimeSpan duration, + CancellationToken cancellationToken = default); + + /// + /// Attempts to renew a previously acquired lease. Fails when the lease + /// has been taken over by another owner (fencing token advanced). + /// + /// The lease to renew. + /// New lease duration from now. + /// Cancellation token. + /// + /// The renewed lease, or when renewal was + /// rejected because ownership was lost. + /// + ValueTask TryRenewAsync( + PubSubLease lease, + TimeSpan duration, + CancellationToken cancellationToken = default); + + /// + /// Releases a held lease so another instance can acquire it + /// immediately. No-op when the lease was already lost. + /// + /// The lease to release. + /// Cancellation token. + ValueTask ReleaseAsync( + PubSubLease lease, + CancellationToken cancellationToken = default); + } +} diff --git a/Libraries/Opc.Ua.PubSub/Redundancy/InMemoryPubSubLeaseStore.cs b/Libraries/Opc.Ua.PubSub/Redundancy/InMemoryPubSubLeaseStore.cs new file mode 100644 index 0000000000..035a5a123e --- /dev/null +++ b/Libraries/Opc.Ua.PubSub/Redundancy/InMemoryPubSubLeaseStore.cs @@ -0,0 +1,138 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.PubSub.Redundancy +{ + /// + /// Single-process backed by an in-memory + /// dictionary. Intended for tests and single-instance deployments; a + /// distributed store is required for genuine multi-instance failover. + /// + public sealed class InMemoryPubSubLeaseStore : IPubSubLeaseStore + { + private readonly TimeProvider m_timeProvider; + private readonly System.Threading.Lock m_gate = new(); + private readonly Dictionary m_leases = new(StringComparer.Ordinal); + private long m_fencingToken; + + /// + /// Initializes a new . + /// + /// + /// Clock used to evaluate lease expiry. Defaults to + /// . + /// + public InMemoryPubSubLeaseStore(TimeProvider? timeProvider = null) + { + m_timeProvider = timeProvider ?? TimeProvider.System; + } + + /// + public ValueTask TryAcquireAsync( + string leaseKey, + string ownerId, + TimeSpan duration, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(leaseKey)) + { + throw new ArgumentException("leaseKey is required.", nameof(leaseKey)); + } + if (string.IsNullOrEmpty(ownerId)) + { + throw new ArgumentException("ownerId is required.", nameof(ownerId)); + } + cancellationToken.ThrowIfCancellationRequested(); + + DateTimeOffset now = m_timeProvider.GetUtcNow(); + lock (m_gate) + { + if (m_leases.TryGetValue(leaseKey, out PubSubLease existing) + && existing.ExpiresAt > now + && !string.Equals(existing.OwnerId, ownerId, StringComparison.Ordinal)) + { + return new ValueTask((PubSubLease?)null); + } + + long token = string.Equals(existing.OwnerId, ownerId, StringComparison.Ordinal) + && existing.ExpiresAt > now + ? existing.FencingToken + : ++m_fencingToken; + var lease = new PubSubLease(leaseKey, ownerId, token, now + duration); + m_leases[leaseKey] = lease; + return new ValueTask(lease); + } + } + + /// + public ValueTask TryRenewAsync( + PubSubLease lease, + TimeSpan duration, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + DateTimeOffset now = m_timeProvider.GetUtcNow(); + lock (m_gate) + { + if (!m_leases.TryGetValue(lease.LeaseKey, out PubSubLease current) + || current.FencingToken != lease.FencingToken + || !string.Equals(current.OwnerId, lease.OwnerId, StringComparison.Ordinal)) + { + return new ValueTask((PubSubLease?)null); + } + var renewed = new PubSubLease( + lease.LeaseKey, lease.OwnerId, lease.FencingToken, now + duration); + m_leases[lease.LeaseKey] = renewed; + return new ValueTask(renewed); + } + } + + /// + public ValueTask ReleaseAsync( + PubSubLease lease, + CancellationToken cancellationToken = default) + { + lock (m_gate) + { + if (m_leases.TryGetValue(lease.LeaseKey, out PubSubLease current) + && current.FencingToken == lease.FencingToken + && string.Equals(current.OwnerId, lease.OwnerId, StringComparison.Ordinal)) + { + m_leases.Remove(lease.LeaseKey); + } + } + return default; + } + } +} diff --git a/Libraries/Opc.Ua.PubSub/Redundancy/PubSubComponentRole.cs b/Libraries/Opc.Ua.PubSub/Redundancy/PubSubComponentRole.cs new file mode 100644 index 0000000000..c58afbc74a --- /dev/null +++ b/Libraries/Opc.Ua.PubSub/Redundancy/PubSubComponentRole.cs @@ -0,0 +1,50 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.PubSub.Redundancy +{ + /// + /// The redundancy role a PubSub component (connection, writer group, or + /// reader group) currently holds on this instance. + /// + public enum PubSubComponentRole + { + /// + /// The component is active on this instance and drives its transport + /// (publishes network messages / dispatches received messages). + /// + Active = 0, + + /// + /// The component is on standby on this instance and does not drive + /// its transport; it waits to take over from a failed active peer. + /// + Standby = 1 + } +} diff --git a/Libraries/Opc.Ua.PubSub/Redundancy/PubSubLease.cs b/Libraries/Opc.Ua.PubSub/Redundancy/PubSubLease.cs new file mode 100644 index 0000000000..b03666e328 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub/Redundancy/PubSubLease.cs @@ -0,0 +1,63 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.PubSub.Redundancy +{ + /// + /// An acquired leadership lease used to elect the active instance of a + /// redundant PubSub component. + /// + /// + /// The is a monotonically increasing value the + /// store assigns each time the lease changes owner; downstream consumers + /// can use it to reject stale writes from a superseded active instance + /// (fencing), per the externalized-state requirements of OPC UA Part 14 + /// §9.1.6. + /// + /// + /// Stable key identifying the contended resource (for example a writer + /// group component id). + /// + /// + /// Identifier of the instance currently holding the lease. + /// + /// + /// Monotonic token incremented on every ownership change. + /// + /// + /// Absolute UTC time at which the lease expires unless renewed. + /// + public readonly record struct PubSubLease( + string LeaseKey, + string OwnerId, + long FencingToken, + DateTimeOffset ExpiresAt); +} diff --git a/Libraries/Opc.Ua.PubSub/Redundancy/PubSubRedundancyMode.cs b/Libraries/Opc.Ua.PubSub/Redundancy/PubSubRedundancyMode.cs new file mode 100644 index 0000000000..ab4f722fb4 --- /dev/null +++ b/Libraries/Opc.Ua.PubSub/Redundancy/PubSubRedundancyMode.cs @@ -0,0 +1,69 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.PubSub.Redundancy +{ + /// + /// PubSub redundancy behaviour for a redundant set of publishers or + /// subscribers, per OPC UA Part 14 §9.1.6. + /// + /// + /// The mode selects how much runtime state a standby instance keeps + /// warm so it can take over from a failed active instance. Cold rebuilds + /// from the shared configuration/runtime-state stores on failover, warm + /// keeps the configuration enabled but paused, and hot additionally + /// tracks live sequence state so take-over introduces no gap. + /// + public enum PubSubRedundancyMode + { + /// + /// No redundancy — the instance is always active. + /// + None = 0, + + /// + /// Cold standby: a standby instance rebuilds configuration and + /// runtime state from the shared stores only after the active + /// instance fails. + /// + Cold = 1, + + /// + /// Warm standby: a standby instance keeps its configuration loaded + /// and paused, ready to resume quickly on failover. + /// + Warm = 2, + + /// + /// Hot standby: a standby instance additionally tracks live + /// sequence/keep-alive state so take-over is seamless. + /// + Hot = 3 + } +} diff --git a/Libraries/Opc.Ua.PubSub/Redundancy/PubSubRoleChangedEventArgs.cs b/Libraries/Opc.Ua.PubSub/Redundancy/PubSubRoleChangedEventArgs.cs new file mode 100644 index 0000000000..c928f1d46b --- /dev/null +++ b/Libraries/Opc.Ua.PubSub/Redundancy/PubSubRoleChangedEventArgs.cs @@ -0,0 +1,64 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.PubSub.Redundancy +{ + /// + /// Raised by an when the + /// redundancy role of a component changes on this instance. + /// + public sealed class PubSubRoleChangedEventArgs : EventArgs + { + /// + /// Initializes a new . + /// + /// + /// Deterministic component identifier whose role changed. + /// + /// The new role. + public PubSubRoleChangedEventArgs(string componentId, PubSubComponentRole role) + { + ComponentId = componentId ?? throw new ArgumentNullException(nameof(componentId)); + Role = role; + } + + /// + /// Deterministic component identifier whose role changed (for + /// example pubsub:writergroup:WriterGroup1). + /// + public string ComponentId { get; } + + /// + /// The role now held by the component on this instance. + /// + public PubSubComponentRole Role { get; } + } +} From 53cd72d3932ef12e65d6dedfc3cf14a2ef8c496c Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Fri, 3 Jul 2026 12:22:12 +0200 Subject: [PATCH 04/21] Add lease-based PubSub activation coordinator (#3942, #3924) LeaseActivationCoordinator elects the active instance of each component via leadership leases in a shared IPubSubLeaseStore: the lease holder is Active, others are Standby and take over automatically when renewal stops. Per-component background renewal/acquire loops raise RoleChanged on transitions; fencing tokens guard against superseded owners. Cross-TFM (uses the TimeProvider.Delay extension and Cancel()). Builds clean (0 warnings) on net10 and net48. --- .../Redundancy/LeaseActivationCoordinator.cs | 336 ++++++++++++++++++ 1 file changed, 336 insertions(+) create mode 100644 Libraries/Opc.Ua.PubSub/Redundancy/LeaseActivationCoordinator.cs diff --git a/Libraries/Opc.Ua.PubSub/Redundancy/LeaseActivationCoordinator.cs b/Libraries/Opc.Ua.PubSub/Redundancy/LeaseActivationCoordinator.cs new file mode 100644 index 0000000000..108340885e --- /dev/null +++ b/Libraries/Opc.Ua.PubSub/Redundancy/LeaseActivationCoordinator.cs @@ -0,0 +1,336 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Opc.Ua.PubSub.Redundancy +{ + /// + /// that elects the active + /// instance of each component through leadership leases held in a shared + /// , per OPC UA Part 14 §9.1.6. + /// + /// + /// Each component is backed by one lease. The instance that holds the + /// lease is ; the others are + /// and continuously attempt to + /// acquire it, taking over automatically when the active instance stops + /// renewing. A distributed lease store provides genuine cross-instance + /// failover; the default in-memory store is single-process. + /// + public sealed class LeaseActivationCoordinator : IPubSubActivationCoordinator, IAsyncDisposable + { + private readonly IPubSubLeaseStore m_leaseStore; + private readonly TimeProvider m_timeProvider; + private readonly ILogger m_logger; + private readonly string m_ownerId; + private readonly TimeSpan m_leaseDuration; + private readonly TimeSpan m_renewInterval; + private readonly TimeSpan m_retryInterval; + private readonly System.Threading.Lock m_gate = new(); + private readonly Dictionary m_loops = new(StringComparer.Ordinal); + private CancellationTokenSource? m_cts; + private bool m_started; + private bool m_disposed; + + /// + /// Initializes a new . + /// + /// Shared lease store used for election. + /// Telemetry context for logging. + /// + /// Stable identifier of this instance. Defaults to a random value. + /// + /// + /// Lease time-to-live. Defaults to 15 seconds. + /// + /// + /// Interval at which the active instance renews its leases. Defaults + /// to a third of . + /// + /// + /// Interval at which a standby instance retries acquisition. Defaults + /// to a third of . + /// + /// Clock used for scheduling. + public LeaseActivationCoordinator( + IPubSubLeaseStore leaseStore, + ITelemetryContext? telemetry = null, + string? ownerId = null, + TimeSpan? leaseDuration = null, + TimeSpan? renewInterval = null, + TimeSpan? retryInterval = null, + TimeProvider? timeProvider = null) + { + m_leaseStore = leaseStore ?? throw new ArgumentNullException(nameof(leaseStore)); + m_timeProvider = timeProvider ?? TimeProvider.System; + m_logger = telemetry.CreateLogger(); + m_ownerId = string.IsNullOrEmpty(ownerId) + ? Guid.NewGuid().ToString("N") + : ownerId!; + m_leaseDuration = leaseDuration is { } d && d > TimeSpan.Zero + ? d + : TimeSpan.FromSeconds(15); + m_renewInterval = renewInterval is { } r && r > TimeSpan.Zero + ? r + : TimeSpan.FromTicks(m_leaseDuration.Ticks / 3); + m_retryInterval = retryInterval is { } t && t > TimeSpan.Zero + ? t + : TimeSpan.FromTicks(m_leaseDuration.Ticks / 3); + } + + /// + public event EventHandler? RoleChanged; + + /// + public ValueTask StartAsync(CancellationToken cancellationToken = default) + { + lock (m_gate) + { + if (m_disposed) + { + throw new ObjectDisposedException(nameof(LeaseActivationCoordinator)); + } + if (!m_started) + { + m_cts = new CancellationTokenSource(); + m_started = true; + } + } + return default; + } + + /// + public async ValueTask StopAsync(CancellationToken cancellationToken = default) + { + List loops; + CancellationTokenSource? cts; + lock (m_gate) + { + m_started = false; + cts = m_cts; + m_cts = null; + loops = [.. m_loops.Values]; + m_loops.Clear(); + } + if (cts is not null) + { + cts.Cancel(); + } + foreach (ComponentLoop loop in loops) + { + await loop.StopAsync().ConfigureAwait(false); + } + cts?.Dispose(); + } + + /// + public ValueTask GetRoleAsync( + string componentId, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(componentId)) + { + throw new ArgumentException("componentId is required.", nameof(componentId)); + } + ComponentLoop loop; + lock (m_gate) + { + if (m_disposed) + { + throw new ObjectDisposedException(nameof(LeaseActivationCoordinator)); + } + if (!m_started || m_cts is null) + { + return new ValueTask(PubSubComponentRole.Standby); + } + if (!m_loops.TryGetValue(componentId, out ComponentLoop? existing)) + { + existing = new ComponentLoop(this, componentId, m_cts.Token); + m_loops[componentId] = existing; + existing.Start(); + } + loop = existing; + } + return new ValueTask(loop.Role); + } + + /// + public async ValueTask DisposeAsync() + { + lock (m_gate) + { + if (m_disposed) + { + return; + } + m_disposed = true; + } + await StopAsync().ConfigureAwait(false); + } + + private void RaiseRoleChanged(string componentId, PubSubComponentRole role) + { + RoleChanged?.Invoke(this, new PubSubRoleChangedEventArgs(componentId, role)); + } + + private sealed class ComponentLoop + { + private readonly LeaseActivationCoordinator m_owner; + private readonly string m_componentId; + private readonly CancellationToken m_token; + private Task? m_task; + private volatile int m_role = (int)PubSubComponentRole.Standby; + + public ComponentLoop( + LeaseActivationCoordinator owner, + string componentId, + CancellationToken token) + { + m_owner = owner; + m_componentId = componentId; + m_token = token; + } + + public PubSubComponentRole Role => (PubSubComponentRole)m_role; + + public void Start() + { + m_task = Task.Run(() => RunAsync(), CancellationToken.None); + } + + public async ValueTask StopAsync() + { + Task? task = m_task; + if (task is not null) + { + try + { + await task.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Expected on shutdown. + } + } + } + + private async Task RunAsync() + { + PubSubLease? held = null; + try + { + while (!m_token.IsCancellationRequested) + { + if (held is null) + { + held = await m_owner.m_leaseStore.TryAcquireAsync( + m_componentId, + m_owner.m_ownerId, + m_owner.m_leaseDuration, + m_token).ConfigureAwait(false); + if (held is not null) + { + SetRole(PubSubComponentRole.Active); + await DelayAsync(m_owner.m_renewInterval).ConfigureAwait(false); + } + else + { + SetRole(PubSubComponentRole.Standby); + await DelayAsync(m_owner.m_retryInterval).ConfigureAwait(false); + } + } + else + { + PubSubLease? renewed = await m_owner.m_leaseStore.TryRenewAsync( + held.Value, + m_owner.m_leaseDuration, + m_token).ConfigureAwait(false); + if (renewed is null) + { + held = null; + SetRole(PubSubComponentRole.Standby); + continue; + } + held = renewed; + await DelayAsync(m_owner.m_renewInterval).ConfigureAwait(false); + } + } + } + catch (OperationCanceledException) + { + // Expected on shutdown. + } + catch (Exception ex) + { + m_owner.m_logger.LogError( + ex, + "PubSub lease coordination loop for '{ComponentId}' faulted.", + m_componentId); + } + finally + { + if (held is not null) + { + try + { + await m_owner.m_leaseStore.ReleaseAsync(held.Value, CancellationToken.None) + .ConfigureAwait(false); + } + catch (Exception ex) + { + m_owner.m_logger.LogDebug( + ex, + "Releasing lease for '{ComponentId}' on shutdown failed.", + m_componentId); + } + } + } + } + + private async Task DelayAsync(TimeSpan delay) + { + await m_owner.m_timeProvider.Delay(delay, m_token).ConfigureAwait(false); + } + + private void SetRole(PubSubComponentRole role) + { + int previous = Interlocked.Exchange(ref m_role, (int)role); + if (previous != (int)role) + { + m_owner.RaiseRoleChanged(m_componentId, role); + } + } + } + } +} From 5b49219c714e6bf5b63bcff69f7e49d55c41e659 Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Fri, 3 Jul 2026 12:33:03 +0200 Subject: [PATCH 05/21] Wire PubSub HA coordination into the DI builder (#3942, #3924) Add IPubSubBuilder.WithActivationCoordinator / WithLeaseStore and register the default AlwaysActiveCoordinator (no-op) so non-redundant deployments are unaffected. Completes the transport-agnostic HA foundation; the runtime pause/resume wiring consumes IPubSubActivationCoordinator from DI. Builds clean (0 warnings) on net10. --- .../DependencyInjection/IPubSubBuilder.cs | 17 +++++++++++++ .../OpcUaPubSubBuilderExtensions.cs | 2 ++ .../DependencyInjection/PubSubBuilder.cs | 25 +++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/Libraries/Opc.Ua.PubSub/DependencyInjection/IPubSubBuilder.cs b/Libraries/Opc.Ua.PubSub/DependencyInjection/IPubSubBuilder.cs index 3915456340..954d59ac61 100644 --- a/Libraries/Opc.Ua.PubSub/DependencyInjection/IPubSubBuilder.cs +++ b/Libraries/Opc.Ua.PubSub/DependencyInjection/IPubSubBuilder.cs @@ -34,6 +34,7 @@ using Opc.Ua.PubSub.Application; using Opc.Ua.PubSub.Configuration; using Opc.Ua.PubSub.DataSets; +using Opc.Ua.PubSub.Redundancy; using Opc.Ua.PubSub.Security; using Opc.Ua.PubSub.Security.Sks; @@ -115,6 +116,22 @@ IPubSubBuilder ConfigureApplication( /// Security-key store. IPubSubBuilder WithSecurityKeyStore(IPubSubSecurityKeyStore store); + /// + /// Uses a custom PubSub activation coordinator that decides, per + /// component, whether this instance is active or on standby in a + /// redundant (high-availability) deployment (Part 14 §9.1.6). + /// + /// Activation coordinator. + IPubSubBuilder WithActivationCoordinator(IPubSubActivationCoordinator coordinator); + + /// + /// Uses a custom PubSub lease store for leader election in a redundant + /// deployment. A distributed store enables genuine multi-instance + /// failover. + /// + /// Lease store. + IPubSubBuilder WithLeaseStore(IPubSubLeaseStore leaseStore); + /// /// Adds a responder-side PubSub Action handler. /// diff --git a/Libraries/Opc.Ua.PubSub/DependencyInjection/OpcUaPubSubBuilderExtensions.cs b/Libraries/Opc.Ua.PubSub/DependencyInjection/OpcUaPubSubBuilderExtensions.cs index f493039215..00f0d21a4c 100644 --- a/Libraries/Opc.Ua.PubSub/DependencyInjection/OpcUaPubSubBuilderExtensions.cs +++ b/Libraries/Opc.Ua.PubSub/DependencyInjection/OpcUaPubSubBuilderExtensions.cs @@ -44,6 +44,7 @@ using Opc.Ua.PubSub.Encoding.Json; using Opc.Ua.PubSub.Encoding.Uadp; using Opc.Ua.PubSub.MetaData; +using Opc.Ua.PubSub.Redundancy; using Opc.Ua.PubSub.Scheduling; using Opc.Ua.PubSub.Security; using Opc.Ua.PubSub.Security.Policies; @@ -267,6 +268,7 @@ private static void RegisterCoreServices(IServiceCollection services) services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); + services.TryAddSingleton(AlwaysActiveCoordinator.Instance); services.TryAddSingleton(); services.TryAddSingleton(); diff --git a/Libraries/Opc.Ua.PubSub/DependencyInjection/PubSubBuilder.cs b/Libraries/Opc.Ua.PubSub/DependencyInjection/PubSubBuilder.cs index 5416a9248d..7726fcd342 100644 --- a/Libraries/Opc.Ua.PubSub/DependencyInjection/PubSubBuilder.cs +++ b/Libraries/Opc.Ua.PubSub/DependencyInjection/PubSubBuilder.cs @@ -37,6 +37,7 @@ using Opc.Ua.PubSub.Application; using Opc.Ua.PubSub.Configuration; using Opc.Ua.PubSub.DataSets; +using Opc.Ua.PubSub.Redundancy; using Opc.Ua.PubSub.Security; using Opc.Ua.PubSub.Security.Sks; using Opc.Ua.PubSub.Transports; @@ -159,6 +160,30 @@ public IPubSubBuilder WithRuntimeStateStore(IPubSubRuntimeStateStore store) return this; } + /// + public IPubSubBuilder WithActivationCoordinator(IPubSubActivationCoordinator coordinator) + { + if (coordinator is null) + { + throw new ArgumentNullException(nameof(coordinator)); + } + + Services.AddSingleton(coordinator); + return this; + } + + /// + public IPubSubBuilder WithLeaseStore(IPubSubLeaseStore leaseStore) + { + if (leaseStore is null) + { + throw new ArgumentNullException(nameof(leaseStore)); + } + + Services.AddSingleton(leaseStore); + return this; + } + /// public IPubSubBuilder WithSecurityKeyStore(IPubSubSecurityKeyStore store) { From 4a3e1c0d6cbb934f7ed14d379dccf8a57be15885 Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Fri, 3 Jul 2026 13:04:13 +0200 Subject: [PATCH 06/21] Wire Kafka into the reference sample + docs; AOT-validated (#3942) Add KafkaUadp/KafkaJson profiles to ConsoleReferencePubSubClient (publisher + subscriber), AddKafkaTransport wiring, profile->URI mapping, kafka://localhost:9092 default, and the Opc.Ua.PubSub.Kafka ProjectReference. Since net10 Kafka uses the pure-managed Dekaf client, Kafka is in the AOT-published sample directly (no conditional hack). Add Docs/PubSubKafka.md (transport config, DI, auth, QoS, AOT note) linked from Docs/README.md and the Docs/PubSub.md transport table. Also implement the two new IPubSubBuilder HA methods (WithActivationCoordinator/WithLeaseStore) in the EthTransportBuilder/UdpTransportBuilder decorators. Validated: NativeAOT publish (win-x64) of the sample succeeds and produces a 26.9 MB native binary with the Kafka/Dekaf transport and ZERO IL2xxx/IL3xxx trim/AOT warnings. --- .../ConsoleReferencePubSubClient.csproj | 1 + .../ConsoleReferencePubSubClient/Program.cs | 109 +++++++++++--- .../PublisherConfigurationBuilder.cs | 26 +++- .../SubscriberConfigurationBuilder.cs | 22 ++- Docs/PubSub.md | 26 +++- Docs/PubSubKafka.md | 142 ++++++++++++++++++ Docs/README.md | 3 +- .../EthTransportBuilder.cs | 13 ++ .../UdpTransportBuilder.cs | 13 ++ 9 files changed, 309 insertions(+), 46 deletions(-) create mode 100644 Docs/PubSubKafka.md diff --git a/Applications/ConsoleReferencePubSubClient/ConsoleReferencePubSubClient.csproj b/Applications/ConsoleReferencePubSubClient/ConsoleReferencePubSubClient.csproj index ef292bb712..761efde022 100644 --- a/Applications/ConsoleReferencePubSubClient/ConsoleReferencePubSubClient.csproj +++ b/Applications/ConsoleReferencePubSubClient/ConsoleReferencePubSubClient.csproj @@ -22,6 +22,7 @@ + diff --git a/Applications/ConsoleReferencePubSubClient/Program.cs b/Applications/ConsoleReferencePubSubClient/Program.cs index 9b0bc51767..5ac39003b0 100644 --- a/Applications/ConsoleReferencePubSubClient/Program.cs +++ b/Applications/ConsoleReferencePubSubClient/Program.cs @@ -49,7 +49,7 @@ namespace Quickstarts.ConsoleReferencePubSubClient /// One executable exposes three command-line-selectable modes: /// /// - /// publisher - publishes sample DataSets over UDP/UADP or MQTT (UADP/JSON). + /// publisher - publishes sample DataSets over UDP/UADP, Ethernet/UADP, MQTT, or Kafka (UADP/JSON). /// /// /// subscriber - receives DataSets and logs each decoded message. @@ -88,14 +88,16 @@ public static async Task Main(string[] args) } /// - /// Builds the publisher subcommand: a UDP/UADP or MQTT publisher + /// Builds the publisher subcommand: a UDP/UADP, Ethernet/UADP, MQTT, or Kafka publisher /// that publishes a built-in sample DataSet. /// private static Command BuildPublisherCommand(Action setExitCode) { var profileOption = new Option("--profile") { - Description = "Transport profile: udp-uadp | mqtt-uadp | mqtt-json.", + Description = + "Transport profile: udp-uadp | eth-uadp | mqtt-uadp | mqtt-json | " + + "kafka-uadp | kafka-json.", DefaultValueFactory = _ => "udp-uadp" }; var configFileOption = new Option("--config-file") @@ -121,7 +123,8 @@ private static Command BuildPublisherCommand(Action setExitCode) { Description = "Transport endpoint URL. Defaults: opc.udp://239.0.0.1:4840 (UDP), " + - "mqtt://localhost:1883 (MQTT)." + "opc.eth://01-00-5E-7F-00-01 (Ethernet), mqtt://localhost:1883 (MQTT), " + + "kafka://localhost:9092 (Kafka)." }; var intervalOption = new Option("--interval") { @@ -131,7 +134,7 @@ private static Command BuildPublisherCommand(Action setExitCode) var command = new Command( "publisher", - "Publish a sample DataSet over UDP/UADP or MQTT (UADP/JSON).") + "Publish a sample DataSet over UDP/UADP, Ethernet/UADP, MQTT, or Kafka (UADP/JSON).") { profileOption, configFileOption, @@ -149,7 +152,7 @@ private static Command BuildPublisherCommand(Action setExitCode) { await Console.Error.WriteLineAsync( $"Unknown --profile value '{profileArg}'. " + - "Expected one of: udp-uadp, mqtt-uadp, mqtt-json.") + "Expected one of: udp-uadp, eth-uadp, mqtt-uadp, mqtt-json, kafka-uadp, kafka-json.") .ConfigureAwait(false); setExitCode(2); return; @@ -169,14 +172,16 @@ await Console.Error.WriteLineAsync( } /// - /// Builds the subscriber subcommand: a UDP/UADP or MQTT subscriber + /// Builds the subscriber subcommand: a UDP/UADP, Ethernet/UADP, MQTT, or Kafka subscriber /// that logs each decoded DataSetMessage. /// private static Command BuildSubscriberCommand(Action setExitCode) { var profileOption = new Option("--profile") { - Description = "Transport profile: udp-uadp | mqtt-uadp | mqtt-json.", + Description = + "Transport profile: udp-uadp | eth-uadp | mqtt-uadp | mqtt-json | " + + "kafka-uadp | kafka-json.", DefaultValueFactory = _ => "udp-uadp" }; var configFileOption = new Option("--config-file") @@ -200,8 +205,10 @@ private static Command BuildSubscriberCommand(Action setExitCode) }; var endpointOption = new Option("--endpoint") { - Description = "Transport endpoint URL. Defaults: opc.udp://239.0.0.1:4840 " + - "(UDP), mqtt://localhost:1883 (MQTT)." + Description = + "Transport endpoint URL. Defaults: opc.udp://239.0.0.1:4840 (UDP), " + + "opc.eth://01-00-5E-7F-00-01 (Ethernet), mqtt://localhost:1883 (MQTT), " + + "kafka://localhost:9092 (Kafka)." }; var command = new Command( @@ -223,7 +230,7 @@ private static Command BuildSubscriberCommand(Action setExitCode) { await Console.Error.WriteLineAsync( $"Unknown --profile value '{profileArg}'. " + - "Expected one of: udp-uadp, mqtt-uadp, mqtt-json.") + "Expected one of: udp-uadp, eth-uadp, mqtt-uadp, mqtt-json, kafka-uadp, kafka-json.") .ConfigureAwait(false); setExitCode(2); return; @@ -373,13 +380,23 @@ private static async Task RunPublisherAsync( .AddUdpTransport() .AddSecurityKeyProvider(SampleSecurity.CreateKeyProvider()) .AddDataSetSource(PublisherConfigurationBuilder.DataSetName, sampleSource); - if (profile == PublisherProfile.EthUadp) + switch (profile) { - publisher.AddEthTransport(); - } - else if (profile != PublisherProfile.UdpUadp) - { - publisher.AddMqttTransport(); + case PublisherProfile.UdpUadp: + break; + case PublisherProfile.EthUadp: + publisher.AddEthTransport(); + break; + case PublisherProfile.MqttUadp: + case PublisherProfile.MqttJson: + publisher.AddMqttTransport(); + break; + case PublisherProfile.KafkaUadp: + case PublisherProfile.KafkaJson: + publisher.AddKafkaTransport(); + break; + default: + throw new ArgumentOutOfRangeException(nameof(profile), profile, null); } publisher.ConfigureApplication(app => { @@ -441,13 +458,23 @@ private static async Task RunSubscriberAsync( sp => new ConsoleLoggingSink( sp.GetRequiredService() .CreateLogger())); - if (profile == SubscriberProfile.EthUadp) - { - subscriber.AddEthTransport(); - } - else if (profile != SubscriberProfile.UdpUadp) + switch (profile) { - subscriber.AddMqttTransport(); + case SubscriberProfile.UdpUadp: + break; + case SubscriberProfile.EthUadp: + subscriber.AddEthTransport(); + break; + case SubscriberProfile.MqttUadp: + case SubscriberProfile.MqttJson: + subscriber.AddMqttTransport(); + break; + case SubscriberProfile.KafkaUadp: + case SubscriberProfile.KafkaJson: + subscriber.AddKafkaTransport(); + break; + default: + throw new ArgumentOutOfRangeException(nameof(profile), profile, null); } subscriber.ConfigureApplication(app => { @@ -719,6 +746,12 @@ private static bool TryParsePublisherProfile(string? text, out PublisherProfile case "eth-uadp": profile = PublisherProfile.EthUadp; return true; + case "kafka-uadp": + profile = PublisherProfile.KafkaUadp; + return true; + case "kafka-json": + profile = PublisherProfile.KafkaJson; + return true; default: profile = PublisherProfile.UdpUadp; return false; @@ -741,6 +774,12 @@ private static bool TryParseSubscriberProfile(string? text, out SubscriberProfil case "eth-uadp": profile = SubscriberProfile.EthUadp; return true; + case "kafka-uadp": + profile = SubscriberProfile.KafkaUadp; + return true; + case "kafka-json": + profile = SubscriberProfile.KafkaJson; + return true; default: profile = SubscriberProfile.UdpUadp; return false; @@ -843,7 +882,17 @@ public enum PublisherProfile /// /// Ethernet (Layer 2) transport with UADP message mapping. /// - EthUadp = 3 + EthUadp = 3, + + /// + /// Kafka broker transport with UADP message mapping. + /// + KafkaUadp = 4, + + /// + /// Kafka broker transport with JSON message mapping. + /// + KafkaJson = 5 } /// @@ -869,7 +918,17 @@ public enum SubscriberProfile /// /// Ethernet (Layer 2) transport with UADP message mapping. /// - EthUadp = 3 + EthUadp = 3, + + /// + /// Kafka broker transport with UADP message mapping. + /// + KafkaUadp = 4, + + /// + /// Kafka broker transport with JSON message mapping. + /// + KafkaJson = 5 } /// diff --git a/Applications/ConsoleReferencePubSubClient/PublisherConfigurationBuilder.cs b/Applications/ConsoleReferencePubSubClient/PublisherConfigurationBuilder.cs index 81b0997e9a..fbf1f0cb4a 100644 --- a/Applications/ConsoleReferencePubSubClient/PublisherConfigurationBuilder.cs +++ b/Applications/ConsoleReferencePubSubClient/PublisherConfigurationBuilder.cs @@ -31,12 +31,13 @@ using Opc.Ua; using Opc.Ua.PubSub.Configuration; using Opc.Ua.PubSub.Eth; +using Opc.Ua.PubSub.Kafka; namespace Quickstarts.ConsoleReferencePubSubClient { /// /// Builds minimal Part 14 - /// payloads for the three demo wire profiles using the fluent + /// payloads for the demo wire profiles using the fluent /// . The payloads use the /// "Simple" DataSet exposed by /// (BoolToggle, Int32 counter, DateTime). @@ -47,7 +48,9 @@ public static class PublisherConfigurationBuilder public const string DefaultUdpEndpoint = "opc.udp://239.0.0.1:4840"; public const string DefaultEthEndpoint = "opc.eth://01-00-5E-7F-00-01"; public const string DefaultMqttEndpoint = "mqtt://localhost:1883"; + public const string DefaultKafkaEndpoint = "kafka://localhost:9092"; private const string MqttQueueName = "Quickstarts/Reference/Simple"; + private const string KafkaTopicName = "Quickstarts.Reference.Simple"; public static string DefaultEndpointFor(PublisherProfile profile) { @@ -55,7 +58,9 @@ public static string DefaultEndpointFor(PublisherProfile profile) { PublisherProfile.UdpUadp => DefaultUdpEndpoint, PublisherProfile.EthUadp => DefaultEthEndpoint, - _ => DefaultMqttEndpoint + PublisherProfile.KafkaUadp or PublisherProfile.KafkaJson => DefaultKafkaEndpoint, + PublisherProfile.MqttUadp or PublisherProfile.MqttJson => DefaultMqttEndpoint, + _ => throw new ArgumentOutOfRangeException(nameof(profile)) }; } @@ -68,13 +73,16 @@ public static PubSubConfigurationDataType Build( int intervalMs) { // UDP and Ethernet are datagram transports (no broker queue); - // the MQTT profiles use broker transport settings instead. + // broker profiles use broker transport settings instead. bool udp = profile is PublisherProfile.UdpUadp or PublisherProfile.EthUadp; // UADP message security (SignAndEncrypt) is wired for the UADP // profiles via the shared StaticSecurityKeyProvider. The JSON // profile has no UADP security wrapper, so it stays unsecured. - bool secured = profile != PublisherProfile.MqttJson; + bool secured = profile is not (PublisherProfile.MqttJson or PublisherProfile.KafkaJson); + string brokerQueueName = profile is PublisherProfile.KafkaUadp or PublisherProfile.KafkaJson + ? KafkaTopicName + : MqttQueueName; string transportProfileUri = profile switch { @@ -82,6 +90,8 @@ public static PubSubConfigurationDataType Build( PublisherProfile.EthUadp => EthProfiles.PubSubEthUadpTransport, PublisherProfile.MqttUadp => Profiles.PubSubMqttUadpTransport, PublisherProfile.MqttJson => Profiles.PubSubMqttJsonTransport, + PublisherProfile.KafkaUadp => KafkaProfiles.PubSubKafkaUadpTransport, + PublisherProfile.KafkaJson => KafkaProfiles.PubSubKafkaJsonTransport, _ => throw new ArgumentOutOfRangeException(nameof(profile)) }; @@ -104,7 +114,7 @@ public static PubSubConfigurationDataType Build( ? new DatagramWriterGroupTransportDataType() : new BrokerWriterGroupTransportDataType { - QueueName = MqttQueueName + QueueName = brokerQueueName }); if (secured) { @@ -126,7 +136,7 @@ public static PubSubConfigurationDataType Build( writer.WithTransportSettings( new BrokerDataSetWriterTransportDataType { - QueueName = MqttQueueName, + QueueName = brokerQueueName, RequestedDeliveryGuarantee = BrokerTransportQualityOfService.BestEffort }); @@ -138,7 +148,7 @@ public static PubSubConfigurationDataType Build( private static IEncodeable WriterGroupMessageSettings(PublisherProfile profile) { - if (profile == PublisherProfile.MqttJson) + if (profile is PublisherProfile.MqttJson or PublisherProfile.KafkaJson) { return new JsonWriterGroupMessageDataType { @@ -163,7 +173,7 @@ private static IEncodeable WriterGroupMessageSettings(PublisherProfile profile) private static IEncodeable WriterMessageSettings(PublisherProfile profile) { - if (profile == PublisherProfile.MqttJson) + if (profile is PublisherProfile.MqttJson or PublisherProfile.KafkaJson) { return new JsonDataSetWriterMessageDataType { diff --git a/Applications/ConsoleReferencePubSubClient/SubscriberConfigurationBuilder.cs b/Applications/ConsoleReferencePubSubClient/SubscriberConfigurationBuilder.cs index fa600d8bdd..6fa2ddf7f8 100644 --- a/Applications/ConsoleReferencePubSubClient/SubscriberConfigurationBuilder.cs +++ b/Applications/ConsoleReferencePubSubClient/SubscriberConfigurationBuilder.cs @@ -31,12 +31,13 @@ using Opc.Ua; using Opc.Ua.PubSub.Configuration; using Opc.Ua.PubSub.Eth; +using Opc.Ua.PubSub.Kafka; namespace Quickstarts.ConsoleReferencePubSubClient { /// /// Builds minimal Part 14 - /// payloads for the three demo wire profiles using the fluent + /// payloads for the demo wire profiles using the fluent /// . Each payload wires one /// PubSubConnection > ReaderGroup > DataSetReader filtered on /// PublisherId / WriterGroupId / DataSetWriterId. @@ -48,7 +49,9 @@ public static class SubscriberConfigurationBuilder public const string DefaultUdpEndpoint = "opc.udp://239.0.0.1:4840"; public const string DefaultEthEndpoint = "opc.eth://01-00-5E-7F-00-01"; public const string DefaultMqttEndpoint = "mqtt://localhost:1883"; + public const string DefaultKafkaEndpoint = "kafka://localhost:9092"; private const string MqttQueueName = "Quickstarts/Reference/Simple"; + private const string KafkaTopicName = "Quickstarts.Reference.Simple"; public static string DefaultEndpointFor(SubscriberProfile profile) { @@ -56,7 +59,9 @@ public static string DefaultEndpointFor(SubscriberProfile profile) { SubscriberProfile.UdpUadp => DefaultUdpEndpoint, SubscriberProfile.EthUadp => DefaultEthEndpoint, - _ => DefaultMqttEndpoint + SubscriberProfile.KafkaUadp or SubscriberProfile.KafkaJson => DefaultKafkaEndpoint, + SubscriberProfile.MqttUadp or SubscriberProfile.MqttJson => DefaultMqttEndpoint, + _ => throw new ArgumentOutOfRangeException(nameof(profile)) }; } @@ -68,13 +73,16 @@ public static PubSubConfigurationDataType Build( ushort dataSetWriterIdFilter) { // UDP and Ethernet are datagram transports (no broker queue); - // the MQTT profiles use broker transport settings instead. + // broker profiles use broker transport settings instead. bool udp = profile is SubscriberProfile.UdpUadp or SubscriberProfile.EthUadp; // UADP message security (SignAndEncrypt) is wired for the UADP // profiles via the shared StaticSecurityKeyProvider. The JSON // profile has no UADP security wrapper, so it stays unsecured. - bool secured = profile != SubscriberProfile.MqttJson; + bool secured = profile is not (SubscriberProfile.MqttJson or SubscriberProfile.KafkaJson); + string brokerQueueName = profile is SubscriberProfile.KafkaUadp or SubscriberProfile.KafkaJson + ? KafkaTopicName + : MqttQueueName; string transportProfileUri = profile switch { @@ -82,6 +90,8 @@ public static PubSubConfigurationDataType Build( SubscriberProfile.EthUadp => EthProfiles.PubSubEthUadpTransport, SubscriberProfile.MqttUadp => Profiles.PubSubMqttUadpTransport, SubscriberProfile.MqttJson => Profiles.PubSubMqttJsonTransport, + SubscriberProfile.KafkaUadp => KafkaProfiles.PubSubKafkaUadpTransport, + SubscriberProfile.KafkaJson => KafkaProfiles.PubSubKafkaJsonTransport, _ => throw new ArgumentOutOfRangeException(nameof(profile)) }; @@ -121,7 +131,7 @@ public static PubSubConfigurationDataType Build( reader.WithTransportSettings( new BrokerDataSetReaderTransportDataType { - QueueName = MqttQueueName, + QueueName = brokerQueueName, RequestedDeliveryGuarantee = BrokerTransportQualityOfService.BestEffort }); @@ -133,7 +143,7 @@ public static PubSubConfigurationDataType Build( private static IEncodeable ReaderMessageSettings(SubscriberProfile profile) { - if (profile == SubscriberProfile.MqttJson) + if (profile is SubscriberProfile.MqttJson or SubscriberProfile.KafkaJson) { return new JsonDataSetReaderMessageDataType { diff --git a/Docs/PubSub.md b/Docs/PubSub.md index de4ef5b223..f9ce27f276 100644 --- a/Docs/PubSub.md +++ b/Docs/PubSub.md @@ -31,15 +31,16 @@ ## At a glance - Targets **OPC UA Part 14 v1.05.06** conformance for the implemented UDP, - MQTT, UADP, JSON, discovery, Action, SKS, and address-space surfaces. -- Five library packages + MQTT, Kafka, UADP, JSON, discovery, Action, SKS, and address-space surfaces. +- Six library packages ([NuGet](https://www.nuget.org/packages?q=OPCFoundation.NetStandard.Opc.Ua.PubSub)): `Opc.Ua.PubSub`, `Opc.Ua.PubSub.Udp`, `Opc.Ua.PubSub.Mqtt`, + `Opc.Ua.PubSub.Kafka`, `Opc.Ua.PubSub.Server`, `Opc.Ua.PubSub.Adapter`. - Multi-TFM: `netstandard2.1`, `net48`, `net472`, `net8.0` (LTS), `net9.0`, `net10.0` (LTS). - Native AOT clean — both reference samples publish with zero `IL2026` / `IL3050` warnings. -- Transports: **UDP** (uni/multi/broadcast), **DTLS over UDP** (`opc.dtls://`, unicast UADP), **MQTT** (3.1.1 + 5.0), and **Ethernet** (`opc.eth://`, Layer 2 UADP with 802.1Q VLAN). +- Transports: **UDP** (uni/multi/broadcast), **DTLS over UDP** (`opc.dtls://`, unicast UADP), **MQTT** (3.1.1 + 5.0), **Kafka** (`kafka://` / `kafkas://`), and **Ethernet** (`opc.eth://`, Layer 2 UADP with 802.1Q VLAN). - Encodings: **UADP** ([§7.2.4](https://reference.opcfoundation.org/specs/OPC-10000-14/v1.05.06/7.2.4)) and **JSON** ([§7.2.5](https://reference.opcfoundation.org/specs/OPC-10000-14/v1.05.06/7.2.5)) with `Verbose` / `Compact` / `RawData` modes. @@ -97,8 +98,8 @@ Actions to an external OPC UA server over a managed client session. │ IPubSubTransportFactory │ │ │ │ │ ┌─────────────────┐ ┌──────────────────────┐ ┌────────────────────┐ -│ Opc.Ua.PubSub. │ │ Opc.Ua.PubSub.Mqtt │ │ third-party plugin │ -│ Udp │ │ MQTTnet 4 / 5 │ │ (custom transport) │ +│ Opc.Ua.PubSub. │ │ Opc.Ua.PubSub.Mqtt │ │ Opc.Ua.PubSub.Kafka│ +│ Udp │ │ MQTTnet 4 / 5 │ │ Dekaf / librdkafka │ └─────────────────┘ └──────────────────────┘ └────────────────────┘ ``` @@ -389,7 +390,7 @@ DI extension methods provided by `Opc.Ua.PubSub`: | `AddPubSubSecurityKeyServiceServer(Action?)` | Registers an in-process SKS with optional initial groups. | Transport-specific extensions -(`Opc.Ua.PubSub.Udp` / `.Mqtt`) supply the matching +(`Opc.Ua.PubSub.Udp` / `.Mqtt` / `.Kafka`) supply the matching `IPubSubTransportFactory` and hang off `IPubSubBuilder` — a transport only makes sense together with the PubSub feature: @@ -397,6 +398,8 @@ only makes sense together with the PubSub feature: unicast / multicast / broadcast. - `IPubSubBuilder.AddMqttTransport(Action?)` — MQTT 3.1.1 + 5.0 via MQTTnet. +- `IPubSubBuilder.AddKafkaTransport(Action?)` — + Apache Kafka broker transport; see [Kafka transport](PubSubKafka.md). - `IPubSubBuilder.AddEthTransport(Action?)` — Ethernet Layer 2 (`opc.eth://`); chain `.WithPcap()` for the SharpPcap (libpcap / Npcap) backend. @@ -410,6 +413,13 @@ Server-side address space — see ## Transports +| Transport | Package | Profiles | Address schemes | +| --------- | ------- | -------- | --------------- | +| UDP / UADP | `Opc.Ua.PubSub.Udp` | `pubsub-udp-uadp` | `opc.udp://`, `opc.dtls://` | +| Ethernet / UADP | `Opc.Ua.PubSub.Eth` | `pubsub-eth-uadp` | `opc.eth://` | +| MQTT / UADP + JSON | `Opc.Ua.PubSub.Mqtt` | `pubsub-mqtt-uadp`, `pubsub-mqtt-json` | `mqtt://`, `mqtts://`, `ws://`, `wss://` | +| Kafka / UADP + JSON | `Opc.Ua.PubSub.Kafka` | `pubsub-kafka-uadp`, `pubsub-kafka-json` | `kafka://`, `kafkas://` | + ### UDP / UADP Implemented in `Opc.Ua.PubSub.Udp`. Wire profile @@ -667,6 +677,10 @@ var options = new MqttConnectionOptions }; ``` +### Apache Kafka + +Implemented in `Opc.Ua.PubSub.Kafka`. Wire profiles [`PubSub Kafka UADP`](http://opcfoundation.org/UA-Profile/Transport/pubsub-kafka-uadp) and [`PubSub Kafka JSON`](http://opcfoundation.org/UA-Profile/Transport/pubsub-kafka-json) implement the Part 14 Annex B.2 Kafka broker mapping over `kafka://` and `kafkas://` endpoints. See [Kafka transport](PubSubKafka.md) for DI registration, topic mapping, SASL/TLS configuration, `content-type` record headers, QoS mapping, and NativeAOT notes. + ### DTLS transport status The `opc.dtls://` transport URI is parsed for Part 14 §7.3.2.4 unicast endpoints and wired through the UDP transport factory when `.WithDtls(...)` is registered on the `IUdpTransportBuilder` returned by `AddUdpTransport()`. The DTLS 1.3 handshake is implemented, including ECDHE negotiation, HelloRetryRequest cookies, and certificate authentication. The key schedule/HKDF, AEAD record protection, and anti-replay window are implemented for the registered runtime profiles. diff --git a/Docs/PubSubKafka.md b/Docs/PubSubKafka.md new file mode 100644 index 0000000000..df565cee2a --- /dev/null +++ b/Docs/PubSubKafka.md @@ -0,0 +1,142 @@ +# Apache Kafka PubSub transport + +`Opc.Ua.PubSub.Kafka` implements the OPC UA Part 14 Annex B.2 Apache Kafka transport mapping for broker-based PubSub deployments. It registers two transport profiles: `pubsub-kafka-uadp` (`KafkaProfiles.PubSubKafkaUadpTransport`) for UADP NetworkMessages and `pubsub-kafka-json` (`KafkaProfiles.PubSubKafkaJsonTransport`) for JSON NetworkMessages. + +Use `kafka://host[:port][,host[:port]...]` for plaintext bootstrap servers and `kafkas://host[:port][,...]` for TLS-protected bootstrap servers. The default Kafka port is `9092`, so `kafka://localhost` and `kafka://localhost:9092` resolve to the same bootstrap server list. + +## Dependency injection + +Register the transport on the PubSub builder with `AddKafkaTransport()`. The extension lives in the `Microsoft.Extensions.DependencyInjection` namespace and registers both Kafka profile factories. Options are supplied with `KafkaConnectionOptions`, either by callback or from the `OpcUa:PubSub:Kafka` configuration section. + +Important `KafkaConnectionOptions` fields include `Endpoint` and `BootstrapServers` (bootstrap broker selection), `GroupId` (subscriber consumer group), `SecurityProtocol`, `SaslMechanism`, `UserName`, `PasswordSecretId`, `AuthenticationProfileUri`, `ResourceUri`, `Tls`, `DeliveryGuarantee`, `AutoOffsetReset`, `EnableAutoCommit`, `MaxMessageSize`, and `Topics.Prefix`. `PasswordSecretId` is resolved through the OPC UA secret store so configuration does not carry plaintext passwords. + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Opc.Ua; +using Opc.Ua.PubSub.Configuration; +using Opc.Ua.PubSub.Kafka; + +const string DataTopic = "opcua.json.data.Line1.Simple"; +const string MetadataTopic = "opcua.json.metadata.Line1.Simple"; + +HostApplicationBuilder publisherHost = Host.CreateApplicationBuilder(args); +publisherHost.Services.AddOpcUa().AddPubSub(pubsub => +{ + pubsub.AddPublisher() + .AddKafkaTransport(options => + { + options.Endpoint = "kafkas://broker1.example.com:9093,broker2.example.com:9093"; + options.BootstrapServers = "broker1.example.com:9093,broker2.example.com:9093"; + options.SecurityProtocol = KafkaSecurityProtocol.SaslSsl; + options.SaslMechanism = KafkaSaslMechanism.ScramSha256; + options.UserName = "pubsub-publisher"; + options.PasswordSecretId = "KafkaPublisherPassword"; + options.DeliveryGuarantee = KafkaQualityOfService.AtLeastOnce; + options.Tls = new KafkaTlsOptions + { + ValidateServerCertificate = true, + CaCertificatePath = "pki/kafka/ca.pem" + }; + }) + .AddDataSetSource("Simple", publishedDataSetSource) + .ConfigureApplication(app => app.UseConfiguration(BuildKafkaPublisherConfiguration())); +}); + +HostApplicationBuilder subscriberHost = Host.CreateApplicationBuilder(args); +subscriberHost.Services.AddOpcUa().AddPubSub(pubsub => +{ + pubsub.AddSubscriber() + .AddKafkaTransport(options => + { + options.Endpoint = "kafkas://broker1.example.com:9093,broker2.example.com:9093"; + options.GroupId = "opcua-reference-subscribers"; + options.SecurityProtocol = KafkaSecurityProtocol.SaslSsl; + options.SaslMechanism = KafkaSaslMechanism.ScramSha256; + options.UserName = "pubsub-subscriber"; + options.PasswordSecretId = "KafkaSubscriberPassword"; + options.AutoOffsetReset = KafkaAutoOffsetReset.Latest; + options.Tls = new KafkaTlsOptions + { + ValidateServerCertificate = true, + CaCertificatePath = "pki/kafka/ca.pem" + }; + }) + .AddSubscribedDataSetSink("Reader 1", sp => subscribedDataSetSink) + .ConfigureApplication(app => app.UseConfiguration(BuildKafkaSubscriberConfiguration())); +}); + +static PubSubConfigurationDataType BuildKafkaPublisherConfiguration() +{ + return PubSubConfigurationBuilder.Create() + .AddPublishedDataSet("Simple", dataSet => dataSet + .AddField("Value", (byte)DataTypes.Int32, DataTypeIds.Int32)) + .AddConnection("Kafka Publisher", connection => connection + .WithPublisherId(new Variant((ushort)1)) + .WithTransportProfile(KafkaProfiles.PubSubKafkaJsonTransport) + .WithAddress("kafkas://broker1.example.com:9093,broker2.example.com:9093") + .AddWriterGroup("WriterGroup 1", group => group + .WithWriterGroupId(100) + .WithTransportSettings(new BrokerWriterGroupTransportDataType + { + QueueName = DataTopic + }) + .WithMessageSettings(new JsonWriterGroupMessageDataType + { + NetworkMessageContentMask = (uint)( + JsonNetworkMessageContentMask.NetworkMessageHeader | + JsonNetworkMessageContentMask.DataSetMessageHeader | + JsonNetworkMessageContentMask.PublisherId) + }) + .AddDataSetWriter("Writer 1", writer => writer + .WithDataSetName("Simple") + .WithDataSetWriterId(1) + .WithTransportSettings(new BrokerDataSetWriterTransportDataType + { + QueueName = DataTopic, + MetaDataQueueName = MetadataTopic, + RequestedDeliveryGuarantee = BrokerTransportQualityOfService.AtLeastOnce + })))) + .Build(); +} + +static PubSubConfigurationDataType BuildKafkaSubscriberConfiguration() +{ + return PubSubConfigurationBuilder.Create() + .AddConnection("Kafka Subscriber", connection => connection + .WithPublisherId(new Variant((ushort)1)) + .WithTransportProfile(KafkaProfiles.PubSubKafkaJsonTransport) + .WithAddress("kafkas://broker1.example.com:9093,broker2.example.com:9093") + .AddReaderGroup("ReaderGroup 1", group => group + .AddDataSetReader("Reader 1", reader => reader + .WithFilter(new Variant((ushort)1), 100, 1) + .WithTransportSettings(new BrokerDataSetReaderTransportDataType + { + QueueName = DataTopic, + MetaDataQueueName = MetadataTopic, + RequestedDeliveryGuarantee = BrokerTransportQualityOfService.AtLeastOnce + }) + .WithMirrorSubscribedDataSet("Reader 1")))) + .Build(); +} +``` + +## Topic mapping + +Kafka uses the OPC UA broker transport settings as topic names. `BrokerDataSetWriterTransportDataType.QueueName` and `BrokerDataSetReaderTransportDataType.QueueName` select the data topic for a specific DataSetWriter/DataSetReader. `BrokerWriterGroupTransportDataType.QueueName` is the writer-group fallback for data messages. `MetaDataQueueName` selects the metadata topic; if it is not set, the transport generates a deterministic fallback topic from `KafkaConnectionOptions.Topics.Prefix`, the encoding (`uadp` or `json`), message type, PublisherId, WriterGroupId, and DataSetWriterId. + +Kafka topic names should use Kafka-safe characters such as letters, digits, `.`, `_`, and `-`. The fallback topic builder uses `.` as the segment separator. + +## Record headers and delivery guarantees + +Every produced Kafka record carries the normative `content-type` record header from Part 14 Annex B.2: `application/opcua+uadp` for `pubsub-kafka-uadp` and `application/json` for `pubsub-kafka-json`. The record key is derived from the PubSubConnection PublisherId when available so records for the same publisher preserve partition ordering. + +`KafkaConnectionOptions.DeliveryGuarantee` maps the Part 14 broker QoS intent to Kafka producer settings: `BestEffort` uses `acks=0`, `AtMostOnce` uses `acks=1`, `AtLeastOnce` uses `acks=all` without idempotence, and `ExactlyOnce` uses `acks=all` with producer idempotence enabled. Per-writer `RequestedDeliveryGuarantee` values on broker transport settings override the connection default when specified. + +## SASL and TLS + +Use `kafkas://` or set `KafkaConnectionOptions.Tls.UseTls` for TLS. `KafkaTlsOptions` carries CA, client certificate, and client key PEM paths. Use `SecurityProtocol = KafkaSecurityProtocol.SaslSsl` with `SaslMechanism`, `UserName`, and `PasswordSecretId` for SASL over TLS. Sending SASL credentials over plaintext `kafka://` is rejected unless `AllowCredentialsOverPlaintext` is explicitly enabled for local development or a controlled network. + +## NativeAOT note + +On `net10.0`, `Opc.Ua.PubSub.Kafka` uses the pure-managed Dekaf Kafka client and is intended to be NativeAOT-compatible. On `net472`, `net48`, `netstandard2.1`, `net8.0`, and `net9.0`, the library uses Confluent.Kafka, which wraps native librdkafka and is not NativeAOT-compatible. diff --git a/Docs/README.md b/Docs/README.md index 54d61fbd32..29f9bc7689 100644 --- a/Docs/README.md +++ b/Docs/README.md @@ -38,9 +38,10 @@ Here is a list of available documentation for different topics: * [AuthorizationService](AuthorizationService.md) - Modern Part 12 `StartRequestToken` / `FinishRequestToken`, `ITokenIssuer`, and GDS token issuance. * [Fuzz testing](../Fuzzing/Fuzzing.md) - SharpFuzz + afl-fuzz + libFuzzer integration. Three areas: `Encoders` (Binary/JSON/XML decoders, built-in type readers, parser entry points), `Certificates` (`X509CRL`, X509 extension parsers, `PEMReader`, `Pkcs10CertificationRequest`, ASN.1 helpers), and `Network` (UA-SC framing via `Opc.Ua.Core.Diagnostics` + internal `TcpMessageParsers` seam on `Opc.Ua.Core`). The [`fuzz-tester`](../.github/agents/fuzz-tester.agent.md) custom agent drives the whole toolchain autonomously: it detects OS-available engines, runs them in parallel, fixes novel findings per repo guidelines, adds the failing input as a regression asset, and pushes one commit per fix until the user says stop. * [KeyCredentialService](KeyCredentialService.md) - Pull, Push, and experimental bridge guidance for Part 12 KeyCredential flows. -* [PubSub (Part 14)](PubSub.md) - Publisher/subscriber support library: architecture, fluent builder, transports (UDP / MQTT 3.1.1 + 5.0 / Ethernet Layer 2), encodings (UADP / JSON), security, and server-side address space. +* [PubSub (Part 14)](PubSub.md) - Publisher/subscriber support library: architecture, fluent builder, transports (UDP / MQTT 3.1.1 + 5.0 / Kafka / Ethernet Layer 2), encodings (UADP / JSON), security, and server-side address space. * [Migration sub-doc](migrate/2.0.x/pubsub.md) - 1.5.378 → 2.0 breaking API, transport, JSON, and field-encoding changes, plus the compatibility matrix. * [Ethernet transport](PubSub.md#transports) - Layer 2 PubSub (`opc.eth://`, EtherType `0xB62C`, 802.1Q VLAN) with native AF_PACKET / BPF, SharpPcap, and in-memory backends. + * [Kafka transport](PubSubKafka.md) - Apache Kafka broker transport (`kafka://`, `kafkas://`) for UADP and JSON PubSub profiles with SASL/TLS and NativeAOT support on `net10.0`. * [External server adapter](PubSub.md#binding-pubsub-to-an-external-opc-ua-server-client-session-adapters) - Bind PubSub publishers, subscribers, and Action responders to an external OPC UA server through `ManagedSession`. * [Dependency Injection extensions](DependencyInjection.md) - `AddPubSub`, `AddPubSubPublisher`, `AddPubSubSubscriber`, `AddPubSubSecurityKeyServiceClient/Server`, `AddPubSubAddressSpace`. * [Profiles](Profiles.md#pubsub-transports) - Datagram-v2, SKS pull / push, AES-128/256-CTR security facets. diff --git a/Libraries/Opc.Ua.PubSub.Eth/DependencyInjection/EthTransportBuilder.cs b/Libraries/Opc.Ua.PubSub.Eth/DependencyInjection/EthTransportBuilder.cs index 46c55f1af3..100110c181 100644 --- a/Libraries/Opc.Ua.PubSub.Eth/DependencyInjection/EthTransportBuilder.cs +++ b/Libraries/Opc.Ua.PubSub.Eth/DependencyInjection/EthTransportBuilder.cs @@ -116,6 +116,19 @@ public IPubSubBuilder WithSecurityKeyStore(IPubSubSecurityKeyStore store) return PubSubBuilder.WithSecurityKeyStore(store); } + /// + public IPubSubBuilder WithActivationCoordinator( + Opc.Ua.PubSub.Redundancy.IPubSubActivationCoordinator coordinator) + { + return PubSubBuilder.WithActivationCoordinator(coordinator); + } + + /// + public IPubSubBuilder WithLeaseStore(Opc.Ua.PubSub.Redundancy.IPubSubLeaseStore leaseStore) + { + return PubSubBuilder.WithLeaseStore(leaseStore); + } + /// public IPubSubBuilder AddActionResponder( PubSubActionTarget target, diff --git a/Libraries/Opc.Ua.PubSub.Udp/DependencyInjection/UdpTransportBuilder.cs b/Libraries/Opc.Ua.PubSub.Udp/DependencyInjection/UdpTransportBuilder.cs index e24b856b9a..41f34be0d7 100644 --- a/Libraries/Opc.Ua.PubSub.Udp/DependencyInjection/UdpTransportBuilder.cs +++ b/Libraries/Opc.Ua.PubSub.Udp/DependencyInjection/UdpTransportBuilder.cs @@ -116,6 +116,19 @@ public IPubSubBuilder WithSecurityKeyStore(IPubSubSecurityKeyStore store) return PubSubBuilder.WithSecurityKeyStore(store); } + /// + public IPubSubBuilder WithActivationCoordinator( + Opc.Ua.PubSub.Redundancy.IPubSubActivationCoordinator coordinator) + { + return PubSubBuilder.WithActivationCoordinator(coordinator); + } + + /// + public IPubSubBuilder WithLeaseStore(Opc.Ua.PubSub.Redundancy.IPubSubLeaseStore leaseStore) + { + return PubSubBuilder.WithLeaseStore(leaseStore); + } + /// public IPubSubBuilder AddActionResponder( PubSubActionTarget target, From 2e057fb33b32eeaf7a5ac7f7fb272bf46979ff39 Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Fri, 3 Jul 2026 13:52:38 +0200 Subject: [PATCH 07/21] Add Opc.Ua.PubSub.Kafka transport tests + Testcontainers Docker test (#3942) New Opc.Ua.PubSub.Kafka.Tests: a FakeKafkaClientAdapter over a shared in-memory topic bus (publisher/subscriber fakes exchange preserved KafkaMessage records) plus endpoint-parser, options, QoS-mapping, encoding/content-type, factory, lifecycle, edge, produce->consume round-trip, DI, and guard tests linked to Part 14 Annex B.2. A Testcontainers.Kafka integration test (net8.0+) runs a real produce->consume round-trip against a Kafka container and Assert.Ignore-skips when Docker is unavailable. 82 non-integration tests pass on net10 (1 Docker test skipped without Docker); Kafka source line coverage 80.28% (meets the 80% bar). Builds clean on net10 and net48. --- .../FakeKafkaClientAdapter.cs | 349 ++++++++++++++++++ .../KafkaBrokerTransportEdgeTests.cs | 214 +++++++++++ .../KafkaBrokerTransportLifecycleTests.cs | 196 ++++++++++ .../KafkaBrokerTransportRoundTripTests.cs | 127 +++++++ .../KafkaClientAdapterGuardTests.cs | 346 +++++++++++++++++ .../KafkaConnectionOptionsTests.cs | 158 ++++++++ .../KafkaEncodingTests.cs | 73 ++++ .../KafkaEndpointParserTests.cs | 118 ++++++ .../KafkaIntegrationDockerTests.cs | 136 +++++++ .../KafkaPubSubTransportFactoryTests.cs | 320 ++++++++++++++++ .../KafkaQosMappingTests.cs | 84 +++++ .../KafkaTestHelper.cs | 189 ++++++++++ ...ansportServiceCollectionExtensionsTests.cs | 180 +++++++++ .../Opc.Ua.PubSub.Kafka.Tests.csproj | 42 +++ 14 files changed, 2532 insertions(+) create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/FakeKafkaClientAdapter.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaBrokerTransportEdgeTests.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaBrokerTransportLifecycleTests.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaBrokerTransportRoundTripTests.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaClientAdapterGuardTests.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaConnectionOptionsTests.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaEncodingTests.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaEndpointParserTests.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaPubSubTransportFactoryTests.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaQosMappingTests.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaTestHelper.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaTransportServiceCollectionExtensionsTests.cs create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/Opc.Ua.PubSub.Kafka.Tests.csproj diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/FakeKafkaClientAdapter.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/FakeKafkaClientAdapter.cs new file mode 100644 index 0000000000..ae0c6058c7 --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/FakeKafkaClientAdapter.cs @@ -0,0 +1,349 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.PubSub.Kafka.Internal; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// In-memory implementation of used by the Kafka tests. + /// + internal sealed class FakeKafkaClientAdapter : IKafkaClientAdapter + { + private readonly FakeKafkaBus m_bus; + private readonly TimeProvider m_timeProvider; + private readonly ConcurrentQueue m_produced = new(); + private readonly System.Threading.Lock m_sync = new(); + private readonly List m_subscriptions = new(); + private readonly List m_unsubscriptions = new(); + private bool m_isConnected; + private bool m_disposed; + + public FakeKafkaClientAdapter(FakeKafkaBus? bus = null, TimeProvider? timeProvider = null) + { + m_bus = bus ?? FakeKafkaBus.Shared; + m_timeProvider = timeProvider ?? TimeProvider.System; + } + + public IReadOnlyCollection ProducedMessages => m_produced; + + public IReadOnlyList Subscriptions + { + get + { + lock (m_sync) + { + return m_subscriptions.ToArray(); + } + } + } + + public IReadOnlyList Unsubscriptions + { + get + { + lock (m_sync) + { + return m_unsubscriptions.ToArray(); + } + } + } + + public int ConnectCount { get; private set; } + + public int DisconnectCount { get; private set; } + + public Func? OnConnect { get; set; } + + public Func? OnDisconnect { get; set; } + + public Func? OnProduce { get; set; } + + public bool IsConnected + { + get + { + lock (m_sync) + { + return m_isConnected; + } + } + } + + public bool Disposed => m_disposed; + + public event EventHandler? IncomingMessage; + + public event EventHandler? ConnectionStateChanged; + + public async ValueTask ConnectAsync(KafkaConnectionOptions options, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ConnectCount++; + if (OnConnect is not null) + { + await OnConnect(options, cancellationToken).ConfigureAwait(false); + } + lock (m_sync) + { + m_isConnected = true; + } + ConnectionStateChanged?.Invoke( + this, + new KafkaConnectionStateChangedEventArgs(isConnected: true, reason: "Connected")); + } + + public async ValueTask DisconnectAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + DisconnectCount++; + if (OnDisconnect is not null) + { + await OnDisconnect(cancellationToken).ConfigureAwait(false); + } + bool wasConnected; + lock (m_sync) + { + wasConnected = m_isConnected; + m_isConnected = false; + } + m_bus.Unsubscribe(this, Subscriptions); + if (wasConnected) + { + ConnectionStateChanged?.Invoke( + this, + new KafkaConnectionStateChangedEventArgs(isConnected: false, reason: "Disconnected")); + } + } + + public ValueTask SubscribeAsync(IReadOnlyList topics, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (m_sync) + { + foreach (string topic in topics) + { + if (!m_subscriptions.Contains(topic)) + { + m_subscriptions.Add(topic); + } + } + } + m_bus.Subscribe(this, topics); + return default; + } + + public ValueTask UnsubscribeAsync(IReadOnlyList topics, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (m_sync) + { + foreach (string topic in topics) + { + m_unsubscriptions.Add(topic); + m_subscriptions.Remove(topic); + } + } + m_bus.Unsubscribe(this, topics); + return default; + } + + public async ValueTask ProduceAsync(KafkaMessage message, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + KafkaMessage copy = Copy(message); + m_produced.Enqueue(copy); + if (OnProduce is not null) + { + await OnProduce(copy, cancellationToken).ConfigureAwait(false); + } + m_bus.Publish(copy, cancellationToken); + } + + public void RaiseIncomingMessage(KafkaMessage message, DateTimeUtc receivedAt) + { + IncomingMessage?.Invoke(this, new KafkaIncomingMessageEventArgs(Copy(message), receivedAt)); + } + + public void RaiseConnectionStateChanged(bool isConnected, string? reason = null) + { + lock (m_sync) + { + m_isConnected = isConnected; + } + ConnectionStateChanged?.Invoke( + this, + new KafkaConnectionStateChangedEventArgs(isConnected, reason)); + } + + public ValueTask DisposeAsync() + { + m_disposed = true; + m_bus.Unsubscribe(this, Subscriptions); + return default; + } + + internal void Deliver(KafkaMessage message) + { + RaiseIncomingMessage( + message, + DateTimeUtc.From(m_timeProvider.GetUtcNow().UtcDateTime)); + } + + private static KafkaMessage Copy(KafkaMessage message) + { + IReadOnlyDictionary? headers = message.Headers is null + ? null + : message.Headers.ToDictionary( + static header => header.Key, + static header => header.Value, + StringComparer.Ordinal); + return new KafkaMessage( + message.Topic, + message.Key.ToArray(), + message.Value.ToArray(), + message.ContentType, + headers); + } + } + + /// + /// Shared in-memory Kafka topic bus used by fake adapters. + /// + internal sealed class FakeKafkaBus + { + private readonly System.Threading.Lock m_sync = new(); + private readonly Dictionary> m_subscribers = new(StringComparer.Ordinal); + + public static FakeKafkaBus Shared { get; } = new FakeKafkaBus(); + + public void Subscribe(FakeKafkaClientAdapter adapter, IReadOnlyList topics) + { + lock (m_sync) + { + foreach (string topic in topics) + { + if (!m_subscribers.TryGetValue(topic, out List? subscribers)) + { + subscribers = new List(); + m_subscribers[topic] = subscribers; + } + if (!subscribers.Contains(adapter)) + { + subscribers.Add(adapter); + } + } + } + } + + public void Unsubscribe(FakeKafkaClientAdapter adapter, IReadOnlyList topics) + { + lock (m_sync) + { + foreach (string topic in topics) + { + if (m_subscribers.TryGetValue(topic, out List? subscribers)) + { + subscribers.Remove(adapter); + if (subscribers.Count == 0) + { + m_subscribers.Remove(topic); + } + } + } + } + } + + public void Publish(KafkaMessage message, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + FakeKafkaClientAdapter[] subscribers; + lock (m_sync) + { + subscribers = m_subscribers.TryGetValue(message.Topic, out List? targets) + ? targets.ToArray() + : Array.Empty(); + } + foreach (FakeKafkaClientAdapter subscriber in subscribers) + { + subscriber.Deliver(message); + } + } + } + + /// + /// Fake Kafka client factory that hands adapters to transports under test. + /// + internal sealed class FakeKafkaClientFactory : IKafkaClientFactory + { + private readonly Queue m_adapters = new(); + private readonly FakeKafkaBus m_bus; + + public FakeKafkaClientFactory(FakeKafkaBus? bus = null) + { + m_bus = bus ?? new FakeKafkaBus(); + Adapter = new FakeKafkaClientAdapter(m_bus); + m_adapters.Enqueue(Adapter); + } + + public FakeKafkaClientFactory(params FakeKafkaClientAdapter[] adapters) + { + if (adapters.Length == 0) + { + throw new ArgumentException("At least one adapter is required.", nameof(adapters)); + } + m_bus = new FakeKafkaBus(); + Adapter = adapters[0]; + foreach (FakeKafkaClientAdapter adapter in adapters) + { + m_adapters.Enqueue(adapter); + } + } + + public FakeKafkaClientAdapter Adapter { get; } + + public int CreateCount { get; private set; } + + IKafkaClientAdapter IKafkaClientFactory.Create(ITelemetryContext telemetry, TimeProvider timeProvider) + { + CreateCount++; + if (m_adapters.Count > 0) + { + return m_adapters.Dequeue(); + } + return new FakeKafkaClientAdapter(m_bus, timeProvider); + } + } +} diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaBrokerTransportEdgeTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaBrokerTransportEdgeTests.cs new file mode 100644 index 0000000000..33ba7abad7 --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaBrokerTransportEdgeTests.cs @@ -0,0 +1,214 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.PubSub.Encoding; +using Opc.Ua.PubSub.Tests; +using Opc.Ua.PubSub.Transports; +using Opc.Ua.Tests; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// Edge-case coverage for Kafka transport guard rails and topic routing. + /// + [TestFixture] + [Category("Unit")] + [TestSpec("B.2", Summary = "Kafka broker transport edge cases")] + [CancelAfter(10000)] + public sealed class KafkaBrokerTransportEdgeTests + { + [Test] + public void ConstructorRejectsInvalidArguments() + { + KafkaEndpoint endpoint = KafkaEndpointParser.Parse(KafkaTestHelper.EndpointUrl); + PubSubConnectionDataType connection = KafkaTestHelper.NewConnection(); + var options = new KafkaConnectionOptions { Endpoint = KafkaTestHelper.EndpointUrl }; + + Assert.That(() => new KafkaBrokerTransport( + null!, endpoint, PubSubTransportDirection.Send, options, new FakeKafkaClientFactory(), + NUnitTelemetryContext.Create(), TimeProvider.System), Throws.TypeOf()); + Assert.That(() => new KafkaBrokerTransport( + connection, endpoint, PubSubTransportDirection.Send, null!, new FakeKafkaClientFactory(), + NUnitTelemetryContext.Create(), TimeProvider.System), Throws.TypeOf()); + Assert.That(() => new KafkaBrokerTransport( + connection, endpoint, PubSubTransportDirection.Send, options, null!, + NUnitTelemetryContext.Create(), TimeProvider.System), Throws.TypeOf()); + Assert.That(() => new KafkaBrokerTransport( + connection, endpoint, PubSubTransportDirection.Send, options, new FakeKafkaClientFactory(), + null!, TimeProvider.System), Throws.TypeOf()); + Assert.That(() => new KafkaBrokerTransport( + connection, endpoint, PubSubTransportDirection.Send, options, new FakeKafkaClientFactory(), + NUnitTelemetryContext.Create(), null!), Throws.TypeOf()); + } + + [Test] + public void SendBeforeOpenThrowsInvalidOperationException() + { + var factory = new FakeKafkaClientFactory(); + KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + + Assert.That( + async () => await transport.SendAsync(new byte[] { 0x01 }, KafkaTestHelper.JsonTopic), + Throws.TypeOf()); + } + + [Test] + public async Task SendWithEmptyOrInvalidTopicThrowsAsync() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + + Assert.That(async () => await transport.SendAsync(new byte[] { 0x01 }, string.Empty), + Throws.TypeOf()); + Assert.That(async () => await transport.SendAsync(new byte[] { 0x01 }, "bad\0topic"), + Throws.TypeOf()); + } + + [Test] + public async Task SendCancellationAndDisposeAreObservedAsync() + { + var factory = new FakeKafkaClientFactory(); + KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + Assert.That(async () => await transport.SendAsync( + new byte[] { 0x01 }, + KafkaTestHelper.JsonTopic, + cts.Token), + Throws.InstanceOf()); + + await transport.DisposeAsync().ConfigureAwait(false); + Assert.That(async () => await transport.SendAsync(new byte[] { 0x01 }, KafkaTestHelper.JsonTopic), + Throws.TypeOf()); + Assert.That(async () => await transport.OpenAsync(CancellationToken.None), + Throws.TypeOf()); + } + + [Test] + public async Task DisposeDuringSendDisconnectsCleanlyAsync() + { + var gate = new TaskCompletionSource(); + var factory = new FakeKafkaClientFactory(); + factory.Adapter.OnProduce = async (_, _) => await gate.Task.ConfigureAwait(false); + KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + + Task sendTask = transport.SendAsync(new byte[] { 0x01 }, KafkaTestHelper.JsonTopic).AsTask(); + Task disposeTask = transport.DisposeAsync().AsTask(); + gate.SetResult(true); + await Task.WhenAll(sendTask, disposeTask).ConfigureAwait(false); + + Assert.That(factory.Adapter.DisconnectCount, Is.EqualTo(1)); + } + + [Test] + public async Task ReceiveWithoutChannelYieldsNoFramesAsync() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport( + factory, + PubSubTransportDirection.Send); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(150)); + int frames = 0; + + await foreach (PubSubTransportFrame _ in transport.ReceiveAsync(cts.Token).ConfigureAwait(false)) + { + frames++; + } + + Assert.That(frames, Is.Zero); + } + + [Test] + public async Task UnsubscribeStopsFakeBusDeliveryAsync() + { + var bus = new FakeKafkaBus(); + var subscriberAdapter = new FakeKafkaClientAdapter(bus); + var publisherAdapter = new FakeKafkaClientAdapter(bus); + var subscriberFactory = new FakeKafkaClientFactory(subscriberAdapter); + var publisherFactory = new FakeKafkaClientFactory(publisherAdapter); + await using KafkaBrokerTransport subscriber = KafkaTestHelper.NewTransport( + subscriberFactory, + PubSubTransportDirection.Receive, + connection: KafkaTestHelper.NewConnection(writer: false, reader: true)); + await using KafkaBrokerTransport publisher = KafkaTestHelper.NewTransport(publisherFactory); + subscriber.Subscriptions.Add(KafkaTestHelper.JsonTopic); + await subscriber.OpenAsync(CancellationToken.None).ConfigureAwait(false); + await publisher.OpenAsync(CancellationToken.None).ConfigureAwait(false); + await subscriberAdapter.UnsubscribeAsync([KafkaTestHelper.JsonTopic], CancellationToken.None) + .ConfigureAwait(false); + + await publisher.SendAsync(new byte[] { 0x01 }, KafkaTestHelper.JsonTopic).ConfigureAwait(false); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(150)); + PubSubTransportFrame? frame = await KafkaTestHelper.ReceiveOneAsync(subscriber, cts.Token) + .ConfigureAwait(false); + + Assert.That(frame, Is.Null); + Assert.That(subscriberAdapter.Unsubscriptions, Does.Contain(KafkaTestHelper.JsonTopic)); + } + + [Test] + public async Task ProducedRecordCarriesContentTypeHeaderAndPartitionKeyAsync() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + + await transport.SendAsync(new byte[] { 0x01 }, KafkaTestHelper.JsonTopic).ConfigureAwait(false); + + KafkaMessage message = KafkaTestHelper.FirstProduced(factory.Adapter); + Assert.That(message.ContentType, Is.EqualTo("application/json")); + Assert.That(message.Key.ToArray(), Is.EqualTo(System.Text.Encoding.UTF8.GetBytes("42"))); + } + + [Test] + public void TopicProviderUsesExplicitAndFallbackTopics() + { + PubSubConnectionDataType connection = KafkaTestHelper.NewConnection( + dataTopic: "custom.data", + metadataTopic: "custom.metadata"); + var factory = new FakeKafkaClientFactory(); + KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory, connection: connection); + WriterGroupDataType writerGroup = connection.WriterGroups[0]; + PublisherId publisherId = PublisherId.From(new Variant((uint)42)); + + Assert.That(transport.BuildDataTopic(publisherId, writerGroup, 3), Is.EqualTo("custom.data")); + Assert.That(transport.BuildMetaDataTopic(publisherId, 1, 3), Is.EqualTo("custom.metadata")); + Assert.That(transport.BuildDiscoveryTopic(publisherId, "status"), Is.EqualTo("opcua.json.status.42")); + } + } +} diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaBrokerTransportLifecycleTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaBrokerTransportLifecycleTests.cs new file mode 100644 index 0000000000..b029c36201 --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaBrokerTransportLifecycleTests.cs @@ -0,0 +1,196 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.PubSub.Diagnostics; +using Opc.Ua.PubSub.Tests; +using Opc.Ua.PubSub.Transports; +using Opc.Ua.Tests; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// Lifecycle and state-event tests for the Kafka broker transport. + /// + [TestFixture] + [Category("Unit")] + [TestSpec("B.2", Summary = "Kafka transport lifecycle")] + [CancelAfter(10000)] + public sealed class KafkaBrokerTransportLifecycleTests + { + [Test] + public async Task OpenCloseCycleSucceedsAsync() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + + Assert.That(transport.IsConnected, Is.False); + + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + Assert.That(transport.IsConnected, Is.True); + Assert.That(factory.Adapter.ConnectCount, Is.EqualTo(1)); + + await transport.CloseAsync(CancellationToken.None).ConfigureAwait(false); + Assert.That(transport.IsConnected, Is.False); + Assert.That(factory.Adapter.DisconnectCount, Is.EqualTo(1)); + } + + [Test] + public async Task OpenOnAlreadyOpenedTransportIsIdempotentAsync() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + + Assert.That(factory.Adapter.ConnectCount, Is.EqualTo(1)); + } + + [Test] + public async Task CloseOnUnopenedTransportDoesNotThrowAsync() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + + await transport.CloseAsync(CancellationToken.None).ConfigureAwait(false); + + Assert.That(transport.IsConnected, Is.False); + Assert.That(factory.Adapter.DisconnectCount, Is.Zero); + } + + [Test] + public async Task DoubleCloseIsIdempotentAsync() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + await transport.CloseAsync(CancellationToken.None).ConfigureAwait(false); + await transport.CloseAsync(CancellationToken.None).ConfigureAwait(false); + + Assert.That(factory.Adapter.DisconnectCount, Is.EqualTo(1)); + } + + [Test] + public async Task DisposeAsyncIsIdempotentAsync() + { + var factory = new FakeKafkaClientFactory(); + KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + + await transport.DisposeAsync().ConfigureAwait(false); + await transport.DisposeAsync().ConfigureAwait(false); + + Assert.That(factory.Adapter.DisconnectCount, Is.EqualTo(1)); + } + + [Test] + public async Task StateChangedFiresOnOpenCloseAndAdapterEventsAsync() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + var events = new List(); + transport.StateChanged += (_, e) => events.Add(e); + + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + factory.Adapter.RaiseConnectionStateChanged(false, "broker reset"); + factory.Adapter.RaiseConnectionStateChanged(true, "reconnected"); + await transport.CloseAsync(CancellationToken.None).ConfigureAwait(false); + + Assert.That(events, Has.Count.GreaterThanOrEqualTo(4)); + Assert.That(events[0].IsConnected, Is.True); + Assert.That(events, Has.Some.Matches(e => + !e.IsConnected && e.Reason == "broker reset")); + Assert.That(events, Has.Some.Matches(e => + e.IsConnected && e.Reason == "reconnected")); + Assert.That(events[^1].IsConnected, Is.False); + } + + [Test] + public async Task SendAsyncRequiresTopicAsync() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + + Assert.That( + async () => await transport.SendAsync(new byte[] { 1, 2, 3 }, topic: null), + Throws.TypeOf()); + } + + [Test] + public async Task SendAsyncProducesMessageAndIncrementsDiagnosticsAsync() + { + var diagnostics = new PubSubDiagnostics(PubSubDiagnosticsLevel.Low); + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport( + factory, + diagnostics: diagnostics); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + + byte[] payload = [9, 8, 7, 6]; + await transport.SendAsync(payload, KafkaTestHelper.JsonTopic).ConfigureAwait(false); + + KafkaMessage message = KafkaTestHelper.FirstProduced(factory.Adapter); + Assert.That(message.Topic, Is.EqualTo(KafkaTestHelper.JsonTopic)); + Assert.That(message.Value.ToArray(), Is.EqualTo(payload)); + Assert.That(message.ContentType, Is.EqualTo("application/json")); + Assert.That(diagnostics.Read(PubSubDiagnosticsCounterKind.SentNetworkMessages), Is.EqualTo(1)); + } + + [Test] + public async Task ReceiveAsyncDeliversIncomingMessagesAsync() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport( + factory, + PubSubTransportDirection.Receive); + transport.Subscriptions.Add(KafkaTestHelper.JsonTopic); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + + byte[] payload = [1, 2, 3]; + factory.Adapter.RaiseIncomingMessage( + new KafkaMessage(KafkaTestHelper.JsonTopic, ReadOnlyMemory.Empty, payload, "application/json", null), + DateTimeUtc.From(DateTime.UtcNow)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + + PubSubTransportFrame? frame = await KafkaTestHelper.ReceiveOneAsync(transport, cts.Token) + .ConfigureAwait(false); + + Assert.That(frame, Is.Not.Null); + Assert.That(frame!.Value.Topic, Is.EqualTo(KafkaTestHelper.JsonTopic)); + Assert.That(frame.Value.Payload.ToArray(), Is.EqualTo(payload)); + } + } +} diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaBrokerTransportRoundTripTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaBrokerTransportRoundTripTests.cs new file mode 100644 index 0000000000..cde177f711 --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaBrokerTransportRoundTripTests.cs @@ -0,0 +1,127 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.PubSub.Tests; +using Opc.Ua.PubSub.Transports; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// Verifies deterministic in-memory Kafka producer to subscriber round trips. + /// + [TestFixture] + [Category("Unit")] + [TestSpec("B.2", Summary = "Kafka transport in-memory round trips")] + [CancelAfter(10000)] + public sealed class KafkaBrokerTransportRoundTripTests + { + [Test] + [TestCase(KafkaProfiles.PubSubKafkaJsonTransport, KafkaTestHelper.JsonTopic, "application/json")] + [TestCase(KafkaProfiles.PubSubKafkaUadpTransport, KafkaTestHelper.UadpTopic, "application/opcua+uadp")] + public async Task PublisherAndSubscriberRoundTripPayloadAsync( + string profile, + string topic, + string contentType) + { + var bus = new FakeKafkaBus(); + var publisherAdapter = new FakeKafkaClientAdapter(bus); + var subscriberAdapter = new FakeKafkaClientAdapter(bus); + var publisherFactory = new FakeKafkaClientFactory(publisherAdapter); + var subscriberFactory = new FakeKafkaClientFactory(subscriberAdapter); + PubSubConnectionDataType pubConnection = KafkaTestHelper.NewConnection(profile: profile); + PubSubConnectionDataType subConnection = KafkaTestHelper.NewConnection( + writer: false, + reader: true, + profile: profile, + dataTopic: topic, + metadataTopic: KafkaTestHelper.MetadataTopic); + await using KafkaBrokerTransport publisher = KafkaTestHelper.NewTransport( + publisherFactory, + PubSubTransportDirection.Send, + connection: pubConnection); + await using KafkaBrokerTransport subscriber = KafkaTestHelper.NewTransport( + subscriberFactory, + PubSubTransportDirection.Receive, + connection: subConnection); + subscriber.Subscriptions.Add(topic); + + await subscriber.OpenAsync(CancellationToken.None).ConfigureAwait(false); + await publisher.OpenAsync(CancellationToken.None).ConfigureAwait(false); + byte[] payload = [0xCA, 0xFE, 0xBA, 0xBE]; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + Task receiveTask = KafkaTestHelper.ReceiveOneAsync(subscriber, cts.Token); + + await publisher.SendAsync(payload, topic).ConfigureAwait(false); + PubSubTransportFrame? frame = await receiveTask.ConfigureAwait(false); + + Assert.That(frame, Is.Not.Null); + Assert.That(frame!.Value.Topic, Is.EqualTo(topic)); + Assert.That(frame.Value.Payload.ToArray(), Is.EqualTo(payload)); + KafkaMessage produced = KafkaTestHelper.FirstProduced(publisherAdapter); + Assert.That(produced.ContentType, Is.EqualTo(contentType)); + } + + [Test] + public async Task SubscriberReceivesDefaultReaderGroupSubscriptionsAsync() + { + var bus = new FakeKafkaBus(); + var publisherAdapter = new FakeKafkaClientAdapter(bus); + var subscriberAdapter = new FakeKafkaClientAdapter(bus); + await using KafkaBrokerTransport publisher = KafkaTestHelper.NewTransport( + new FakeKafkaClientFactory(publisherAdapter)); + await using KafkaBrokerTransport subscriber = KafkaTestHelper.NewTransport( + new FakeKafkaClientFactory(subscriberAdapter), + PubSubTransportDirection.Receive, + connection: KafkaTestHelper.NewConnection( + writer: false, + reader: true, + dataTopic: KafkaTestHelper.JsonTopic, + metadataTopic: KafkaTestHelper.MetadataTopic)); + + await subscriber.OpenAsync(CancellationToken.None).ConfigureAwait(false); + await publisher.OpenAsync(CancellationToken.None).ConfigureAwait(false); + Assert.That(subscriberAdapter.Subscriptions, Does.Contain(KafkaTestHelper.JsonTopic)); + Assert.That(subscriberAdapter.Subscriptions, Does.Contain(KafkaTestHelper.MetadataTopic)); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + Task receiveTask = KafkaTestHelper.ReceiveOneAsync(subscriber, cts.Token); + await publisher.SendAsync(new byte[] { 0x11, 0x22 }, KafkaTestHelper.MetadataTopic) + .ConfigureAwait(false); + PubSubTransportFrame? frame = await receiveTask.ConfigureAwait(false); + + Assert.That(frame, Is.Not.Null); + Assert.That(frame!.Value.Topic, Is.EqualTo(KafkaTestHelper.MetadataTopic)); + Assert.That(frame.Value.Payload.ToArray(), Is.EqualTo(new byte[] { 0x11, 0x22 })); + } + } +} diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaClientAdapterGuardTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaClientAdapterGuardTests.cs new file mode 100644 index 0000000000..08873cce4f --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaClientAdapterGuardTests.cs @@ -0,0 +1,346 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#if NET10_0_OR_GREATER +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.PubSub.Kafka.Internal; +using Opc.Ua.PubSub.Tests; +using Opc.Ua.Tests; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// Guard tests for the default Dekaf-backed Kafka adapter that do not require a broker. + /// + [TestFixture] + [Category("Unit")] + [TestSpec("B.2", Summary = "Kafka default adapter guard behavior")] + [CancelAfter(10000)] + public sealed class KafkaClientAdapterGuardTests + { + [Test] + public void ConstructorRejectsInvalidArguments() + { + Assert.That( + () => new DekafKafkaClientAdapter(null!, TimeProvider.System), + Throws.TypeOf()); + Assert.That( + () => new DekafKafkaClientAdapter(NUnitTelemetryContext.Create(), null!), + Throws.TypeOf()); + } + + [Test] + public void FactoryCreatesDekafAdapter() + { + var factory = new DekafKafkaClientFactory(); + + object adapter = factory.Create(NUnitTelemetryContext.Create(), TimeProvider.System); + + Assert.That(adapter, Is.InstanceOf()); + } + + [Test] + public async Task ConnectDisconnectTransitionsStateAsync() + { + await using var adapter = new DekafKafkaClientAdapter(NUnitTelemetryContext.Create(), TimeProvider.System); + var events = new List(); + adapter.ConnectionStateChanged += (_, e) => events.Add(e); + var options = new KafkaConnectionOptions + { + Endpoint = KafkaTestHelper.EndpointUrl + }; + + await adapter.ConnectAsync(options, CancellationToken.None).ConfigureAwait(false); + await adapter.ConnectAsync(options, CancellationToken.None).ConfigureAwait(false); + await adapter.DisconnectAsync(CancellationToken.None).ConfigureAwait(false); + await adapter.DisconnectAsync(CancellationToken.None).ConfigureAwait(false); + + Assert.That(adapter.IsConnected, Is.False); + Assert.That(events, Has.Count.EqualTo(2)); + Assert.That(events[0].IsConnected, Is.True); + Assert.That(events[1].IsConnected, Is.False); + } + + [Test] + public async Task ConnectValidatesCredentialsTlsAndSaslAsync() + { + await using var adapter = new DekafKafkaClientAdapter(NUnitTelemetryContext.Create(), TimeProvider.System); + + Assert.That(async () => await adapter.ConnectAsync(null!, CancellationToken.None), + Throws.TypeOf()); + Assert.That(async () => await adapter.ConnectAsync(new KafkaConnectionOptions + { + Endpoint = KafkaTestHelper.EndpointUrl, + SaslMechanism = KafkaSaslMechanism.Plain, + UserName = "alice" + }, CancellationToken.None), Throws.TypeOf()); + Assert.That(async () => await adapter.ConnectAsync(new KafkaConnectionOptions + { + Endpoint = KafkaTestHelper.EndpointUrl, + SaslMechanism = KafkaSaslMechanism.OAuthBearer, + AllowCredentialsOverPlaintext = true + }, CancellationToken.None), Throws.TypeOf()); + Assert.That(async () => await adapter.ConnectAsync(new KafkaConnectionOptions + { + Endpoint = KafkaTestHelper.EndpointUrl, + Tls = new KafkaTlsOptions + { + UseTls = true, + ClientCertificatePath = "client.pem" + } + }, CancellationToken.None), Throws.TypeOf()); + } + + [Test] + public async Task SubscribeUnsubscribeGuardPathsDoNotRequireBrokerAsync() + { + await using var adapter = new DekafKafkaClientAdapter(NUnitTelemetryContext.Create(), TimeProvider.System); + await adapter.ConnectAsync(new KafkaConnectionOptions { Endpoint = KafkaTestHelper.EndpointUrl }, + CancellationToken.None).ConfigureAwait(false); + + await adapter.SubscribeAsync(Array.Empty(), CancellationToken.None).ConfigureAwait(false); + await adapter.UnsubscribeAsync(Array.Empty(), CancellationToken.None).ConfigureAwait(false); + Assert.That(async () => await adapter.SubscribeAsync(null!, CancellationToken.None), + Throws.TypeOf()); + Assert.That(async () => await adapter.UnsubscribeAsync(null!, CancellationToken.None), + Throws.TypeOf()); + } + + [Test] + public async Task ProduceAndDisposedGuardsDoNotRequireBrokerAsync() + { + var adapter = new DekafKafkaClientAdapter(NUnitTelemetryContext.Create(), TimeProvider.System); + await adapter.ConnectAsync(new KafkaConnectionOptions { Endpoint = KafkaTestHelper.EndpointUrl }, + CancellationToken.None).ConfigureAwait(false); + + Assert.That(async () => await adapter.ProduceAsync(default, CancellationToken.None), + Throws.TypeOf()); + + await adapter.DisposeAsync().ConfigureAwait(false); + Assert.That(async () => await adapter.ConnectAsync(new KafkaConnectionOptions(), CancellationToken.None), + Throws.TypeOf()); + Assert.That(async () => await adapter.SubscribeAsync(Array.Empty(), CancellationToken.None), + Throws.TypeOf()); + Assert.That(async () => await adapter.UnsubscribeAsync(Array.Empty(), CancellationToken.None), + Throws.TypeOf()); + } + + [Test] + public void PrivateMappingHelpersCoverKafkaConfigurationBranches() + { + var endpointOptions = new KafkaConnectionOptions { Endpoint = "kafka://broker1,broker2:19092" }; + var groupOptions = new KafkaConnectionOptions { ClientId = "client-a" }; + var passwordOptions = new KafkaConnectionOptions { PasswordBytes = System.Text.Encoding.UTF8.GetBytes("pw") }; + var tlsOptions = new KafkaConnectionOptions + { + Tls = new KafkaTlsOptions + { + UseTls = true, + ValidateServerCertificate = false, + CaCertificatePath = "ca.pem", + ClientCertificatePath = "client.pem", + ClientKeyPath = "client.key" + } + }; + var headersMessage = new KafkaMessage( + KafkaTestHelper.JsonTopic, + new byte[] { 0x01 }, + new byte[] { 0x02 }, + "application/json", + new Dictionary { ["x-opcua"] = "value" }); + + Assert.That(InvokePrivate("ResolveBootstrapServers", endpointOptions), + Is.EqualTo("broker1:9092,broker2:19092")); + Assert.That(InvokePrivate("ResolveGroupId", groupOptions), Does.StartWith("client-a-")); + Assert.That(InvokePrivate("ResolvePassword", passwordOptions), Is.EqualTo("pw")); + Assert.That(InvokePrivate("CreateHeaders", headersMessage), Is.Not.Null); + Assert.That(InvokePrivate("CreateTlsConfig", new KafkaConnectionOptions()), Is.Not.Null); + Assert.That(InvokePrivate("CreateTlsConfig", tlsOptions), Is.Not.Null); + Assert.That(InvokePrivate("MapAcks", KafkaAcks.None).ToString(), Is.EqualTo("None")); + Assert.That(InvokePrivate("MapAcks", KafkaAcks.Leader).ToString(), Is.EqualTo("Leader")); + Assert.That(InvokePrivate("MapAcks", KafkaAcks.All).ToString(), Is.EqualTo("All")); + Assert.That(InvokePrivate("MapAutoOffsetReset", KafkaAutoOffsetReset.Earliest).ToString(), + Is.EqualTo("Earliest")); + Assert.That(InvokePrivate("MapAutoOffsetReset", KafkaAutoOffsetReset.Latest).ToString(), + Is.EqualTo("Latest")); + } + + [Test] + public void PrivateValidationHelpersThrowForMissingSaslInputs() + { + object exception = InvokePrivate( + "CreateUnsupportedSaslMechanismException", + KafkaSaslMechanism.OAuthBearer); + + Assert.That(() => InvokePrivate("RequireUserName", new KafkaConnectionOptions()), + Throws.TypeOf()); + Assert.That(() => InvokePrivate("ResolvePassword", new KafkaConnectionOptions()), + Throws.TypeOf()); + Assert.That(exception, Is.InstanceOf()); + } + + [Test] + public void PrivateProducerAndConsumerConfigHelpersCoverSecurityBranches() + { + var plaintext = new KafkaConnectionOptions + { + BootstrapServers = "broker.example.com:9092", + ClientId = "client-a", + GroupId = "group-a", + DeliveryGuarantee = KafkaQualityOfService.BestEffort, + AutoOffsetReset = KafkaAutoOffsetReset.Earliest, + EnableAutoCommit = false + }; + var tls = new KafkaConnectionOptions + { + BootstrapServers = "broker.example.com:9092", + ClientId = "client-tls", + GroupId = "group-tls", + SecurityProtocol = KafkaSecurityProtocol.Ssl, + Tls = new KafkaTlsOptions + { + UseTls = true, + ValidateServerCertificate = true, + CaCertificatePath = "ca.pem" + } + }; + var saslPlain = new KafkaConnectionOptions + { + BootstrapServers = "broker.example.com:9092", + UserName = "alice", + PasswordBytes = System.Text.Encoding.UTF8.GetBytes("password"), + SaslMechanism = KafkaSaslMechanism.Plain, + AllowCredentialsOverPlaintext = true + }; + var saslScram256 = new KafkaConnectionOptions + { + BootstrapServers = "broker.example.com:9092", + UserName = "alice", + PasswordBytes = System.Text.Encoding.UTF8.GetBytes("password"), + SaslMechanism = KafkaSaslMechanism.ScramSha256, + SecurityProtocol = KafkaSecurityProtocol.SaslSsl + }; + var saslScram512 = new KafkaConnectionOptions + { + BootstrapServers = "broker.example.com:9092", + UserName = "alice", + PasswordBytes = System.Text.Encoding.UTF8.GetBytes("password"), + SaslMechanism = KafkaSaslMechanism.ScramSha512, + Tls = new KafkaTlsOptions { UseTls = true } + }; + + InvokeConfig("ApplyProducerConfig", CreateProducerBuilder(), plaintext); + InvokeConfig("ApplyConsumerConfig", CreateConsumerBuilder(), plaintext); + InvokeConfig("ApplyProducerConfig", CreateProducerBuilder(), tls); + InvokeConfig("ApplyConsumerConfig", CreateConsumerBuilder(), tls); + InvokeConfig("ApplyProducerConfig", CreateProducerBuilder(), saslPlain); + InvokeConfig("ApplyConsumerConfig", CreateConsumerBuilder(), saslPlain); + InvokeConfig("ApplyProducerConfig", CreateProducerBuilder(), saslScram256); + InvokeConfig("ApplyConsumerConfig", CreateConsumerBuilder(), saslScram256); + InvokeConfig("ApplyProducerConfig", CreateProducerBuilder(), saslScram512); + InvokeConfig("ApplyConsumerConfig", CreateConsumerBuilder(), saslScram512); + + Assert.That( + () => InvokeConfig( + "ApplyProducerConfig", + CreateProducerBuilder(), + new KafkaConnectionOptions + { + BootstrapServers = "broker.example.com:9092", + SaslMechanism = KafkaSaslMechanism.Plain, + PasswordBytes = System.Text.Encoding.UTF8.GetBytes("password"), + AllowCredentialsOverPlaintext = true + }), + Throws.TypeOf()); + } + + private static T InvokePrivate(string methodName, params object?[] args) + { + MethodInfo method = typeof(DekafKafkaClientAdapter).GetMethod( + methodName, + BindingFlags.NonPublic | BindingFlags.Static)!; + try + { + return (T)method.Invoke(null, args)!; + } + catch (TargetInvocationException ex) when (ex.InnerException is not null) + { + throw ex.InnerException; + } + } + + private static void InvokeConfig(string methodName, object builder, KafkaConnectionOptions options) + { + MethodInfo method = typeof(DekafKafkaClientAdapter).GetMethod( + methodName, + BindingFlags.NonPublic | BindingFlags.Static)!; + try + { + method.Invoke(null, new[] { builder, options }); + } + catch (TargetInvocationException ex) when (ex.InnerException is not null) + { + throw ex.InnerException; + } + } + + private static object CreateProducerBuilder() + { + MethodInfo method = GetDekafKafkaMethod("CreateProducer") + .MakeGenericMethod(typeof(byte[]), typeof(byte[])); + return method.Invoke(null, null)!; + } + + private static object CreateConsumerBuilder() + { + MethodInfo method = GetDekafKafkaMethod("CreateConsumer") + .MakeGenericMethod(typeof(byte[]), typeof(byte[])); + return method.Invoke(null, null)!; + } + + private static MethodInfo GetDekafKafkaMethod(string name) + { + Type kafkaType = Type.GetType("Dekaf.Kafka, Dekaf", throwOnError: true)!; + foreach (MethodInfo method in kafkaType.GetMethods(BindingFlags.Public | BindingFlags.Static)) + { + if (method.Name == name && method.IsGenericMethodDefinition) + { + return method; + } + } + throw new MissingMethodException("Dekaf.Kafka", name); + } + } +} +#endif diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaConnectionOptionsTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaConnectionOptionsTests.cs new file mode 100644 index 0000000000..6b869227cf --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaConnectionOptionsTests.cs @@ -0,0 +1,158 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Microsoft.Extensions.Configuration; +using NUnit.Framework; +using Opc.Ua.PubSub.Tests; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// Verifies Kafka connection option defaults and configuration binding. + /// + [TestFixture] + [Category("Unit")] + [TestSpec("B.2", Summary = "Kafka connection options")] + public sealed class KafkaConnectionOptionsTests + { + [Test] + public void DefaultsMatchSecureBrokerTransportDefaults() + { + var options = new KafkaConnectionOptions(); + + Assert.That(options.Endpoint, Is.EqualTo(string.Empty)); + Assert.That(options.BootstrapServers, Is.EqualTo(string.Empty)); + Assert.That(options.ClientId, Is.Null); + Assert.That(options.GroupId, Is.Null); + Assert.That(options.SecurityProtocol, Is.EqualTo(KafkaSecurityProtocol.Plaintext)); + Assert.That(options.SaslMechanism, Is.EqualTo(KafkaSaslMechanism.None)); + Assert.That(options.UserName, Is.Null); + Assert.That(options.PasswordSecretId, Is.Null); + Assert.That(options.AuthenticationProfileUri, Is.Null); + Assert.That(options.ResourceUri, Is.Null); + Assert.That(options.AllowCredentialsOverPlaintext, Is.False); + Assert.That(options.Tls, Is.Null); + Assert.That(options.DeliveryGuarantee, Is.EqualTo(KafkaQualityOfService.AtLeastOnce)); + Assert.That(options.AutoOffsetReset, Is.EqualTo(KafkaAutoOffsetReset.Latest)); + Assert.That(options.EnableAutoCommit, Is.True); + Assert.That(options.Topics.Prefix, Is.EqualTo("opcua")); + Assert.That(options.ConnectTimeout, Is.EqualTo(TimeSpan.FromSeconds(10))); + Assert.That(options.MessageTimeout, Is.EqualTo(TimeSpan.FromSeconds(30))); + Assert.That(options.MaxMessageSize, Is.EqualTo(1048576)); + } + + [Test] + public void ConfigurationBindingPopulatesScalarAndNestedProperties() + { + IConfiguration configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Endpoint"] = "kafkas://broker.example.com:19092", + ["BootstrapServers"] = "broker.example.com:19092", + ["ClientId"] = "pub-client", + ["GroupId"] = "sub-group", + ["SecurityProtocol"] = "SaslSsl", + ["SaslMechanism"] = "ScramSha512", + ["UserName"] = "alice", + ["PasswordSecretId"] = "InMemory:kafka-password", + ["AuthenticationProfileUri"] = "http://opcfoundation.org/UA/Security/UserToken/Server/Password", + ["ResourceUri"] = "kafka-resource", + ["AllowCredentialsOverPlaintext"] = "true", + ["DeliveryGuarantee"] = "ExactlyOnce", + ["AutoOffsetReset"] = "Earliest", + ["EnableAutoCommit"] = "false", + ["Topics:Prefix"] = "plant.a", + ["ConnectTimeout"] = "00:00:04", + ["MessageTimeout"] = "00:00:06", + ["MaxMessageSize"] = "4096", + ["Tls:UseTls"] = "true", + ["Tls:ValidateServerCertificate"] = "false", + ["Tls:CaCertificatePath"] = "certs/ca.pem", + ["Tls:ClientCertificatePath"] = "certs/client.pem", + ["Tls:ClientKeyPath"] = "certs/client.key" + }) + .Build(); + var options = new KafkaConnectionOptions(); + + configuration.Bind(options); + + Assert.That(options.Endpoint, Is.EqualTo("kafkas://broker.example.com:19092")); + Assert.That(options.BootstrapServers, Is.EqualTo("broker.example.com:19092")); + Assert.That(options.ClientId, Is.EqualTo("pub-client")); + Assert.That(options.GroupId, Is.EqualTo("sub-group")); + Assert.That(options.SecurityProtocol, Is.EqualTo(KafkaSecurityProtocol.SaslSsl)); + Assert.That(options.SaslMechanism, Is.EqualTo(KafkaSaslMechanism.ScramSha512)); + Assert.That(options.UserName, Is.EqualTo("alice")); + Assert.That(options.PasswordSecretId, Is.EqualTo("InMemory:kafka-password")); + Assert.That(options.AuthenticationProfileUri, Does.Contain("Password")); + Assert.That(options.ResourceUri, Is.EqualTo("kafka-resource")); + Assert.That(options.AllowCredentialsOverPlaintext, Is.True); + Assert.That(options.DeliveryGuarantee, Is.EqualTo(KafkaQualityOfService.ExactlyOnce)); + Assert.That(options.AutoOffsetReset, Is.EqualTo(KafkaAutoOffsetReset.Earliest)); + Assert.That(options.EnableAutoCommit, Is.False); + Assert.That(options.Topics.Prefix, Is.EqualTo("plant.a")); + Assert.That(options.ConnectTimeout, Is.EqualTo(TimeSpan.FromSeconds(4))); + Assert.That(options.MessageTimeout, Is.EqualTo(TimeSpan.FromSeconds(6))); + Assert.That(options.MaxMessageSize, Is.EqualTo(4096)); + Assert.That(options.Tls, Is.Not.Null); + Assert.That(options.Tls!.UseTls, Is.True); + Assert.That(options.Tls.ValidateServerCertificate, Is.False); + Assert.That(options.Tls.CaCertificatePath, Is.EqualTo("certs/ca.pem")); + Assert.That(options.Tls.ClientCertificatePath, Is.EqualTo("certs/client.pem")); + Assert.That(options.Tls.ClientKeyPath, Is.EqualTo("certs/client.key")); + } + + [Test] + public void OptionsTypeDoesNotExposePlainPasswordProperty() + { + PropertyInfo[] properties = typeof(KafkaConnectionOptions) + .GetProperties(BindingFlags.Public | BindingFlags.Instance); + IEnumerable propertyNames = properties.Select(static p => p.Name); + + Assert.That(propertyNames, Does.Not.Contain("Password")); + Assert.That(propertyNames, Does.Contain("PasswordSecretId")); + } + + [Test] + public void TlsOptionsDefaultsValidateBrokerCertificate() + { + var tls = new KafkaTlsOptions(); + + Assert.That(tls.UseTls, Is.False); + Assert.That(tls.ValidateServerCertificate, Is.True); + Assert.That(tls.CaCertificatePath, Is.Null); + Assert.That(tls.ClientCertificatePath, Is.Null); + Assert.That(tls.ClientKeyPath, Is.Null); + } + } +} diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaEncodingTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaEncodingTests.cs new file mode 100644 index 0000000000..165c932ddd --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaEncodingTests.cs @@ -0,0 +1,73 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using NUnit.Framework; +using Opc.Ua.PubSub.Tests; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// Verifies profile and message encoding helpers for Kafka records. + /// + [TestFixture] + [Category("Unit")] + [TestSpec("B.2", Summary = "Kafka profile content types")] + public sealed class KafkaEncodingTests + { + [Test] + [TestCase(KafkaEncoding.Uadp, "uadp", "application/opcua+uadp")] + [TestCase(KafkaEncoding.Json, "json", "application/json")] + public void EncodingMapsToTopicSegmentAndContentType( + KafkaEncoding encoding, + string expectedSegment, + string expectedContentType) + { + Assert.That(encoding.ToTopicSegment(), Is.EqualTo(expectedSegment)); + Assert.That(encoding.ToContentType(), Is.EqualTo(expectedContentType)); + } + + [Test] + public void UndefinedEncodingThrowsArgumentOutOfRangeException() + { + Assert.That( + () => ((KafkaEncoding)123).ToContentType(), + Throws.TypeOf()); + Assert.That( + () => ((KafkaEncoding)123).ToTopicSegment(), + Throws.TypeOf()); + } + + [Test] + public void KafkaProfilesExposeJsonAndUadpProfileUris() + { + Assert.That(KafkaProfiles.PubSubKafkaJsonTransport, Does.Contain("pubsub-kafka-json")); + Assert.That(KafkaProfiles.PubSubKafkaUadpTransport, Does.Contain("pubsub-kafka-uadp")); + } + } +} diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaEndpointParserTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaEndpointParserTests.cs new file mode 100644 index 0000000000..8d6d43f129 --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaEndpointParserTests.cs @@ -0,0 +1,118 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using NUnit.Framework; +using Opc.Ua.PubSub.Tests; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// Validates Kafka endpoint parsing for Part 14 Annex B.2 broker URLs. + /// + [TestFixture] + [Category("Unit")] + [TestSpec("B.2", Summary = "Kafka endpoint URL parsing")] + public sealed class KafkaEndpointParserTests + { + [Test] + public void ParseKafkaSchemeUsesDefaultPort() + { + KafkaEndpoint endpoint = KafkaEndpointParser.Parse("kafka://broker.example.com"); + + Assert.That(endpoint.BootstrapServers, Is.EqualTo("broker.example.com:9092")); + Assert.That(endpoint.UseTls, Is.False); + } + + [Test] + public void ParseKafkasSchemeUsesDefaultPortAndTls() + { + KafkaEndpoint endpoint = KafkaEndpointParser.Parse("kafkas://broker.example.com"); + + Assert.That(endpoint.BootstrapServers, Is.EqualTo("broker.example.com:9092")); + Assert.That(endpoint.UseTls, Is.True); + } + + [Test] + public void ParseCommaSeparatedBootstrapListNormalizesEntries() + { + KafkaEndpoint endpoint = KafkaEndpointParser.Parse( + "kafka://broker1.example.com:19092, broker2.example.com,broker3.example.com:29092/path"); + + Assert.That( + endpoint.BootstrapServers, + Is.EqualTo("broker1.example.com:19092,broker2.example.com:9092,broker3.example.com:29092")); + } + + [Test] + public void ParseIpv6BootstrapServersPreservesBrackets() + { + KafkaEndpoint endpoint = KafkaEndpointParser.Parse("kafka://[::1],[2001:db8::1]:19092"); + + Assert.That(endpoint.BootstrapServers, Is.EqualTo("[::1]:9092,[2001:db8::1]:19092")); + } + + [Test] + public void ParseSchemeIsCaseInsensitive() + { + KafkaEndpoint endpoint = KafkaEndpointParser.Parse("KAFKAS://broker.example.com:19092"); + + Assert.That(endpoint.UseTls, Is.True); + Assert.That(endpoint.BootstrapServers, Is.EqualTo("broker.example.com:19092")); + } + + [Test] + public void ParseNullUrlThrowsArgumentNullException() + { + Assert.That( + () => KafkaEndpointParser.Parse(null!), + Throws.TypeOf()); + } + + [Test] + [TestCase("")] + [TestCase("http://broker.example.com")] + [TestCase("kafka:/missing-slash")] + [TestCase("kafka://")] + [TestCase("kafka://broker.example.com,")] + [TestCase("kafka://:9092")] + [TestCase("kafka://broker.example.com:0")] + [TestCase("kafka://broker.example.com:70000")] + [TestCase("kafka://broker.example.com:abc")] + [TestCase("kafka://[::1")] + [TestCase("kafka://[]:9092")] + [TestCase("kafka://[::1]x")] + public void ParseMalformedUrlThrowsFormatException(string url) + { + Assert.That( + () => KafkaEndpointParser.Parse(url), + Throws.TypeOf()); + } + } +} diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs new file mode 100644 index 0000000000..7af20b809b --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs @@ -0,0 +1,136 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#if NET8_0_OR_GREATER +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Opc.Ua.PubSub.Tests; +using Opc.Ua.PubSub.Transports; +using Opc.Ua.Tests; +using Testcontainers.Kafka; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// Docker-backed Kafka broker integration test using Testcontainers. + /// + [TestFixture] + [Category("Integration")] + [TestSpec("B.2", Summary = "Kafka real broker round trip")] + [CancelAfter(60000)] + public sealed class KafkaIntegrationDockerTests + { + private KafkaContainer? m_container; + + [OneTimeSetUp] + public async Task OneTimeSetUpAsync() + { + try + { + m_container = new KafkaBuilder("confluentinc/cp-kafka:7.5.12").Build(); + await m_container.StartAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + Assert.Ignore($"Docker/Kafka not available: {ex.Message}"); + } + } + + [OneTimeTearDown] + public async Task OneTimeTearDownAsync() + { + if (m_container is not null) + { + await m_container.DisposeAsync().ConfigureAwait(false); + } + } + + [Test] + public async Task RealBrokerRoundTripsPayloadAsync() + { + Assert.That(m_container, Is.Not.Null); + string bootstrapAddress = m_container!.GetBootstrapAddress(); + string topic = "opcua-json-data-" + Guid.NewGuid().ToString("N"); + string url = "kafka://" + bootstrapAddress; + IConfigurationRoot configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new System.Collections.Generic.Dictionary + { + ["OpcUa:PubSub:Kafka:AutoOffsetReset"] = "Earliest", + ["OpcUa:PubSub:Kafka:GroupId"] = "opcua-tests-" + Guid.NewGuid().ToString("N"), + ["OpcUa:PubSub:Kafka:ClientId"] = "opcua-kafka-tests" + }) + .Build(); + var services = new ServiceCollection(); + services.AddOpcUa().AddPubSub(pubsub => pubsub.AddKafkaTransport(configuration)); + await using ServiceProvider serviceProvider = services.BuildServiceProvider(); + KafkaPubSubTransportFactory factory = serviceProvider + .GetServices() + .OfType() + .Single(f => f.TransportProfileUri == KafkaProfiles.PubSubKafkaJsonTransport); + PubSubConnectionDataType pubConnection = KafkaTestHelper.NewConnection(url: url, dataTopic: topic); + PubSubConnectionDataType subConnection = KafkaTestHelper.NewConnection( + url: url, + writer: false, + reader: true, + dataTopic: topic, + metadataTopic: topic + ".metadata"); + await using IPubSubTransport publisher = factory.Create( + pubConnection, + NUnitTelemetryContext.Create(), + TimeProvider.System); + await using IPubSubTransport subscriber = factory.Create( + subConnection, + NUnitTelemetryContext.Create(), + TimeProvider.System); + if (subscriber is KafkaBrokerTransport kafkaSubscriber) + { + kafkaSubscriber.Subscriptions.Add(topic); + } + + await subscriber.OpenAsync(CancellationToken.None).ConfigureAwait(false); + await publisher.OpenAsync(CancellationToken.None).ConfigureAwait(false); + byte[] payload = [0x10, 0x20, 0x30, 0x40]; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + Task receiveTask = KafkaTestHelper.ReceiveOneAsync(subscriber, cts.Token); + + await publisher.SendAsync(payload, topic).ConfigureAwait(false); + PubSubTransportFrame? frame = await receiveTask.ConfigureAwait(false); + + Assert.That(frame, Is.Not.Null); + Assert.That(frame!.Value.Topic, Is.EqualTo(topic)); + Assert.That(frame.Value.Payload.ToArray(), Is.EqualTo(payload)); + } + } +} +#endif diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaPubSubTransportFactoryTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaPubSubTransportFactoryTests.cs new file mode 100644 index 0000000000..8212b6b42d --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaPubSubTransportFactoryTests.cs @@ -0,0 +1,320 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using Microsoft.Extensions.Options; +using Moq; +using NUnit.Framework; +using Opc.Ua.PubSub.Tests; +using Opc.Ua.PubSub.Transports; +using Opc.Ua.Tests; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// Verifies Kafka transport factory profile validation and connection resolution. + /// + [TestFixture] + [Category("Unit")] + [TestSpec("B.2", Summary = "Kafka transport factory")] + [CancelAfter(5000)] + public sealed class KafkaPubSubTransportFactoryTests + { + [Test] + public void ConstructorRejectsInvalidArguments() + { + Assert.That( + () => new KafkaPubSubTransportFactory( + string.Empty, + new FakeKafkaClientFactory(), + Options.Create(new KafkaConnectionOptions())), + Throws.TypeOf()); + Assert.That( + () => new KafkaPubSubTransportFactory( + Profiles.PubSubUdpUadpTransport, + new FakeKafkaClientFactory(), + Options.Create(new KafkaConnectionOptions())), + Throws.TypeOf()); + Assert.That( + () => new KafkaPubSubTransportFactory( + KafkaProfiles.PubSubKafkaJsonTransport, + clientFactory: null!, + Options.Create(new KafkaConnectionOptions())), + Throws.TypeOf()); + Assert.That( + () => new KafkaPubSubTransportFactory( + KafkaProfiles.PubSubKafkaJsonTransport, + new FakeKafkaClientFactory(), + defaultOptions: null!), + Throws.TypeOf()); + } + + [Test] + public void TransportProfileUriReturnsConstructorValue() + { + KafkaPubSubTransportFactory factory = KafkaTestHelper.NewFactory( + KafkaProfiles.PubSubKafkaUadpTransport); + + Assert.That(factory.TransportProfileUri, Is.EqualTo(KafkaProfiles.PubSubKafkaUadpTransport)); + } + + [Test] + public void CreateValidConnectionReturnsKafkaBrokerTransport() + { + IPubSubTransport transport = KafkaTestHelper.NewFactory().Create( + KafkaTestHelper.NewConnection(), + NUnitTelemetryContext.Create(), + TimeProvider.System); + + Assert.That(transport, Is.InstanceOf()); + Assert.That(transport.TransportProfileUri, Is.EqualTo(KafkaProfiles.PubSubKafkaJsonTransport)); + } + + [Test] + public void CreateReadsNetworkAddressUrlAndAppliesTlsEndpoint() + { + KafkaPubSubTransportFactory factory = KafkaTestHelper.NewFactory(); + PubSubConnectionDataType connection = KafkaTestHelper.NewConnection("kafkas://broker.example.com:19092"); + + var transport = (KafkaBrokerTransport)factory.Create( + connection, + NUnitTelemetryContext.Create(), + TimeProvider.System); + + Assert.That(transport.Endpoint.BootstrapServers, Is.EqualTo("broker.example.com:19092")); + Assert.That(transport.Endpoint.UseTls, Is.True); + Assert.That(transport.Options.Endpoint, Is.EqualTo("kafkas://broker.example.com:19092")); + Assert.That(transport.Options.BootstrapServers, Is.EqualTo("broker.example.com:19092")); + Assert.That(transport.Options.Tls, Is.Not.Null); + Assert.That(transport.Options.Tls!.UseTls, Is.True); + Assert.That(transport.Options.SecurityProtocol, Is.EqualTo(KafkaSecurityProtocol.Ssl)); + } + + [Test] + public void CreateAppliesBrokerTransportAuthenticationSettings() + { + KafkaPubSubTransportFactory factory = KafkaTestHelper.NewFactory(); + PubSubConnectionDataType connection = KafkaTestHelper.NewConnection(); + connection.TransportSettings = new ExtensionObject(new BrokerWriterGroupTransportDataType + { + AuthenticationProfileUri = "http://opcfoundation.org/UA/Security/UserToken/Server/Password", + ResourceUri = "alice" + }); + + var transport = (KafkaBrokerTransport)factory.Create( + connection, + NUnitTelemetryContext.Create(), + TimeProvider.System); + + Assert.That(transport.Options.AuthenticationProfileUri, Does.Contain("Password")); + Assert.That(transport.Options.ResourceUri, Is.EqualTo("alice")); + Assert.That(transport.Options.UserName, Is.EqualTo("alice")); + } + + [Test] + public void CreateDoesNotOverrideDefaultAuthenticationProfile() + { + var options = new KafkaConnectionOptions + { + AuthenticationProfileUri = "default-profile", + ResourceUri = "default-resource", + UserName = "configured" + }; + KafkaPubSubTransportFactory factory = KafkaTestHelper.NewFactory(options: options); + PubSubConnectionDataType connection = KafkaTestHelper.NewConnection(); + connection.TransportSettings = new ExtensionObject(new BrokerWriterGroupTransportDataType + { + AuthenticationProfileUri = "broker-profile", + ResourceUri = "broker-resource" + }); + + var transport = (KafkaBrokerTransport)factory.Create( + connection, + NUnitTelemetryContext.Create(), + TimeProvider.System); + + Assert.That(transport.Options.AuthenticationProfileUri, Is.EqualTo("default-profile")); + Assert.That(transport.Options.ResourceUri, Is.EqualTo("default-resource")); + Assert.That(transport.Options.UserName, Is.EqualTo("configured")); + } + + [Test] + public void CreateResolvesPasswordViaSecretRegistryMock() + { + byte[] expected = [0xAA, 0xBB, 0xCC]; + var secret = new TestSecret(expected); + var registry = new Mock(MockBehavior.Strict); + registry.Setup(r => r.TryGet(It.Is(id => + id.StoreType == InMemorySecretStore.DefaultStoreType && id.Name == "kafka-password"))) + .Returns(secret); + KafkaPubSubTransportFactory factory = KafkaTestHelper.NewFactory( + options: new KafkaConnectionOptions { PasswordSecretId = "InMemory:kafka-password" }, + secretRegistry: registry.Object); + + var transport = (KafkaBrokerTransport)factory.Create( + KafkaTestHelper.NewConnection(), + NUnitTelemetryContext.Create(), + TimeProvider.System); + + Assert.That(transport.Options.PasswordBytes, Is.EqualTo(expected)); + Assert.That(secret.DisposeCount, Is.EqualTo(1)); + registry.VerifyAll(); + } + + [Test] + public void CreatePasswordSecretFailuresThrowInvalidOperationException() + { + KafkaPubSubTransportFactory noRegistry = KafkaTestHelper.NewFactory(options: new KafkaConnectionOptions + { + PasswordSecretId = "InMemory:kafka-password" + }); + var registry = new Mock(MockBehavior.Strict); + registry.Setup(r => r.TryGet(It.IsAny())).Returns((ISecret?)null); + KafkaPubSubTransportFactory missingSecret = KafkaTestHelper.NewFactory( + options: new KafkaConnectionOptions { PasswordSecretId = "missing-secret" }, + secretRegistry: registry.Object); + + Assert.That( + () => noRegistry.Create(KafkaTestHelper.NewConnection(), NUnitTelemetryContext.Create(), TimeProvider.System), + Throws.TypeOf()); + Assert.That( + () => missingSecret.Create( + KafkaTestHelper.NewConnection(), + NUnitTelemetryContext.Create(), + TimeProvider.System), + Throws.TypeOf()); + } + + [Test] + public void CreateDeterminesDirectionFromWriterAndReaderGroups() + { + Assert.That(CreateDirection(KafkaTestHelper.NewConnection(writer: true, reader: false)), + Is.EqualTo(PubSubTransportDirection.Send)); + Assert.That(CreateDirection(KafkaTestHelper.NewConnection(writer: false, reader: true)), + Is.EqualTo(PubSubTransportDirection.Receive)); + Assert.That(CreateDirection(KafkaTestHelper.NewConnection(writer: true, reader: true)), + Is.EqualTo(PubSubTransportDirection.SendReceive)); + } + + [Test] + public void CreateNoGroupsDefaultsToSendReceiveDirection() + { + var connection = new PubSubConnectionDataType + { + Name = "Conn", + TransportProfileUri = KafkaProfiles.PubSubKafkaJsonTransport, + Address = new ExtensionObject(new NetworkAddressUrlDataType + { + Url = KafkaTestHelper.EndpointUrl + }) + }; + + Assert.That(CreateDirection(connection), Is.EqualTo(PubSubTransportDirection.SendReceive)); + } + + [Test] + public void CreateWriterGroupDeliveryGuaranteeOverridesDefaultOptions() + { + PubSubConnectionDataType connection = KafkaTestHelper.NewConnection( + requestedDeliveryGuarantee: BrokerTransportQualityOfService.ExactlyOnce); + + var transport = (KafkaBrokerTransport)KafkaTestHelper.NewFactory().Create( + connection, + NUnitTelemetryContext.Create(), + TimeProvider.System); + + Assert.That(transport.Options.DeliveryGuarantee, Is.EqualTo(KafkaQualityOfService.ExactlyOnce)); + } + + [Test] + public void CreateUadpFactoryProducesUadpTransport() + { + KafkaPubSubTransportFactory factory = KafkaTestHelper.NewFactory(KafkaProfiles.PubSubKafkaUadpTransport); + PubSubConnectionDataType connection = KafkaTestHelper.NewConnection( + profile: KafkaProfiles.PubSubKafkaUadpTransport); + + IPubSubTransport transport = factory.Create( + connection, + NUnitTelemetryContext.Create(), + TimeProvider.System); + + Assert.That(transport.TransportProfileUri, Is.EqualTo(KafkaProfiles.PubSubKafkaUadpTransport)); + } + + [Test] + public void CreateRejectsInvalidCreateArgumentsAndAddresses() + { + KafkaPubSubTransportFactory factory = KafkaTestHelper.NewFactory(); + PubSubConnectionDataType connection = KafkaTestHelper.NewConnection(); + var noAddress = new PubSubConnectionDataType { Name = "NoAddress" }; + var noUrl = new PubSubConnectionDataType + { + Name = "NoUrl", + Address = new ExtensionObject(new NetworkAddressUrlDataType()) + }; + + Assert.That(() => factory.Create(null!, NUnitTelemetryContext.Create(), TimeProvider.System), + Throws.TypeOf()); + Assert.That(() => factory.Create(connection, null!, TimeProvider.System), + Throws.TypeOf()); + Assert.That(() => factory.Create(connection, NUnitTelemetryContext.Create(), null!), + Throws.TypeOf()); + Assert.That(() => factory.Create(noAddress, NUnitTelemetryContext.Create(), TimeProvider.System), + Throws.TypeOf()); + Assert.That(() => factory.Create(noUrl, NUnitTelemetryContext.Create(), TimeProvider.System), + Throws.TypeOf()); + } + + private static PubSubTransportDirection CreateDirection(PubSubConnectionDataType connection) + { + return KafkaTestHelper.NewFactory() + .Create(connection, NUnitTelemetryContext.Create(), TimeProvider.System) + .Direction; + } + + private sealed class TestSecret : ISecret + { + private readonly byte[] m_bytes; + + public TestSecret(byte[] bytes) + { + m_bytes = bytes; + } + + public int DisposeCount { get; private set; } + + public ReadOnlySpan Bytes => m_bytes; + + public void Dispose() + { + DisposeCount++; + } + } + } +} diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaQosMappingTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaQosMappingTests.cs new file mode 100644 index 0000000000..7c79ceab17 --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaQosMappingTests.cs @@ -0,0 +1,84 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using NUnit.Framework; +using Opc.Ua.PubSub.Tests; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// Verifies Part 14 Annex B.2 delivery guarantees map to Kafka producer settings. + /// + [TestFixture] + [Category("Unit")] + [TestSpec("B.2", Summary = "Kafka QoS to acks and idempotence mapping")] + public sealed class KafkaQosMappingTests + { + [Test] + [TestCase(KafkaQualityOfService.BestEffort, KafkaAcks.None, false)] + [TestCase(KafkaQualityOfService.AtMostOnce, KafkaAcks.Leader, false)] + [TestCase(KafkaQualityOfService.AtLeastOnce, KafkaAcks.All, false)] + [TestCase(KafkaQualityOfService.ExactlyOnce, KafkaAcks.All, true)] + public void QualityOfServiceMapsToKafkaDeliveryGuarantee( + KafkaQualityOfService qos, + KafkaAcks expectedAcks, + bool expectedIdempotence) + { + KafkaDeliveryGuarantee guarantee = qos.ToDeliveryGuarantee(); + + Assert.That(guarantee.Acks, Is.EqualTo(expectedAcks)); + Assert.That(guarantee.EnableIdempotence, Is.EqualTo(expectedIdempotence)); + } + + [Test] + [TestCase(BrokerTransportQualityOfService.NotSpecified, KafkaQualityOfService.AtLeastOnce)] + [TestCase(BrokerTransportQualityOfService.BestEffort, KafkaQualityOfService.BestEffort)] + [TestCase(BrokerTransportQualityOfService.AtMostOnce, KafkaQualityOfService.AtMostOnce)] + [TestCase(BrokerTransportQualityOfService.AtLeastOnce, KafkaQualityOfService.AtLeastOnce)] + [TestCase(BrokerTransportQualityOfService.ExactlyOnce, KafkaQualityOfService.ExactlyOnce)] + public void BrokerGuaranteeMapsAllPart14Values( + BrokerTransportQualityOfService brokerGuarantee, + KafkaQualityOfService expected) + { + KafkaQualityOfService actual = KafkaQualityOfServiceExtensions.FromBrokerGuarantee( + brokerGuarantee, + KafkaQualityOfService.AtLeastOnce); + + Assert.That(actual, Is.EqualTo(expected)); + } + + [Test] + public void UndefinedQualityOfServiceThrowsArgumentOutOfRangeException() + { + Assert.That( + () => ((KafkaQualityOfService)123).ToDeliveryGuarantee(), + Throws.TypeOf()); + } + } +} diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaTestHelper.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaTestHelper.cs new file mode 100644 index 0000000000..b1e2476f55 --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaTestHelper.cs @@ -0,0 +1,189 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Concurrent; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using NUnit.Framework; +using Opc.Ua.PubSub.Diagnostics; +using Opc.Ua.PubSub.Kafka.Internal; +using Opc.Ua.PubSub.Transports; +using Opc.Ua.Tests; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + internal static class KafkaTestHelper + { + public const string EndpointUrl = "kafka://broker.example.com:9092"; + public const string JsonTopic = "opcua.json.data.42.1.3"; + public const string UadpTopic = "opcua.uadp.data.42.1.3"; + public const string MetadataTopic = "opcua.json.metadata.42.1.3"; + + public static PubSubConnectionDataType NewConnection( + string url = EndpointUrl, + bool writer = true, + bool reader = false, + string profile = KafkaProfiles.PubSubKafkaJsonTransport, + string? dataTopic = null, + string? metadataTopic = null, + BrokerTransportQualityOfService requestedDeliveryGuarantee = + BrokerTransportQualityOfService.NotSpecified) + { + var connection = new PubSubConnectionDataType + { + Name = writer ? "Publisher" : "Subscriber", + TransportProfileUri = profile, + PublisherId = new Variant((uint)42), + Address = new ExtensionObject(new NetworkAddressUrlDataType { Url = url }) + }; + if (writer) + { + var group = new WriterGroupDataType + { + Name = "WriterGroup1", + WriterGroupId = 1, + MessageSettings = new ExtensionObject(CreateWriterGroupMessageSettings(profile)) + }; + if (requestedDeliveryGuarantee != BrokerTransportQualityOfService.NotSpecified) + { + group.TransportSettings = new ExtensionObject(new BrokerWriterGroupTransportDataType + { + RequestedDeliveryGuarantee = requestedDeliveryGuarantee + }); + } + var dataSetWriter = new DataSetWriterDataType + { + Name = "DataSetWriter1", + DataSetWriterId = 3 + }; + if (dataTopic is not null || metadataTopic is not null) + { + dataSetWriter.TransportSettings = new ExtensionObject(new BrokerDataSetWriterTransportDataType + { + QueueName = dataTopic, + MetaDataQueueName = metadataTopic + }); + } + group.DataSetWriters = group.DataSetWriters.AddItem(dataSetWriter); + connection.WriterGroups = connection.WriterGroups.AddItem(group); + } + if (reader) + { + var readerGroup = new ReaderGroupDataType { Name = "ReaderGroup1" }; + var dataSetReader = new DataSetReaderDataType { Name = "DataSetReader1" }; + if (dataTopic is not null || metadataTopic is not null) + { + dataSetReader.TransportSettings = new ExtensionObject(new BrokerDataSetReaderTransportDataType + { + QueueName = dataTopic, + MetaDataQueueName = metadataTopic + }); + } + readerGroup.DataSetReaders = readerGroup.DataSetReaders.AddItem(dataSetReader); + connection.ReaderGroups = connection.ReaderGroups.AddItem(readerGroup); + } + return connection; + } + + public static KafkaBrokerTransport NewTransport( + FakeKafkaClientFactory factory, + PubSubTransportDirection direction = PubSubTransportDirection.Send, + KafkaConnectionOptions? options = null, + PubSubConnectionDataType? connection = null, + IPubSubDiagnostics? diagnostics = null) + { + PubSubConnectionDataType conn = connection ?? NewConnection(); + return new KafkaBrokerTransport( + conn, + KafkaEndpointParser.Parse(EndpointUrl), + direction, + options ?? new KafkaConnectionOptions + { + Endpoint = EndpointUrl, + BootstrapServers = "broker.example.com:9092" + }, + factory, + NUnitTelemetryContext.Create(), + TimeProvider.System, + diagnostics); + } + + public static KafkaPubSubTransportFactory NewFactory( + string transportProfileUri = KafkaProfiles.PubSubKafkaJsonTransport, + IKafkaClientFactory? clientFactory = null, + KafkaConnectionOptions? options = null, + ISecretRegistry? secretRegistry = null, + IPubSubDiagnostics? diagnostics = null) + { + return new KafkaPubSubTransportFactory( + transportProfileUri, + clientFactory ?? new FakeKafkaClientFactory(), + Options.Create(options ?? new KafkaConnectionOptions()), + secretRegistry, + diagnostics); + } + + public static async Task ReceiveOneAsync( + IPubSubTransport transport, + CancellationToken cancellationToken) + { + try + { + await foreach (PubSubTransportFrame frame in transport.ReceiveAsync(cancellationToken) + .ConfigureAwait(false)) + { + return frame; + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return null; + } + return null; + } + + public static KafkaMessage FirstProduced(FakeKafkaClientAdapter adapter) + { + Assert.That(adapter.ProducedMessages, Has.Count.EqualTo(1)); + var queue = (ConcurrentQueue)adapter.ProducedMessages; + bool hasValue = queue.TryPeek(out KafkaMessage message); + Assert.That(hasValue, Is.True); + return message; + } + + private static IEncodeable CreateWriterGroupMessageSettings(string profile) + { + return string.Equals(profile, KafkaProfiles.PubSubKafkaUadpTransport, StringComparison.Ordinal) + ? new UadpWriterGroupMessageDataType() + : new JsonWriterGroupMessageDataType(); + } + } +} diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaTransportServiceCollectionExtensionsTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaTransportServiceCollectionExtensionsTests.cs new file mode 100644 index 0000000000..9aa044c789 --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaTransportServiceCollectionExtensionsTests.cs @@ -0,0 +1,180 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using NUnit.Framework; +using Opc.Ua.PubSub.Kafka.Internal; +using Opc.Ua.PubSub.Tests; +using Opc.Ua.PubSub.Transports; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// Verifies Kafka transport dependency injection registration and option binding. + /// + [TestFixture] + [Category("Unit")] + [TestSpec("B.2", Summary = "Kafka transport DI binding")] + public sealed class KafkaTransportServiceCollectionExtensionsTests + { + [Test] + public async Task AddKafkaTransportWithCallbackRegistersFactoriesAndOptionsAsync() + { + var services = new ServiceCollection(); + + services.AddOpcUa().AddPubSub(pubsub => pubsub.AddKafkaTransport(options => + { + options.Endpoint = "kafka://broker.example.com:9092"; + options.ClientId = "callback-client"; + options.GroupId = "callback-group"; + options.Topics.Prefix = "callback"; + options.DeliveryGuarantee = KafkaQualityOfService.ExactlyOnce; + })); + + await using ServiceProvider serviceProvider = services.BuildServiceProvider(); + KafkaConnectionOptions options = serviceProvider + .GetRequiredService>() + .Value; + KafkaPubSubTransportFactory[] factories = serviceProvider + .GetServices() + .OfType() + .ToArray(); + + Assert.That(options.ClientId, Is.EqualTo("callback-client")); + Assert.That(options.GroupId, Is.EqualTo("callback-group")); + Assert.That(options.Topics.Prefix, Is.EqualTo("callback")); + Assert.That(options.DeliveryGuarantee, Is.EqualTo(KafkaQualityOfService.ExactlyOnce)); + Assert.That(factories, Has.Length.EqualTo(2)); + Assert.That( + factories.Select(static f => f.TransportProfileUri), + Is.EquivalentTo(new[] + { + KafkaProfiles.PubSubKafkaJsonTransport, + KafkaProfiles.PubSubKafkaUadpTransport + })); + Assert.That(serviceProvider.GetRequiredService(), Is.Not.Null); + } + + [Test] + public async Task AddKafkaTransportWithConfigurationBindsDefaultSectionAsync() + { + var services = new ServiceCollection(); + IConfigurationRoot configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["OpcUa:PubSub:Kafka:Endpoint"] = "kafkas://broker.example.com:19092", + ["OpcUa:PubSub:Kafka:BootstrapServers"] = "broker.example.com:19092", + ["OpcUa:PubSub:Kafka:ClientId"] = "bound-client", + ["OpcUa:PubSub:Kafka:GroupId"] = "bound-group", + ["OpcUa:PubSub:Kafka:SecurityProtocol"] = "SaslSsl", + ["OpcUa:PubSub:Kafka:SaslMechanism"] = "Plain", + ["OpcUa:PubSub:Kafka:UserName"] = "alice", + ["OpcUa:PubSub:Kafka:PasswordSecretId"] = "InMemory:kafka-password", + ["OpcUa:PubSub:Kafka:AuthenticationProfileUri"] = "profile-uri", + ["OpcUa:PubSub:Kafka:ResourceUri"] = "resource-uri", + ["OpcUa:PubSub:Kafka:AllowCredentialsOverPlaintext"] = "true", + ["OpcUa:PubSub:Kafka:Topics:Prefix"] = "plant.a", + ["OpcUa:PubSub:Kafka:DeliveryGuarantee"] = "BestEffort", + ["OpcUa:PubSub:Kafka:AutoOffsetReset"] = "Earliest", + ["OpcUa:PubSub:Kafka:EnableAutoCommit"] = "false", + ["OpcUa:PubSub:Kafka:ConnectTimeout"] = "00:00:03", + ["OpcUa:PubSub:Kafka:MessageTimeout"] = "00:00:04", + ["OpcUa:PubSub:Kafka:MaxMessageSize"] = "2048", + ["OpcUa:PubSub:Kafka:Tls:UseTls"] = "true", + ["OpcUa:PubSub:Kafka:Tls:ValidateServerCertificate"] = "false", + ["OpcUa:PubSub:Kafka:Tls:CaCertificatePath"] = "ca.pem", + ["OpcUa:PubSub:Kafka:Tls:ClientCertificatePath"] = "client.pem", + ["OpcUa:PubSub:Kafka:Tls:ClientKeyPath"] = "client.key" + }) + .Build(); + + services.AddOpcUa().AddPubSub(pubsub => pubsub.AddKafkaTransport(configuration)); + + await using ServiceProvider serviceProvider = services.BuildServiceProvider(); + KafkaConnectionOptions options = serviceProvider + .GetRequiredService>() + .Value; + + Assert.That(options.Endpoint, Is.EqualTo("kafkas://broker.example.com:19092")); + Assert.That(options.BootstrapServers, Is.EqualTo("broker.example.com:19092")); + Assert.That(options.ClientId, Is.EqualTo("bound-client")); + Assert.That(options.GroupId, Is.EqualTo("bound-group")); + Assert.That(options.SecurityProtocol, Is.EqualTo(KafkaSecurityProtocol.SaslSsl)); + Assert.That(options.SaslMechanism, Is.EqualTo(KafkaSaslMechanism.Plain)); + Assert.That(options.UserName, Is.EqualTo("alice")); + Assert.That(options.PasswordSecretId, Is.EqualTo("InMemory:kafka-password")); + Assert.That(options.AuthenticationProfileUri, Is.EqualTo("profile-uri")); + Assert.That(options.ResourceUri, Is.EqualTo("resource-uri")); + Assert.That(options.AllowCredentialsOverPlaintext, Is.True); + Assert.That(options.Topics.Prefix, Is.EqualTo("plant.a")); + Assert.That(options.DeliveryGuarantee, Is.EqualTo(KafkaQualityOfService.BestEffort)); + Assert.That(options.AutoOffsetReset, Is.EqualTo(KafkaAutoOffsetReset.Earliest)); + Assert.That(options.EnableAutoCommit, Is.False); + Assert.That(options.ConnectTimeout, Is.EqualTo(TimeSpan.FromSeconds(3))); + Assert.That(options.MessageTimeout, Is.EqualTo(TimeSpan.FromSeconds(4))); + Assert.That(options.MaxMessageSize, Is.EqualTo(2048)); + Assert.That(options.Tls, Is.Not.Null); + Assert.That(options.Tls!.UseTls, Is.True); + Assert.That(options.Tls.ValidateServerCertificate, Is.False); + Assert.That(options.Tls.CaCertificatePath, Is.EqualTo("ca.pem")); + Assert.That(options.Tls.ClientCertificatePath, Is.EqualTo("client.pem")); + Assert.That(options.Tls.ClientKeyPath, Is.EqualTo("client.key")); + } + + [Test] + public async Task AddKafkaTransportWithExplicitSectionBindsOptionsAsync() + { + var services = new ServiceCollection(); + IConfigurationRoot configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Custom:Endpoint"] = "kafka://broker.example.com:9092", + ["Custom:Topics:Prefix"] = "custom" + }) + .Build(); + + services.AddOpcUa().AddPubSub(pubsub => + pubsub.AddKafkaTransport(configuration.GetSection("Custom"))); + + await using ServiceProvider serviceProvider = services.BuildServiceProvider(); + KafkaConnectionOptions options = serviceProvider + .GetRequiredService>() + .Value; + + Assert.That(options.Endpoint, Is.EqualTo("kafka://broker.example.com:9092")); + Assert.That(options.Topics.Prefix, Is.EqualTo("custom")); + } + } +} diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/Opc.Ua.PubSub.Kafka.Tests.csproj b/Tests/Opc.Ua.PubSub.Kafka.Tests/Opc.Ua.PubSub.Kafka.Tests.csproj new file mode 100644 index 0000000000..e3d2da5e62 --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/Opc.Ua.PubSub.Kafka.Tests.csproj @@ -0,0 +1,42 @@ + + + Exe + $(TestsTargetFrameworks) + Opc.Ua.PubSub.Kafka.Tests + enable + false + $(NoWarn);CS1591;CA2007;CA2000;CA1014 + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + From 97a0ec45e93d8dfd2762bb1983b26530df8516fe Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Fri, 3 Jul 2026 14:36:47 +0200 Subject: [PATCH 08/21] Wire active/standby HA activation into the PubSub runtime + failover tests (#3942, #3924) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consume IPubSubActivationCoordinator in the runtime so redundant publishers/subscribers fail over (Part 14 Sec 9.1.6): PubSubApplication starts the coordinator before enabling connections and stops it after disabling them; WriterGroup and ReaderGroup consult the coordinator per component id (pubsub:writergroup:: / pubsub:readergroup:...) and gate at the enable/pause boundary — Standby pauses publishing/dispatch, Active resumes, driven by RoleChanged. Strictly no-op under the default AlwaysActiveCoordinator, so non-redundant behaviour is unchanged. Kafka: AtLeastOnce delivery guarantee now also enables idempotent producing (alongside ExactlyOnce); GroupId already drives Kafka-native consumer-group subscriber failover. Tests: new PubSubActivationFailoverTests (leader election + failover, fencing-token monotonicity, writer-group standby pause/resume, reader-group standby suppression) over the in-memory lease store with a FakeTimeProvider. Opc.Ua.PubSub.Tests 1134 passed/0 failed (1130 existing + 4 new); Kafka tests 82 passed; net10/net48 build 0 warnings. --- .../KafkaQualityOfService.cs | 2 +- .../Application/PubSubApplication.cs | 47 +- .../Connections/PubSubConnection.cs | 38 +- .../OpcUaPubSubBuilderExtensions.cs | 4 +- Libraries/Opc.Ua.PubSub/Groups/ReaderGroup.cs | 125 ++++- Libraries/Opc.Ua.PubSub/Groups/WriterGroup.cs | 124 ++++- .../KafkaQosMappingTests.cs | 2 +- .../PubSubActivationFailoverTests.cs | 439 ++++++++++++++++++ 8 files changed, 767 insertions(+), 14 deletions(-) create mode 100644 Tests/Opc.Ua.PubSub.Tests/Redundancy/PubSubActivationFailoverTests.cs diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaQualityOfService.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaQualityOfService.cs index 1fdbbdd07e..aae53f84c3 100644 --- a/Libraries/Opc.Ua.PubSub.Kafka/KafkaQualityOfService.cs +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaQualityOfService.cs @@ -149,7 +149,7 @@ public static KafkaDeliveryGuarantee ToDeliveryGuarantee( KafkaQualityOfService.AtMostOnce => new KafkaDeliveryGuarantee(KafkaAcks.Leader, EnableIdempotence: false), KafkaQualityOfService.AtLeastOnce => - new KafkaDeliveryGuarantee(KafkaAcks.All, EnableIdempotence: false), + new KafkaDeliveryGuarantee(KafkaAcks.All, EnableIdempotence: true), KafkaQualityOfService.ExactlyOnce => new KafkaDeliveryGuarantee(KafkaAcks.All, EnableIdempotence: true), _ => throw new ArgumentOutOfRangeException(nameof(qualityOfService)) diff --git a/Libraries/Opc.Ua.PubSub/Application/PubSubApplication.cs b/Libraries/Opc.Ua.PubSub/Application/PubSubApplication.cs index d76a4d58f1..483b21f2de 100644 --- a/Libraries/Opc.Ua.PubSub/Application/PubSubApplication.cs +++ b/Libraries/Opc.Ua.PubSub/Application/PubSubApplication.cs @@ -40,6 +40,7 @@ using Opc.Ua.PubSub.Encoding; using Opc.Ua.PubSub.Groups; using Opc.Ua.PubSub.MetaData; +using Opc.Ua.PubSub.Redundancy; using Opc.Ua.PubSub.Scheduling; using Opc.Ua.PubSub.Security; using Opc.Ua.PubSub.StateMachine; @@ -74,6 +75,7 @@ public sealed class PubSubApplication : IPubSubApplication private readonly INetworkMessageDecoder[] m_decoderArray; private readonly IPubSubSecurityPolicy[] m_securityPolicies; private readonly IPubSubScheduler m_scheduler; + private readonly IPubSubActivationCoordinator m_activationCoordinator; private readonly TimeProvider m_timeProvider; private readonly IReadOnlyDictionary? m_publishedDataSetSources; @@ -134,6 +136,7 @@ private readonly Dictionary m_connectionNodeIdsByName /// Optional per-connection maximum message size resolver. /// Optional external configuration store. /// Optional external runtime-state store. + /// Optional high-availability activation coordinator. public PubSubApplication( PubSubConfigurationSnapshot snapshot, IEnumerable transportFactories, @@ -150,7 +153,8 @@ public PubSubApplication( IPubSubSecurityWrapperResolver? securityWrapperResolver = null, Func? maxNetworkMessageSizeResolver = null, IPubSubConfigurationStore? configurationStore = null, - IPubSubRuntimeStateStore? runtimeStateStore = null) + IPubSubRuntimeStateStore? runtimeStateStore = null, + IPubSubActivationCoordinator? activationCoordinator = null) : this( snapshot, transportFactories, @@ -169,7 +173,8 @@ public PubSubApplication( configurationStore, runtimeStateStore, dataSetSourceProvider: null, - dataSetSinkProvider: null) + dataSetSinkProvider: null, + activationCoordinator) { } @@ -206,6 +211,7 @@ public PubSubApplication( /// Optional per-connection maximum message size resolver. /// Optional external configuration store. /// Optional external runtime-state store. + /// Optional high-availability activation coordinator. public PubSubApplication( PubSubConfigurationSnapshot snapshot, IEnumerable transportFactories, @@ -224,7 +230,8 @@ public PubSubApplication( IPubSubSecurityWrapperResolver? securityWrapperResolver = null, Func? maxNetworkMessageSizeResolver = null, IPubSubConfigurationStore? configurationStore = null, - IPubSubRuntimeStateStore? runtimeStateStore = null) + IPubSubRuntimeStateStore? runtimeStateStore = null, + IPubSubActivationCoordinator? activationCoordinator = null) : this( snapshot, transportFactories, @@ -243,7 +250,8 @@ public PubSubApplication( configurationStore, runtimeStateStore, dataSetSourceProvider, - dataSetSinkProvider) + dataSetSinkProvider, + activationCoordinator) { } @@ -265,7 +273,8 @@ private PubSubApplication( IPubSubConfigurationStore? configurationStore = null, IPubSubRuntimeStateStore? runtimeStateStore = null, IDataSetSourceProvider? dataSetSourceProvider = null, - IDataSetSinkProvider? dataSetSinkProvider = null) + IDataSetSinkProvider? dataSetSinkProvider = null, + IPubSubActivationCoordinator? activationCoordinator = null) { if (snapshot is null) { @@ -312,6 +321,7 @@ private PubSubApplication( m_decoderArray = decoders.ToArray(); m_securityPolicies = securityPolicies.ToArray(); m_scheduler = scheduler; + m_activationCoordinator = activationCoordinator ?? AlwaysActiveCoordinator.Instance; m_timeProvider = timeProvider; m_publishedDataSetSources = publishedDataSetSources; m_subscribedDataSetSinks = subscribedDataSetSinks; @@ -531,7 +541,8 @@ public void SetAddressSpaceNamespaceIndex(ushort namespaceIndex) securityContext?.WrapOptions ?? UadpSecurityWrapOptions.SignAndEncrypt, maxMessageSize, requiredSecurityMode, - m_scheduler); + m_scheduler, + m_activationCoordinator); lock (m_gate) { for (int i = 0; i < m_actionHandlers.Count; i++) @@ -598,6 +609,18 @@ public async ValueTask StartAsync(CancellationToken cancellationToken = default) m_started = true; connections = [.. m_connections]; } + try + { + await m_activationCoordinator.StartAsync(cancellationToken).ConfigureAwait(false); + } + catch + { + lock (m_gate) + { + m_started = false; + } + throw; + } _ = State.TryEnable(); foreach (PubSubConnection connection in connections) { @@ -638,6 +661,10 @@ public async ValueTask StartAsync(CancellationToken cancellationToken = default) { _ = State.TryResumeCascade(); } + foreach (PubSubConnection connection in connections) + { + await connection.ApplyActivationRolesAsync(cancellationToken).ConfigureAwait(false); + } } /// @@ -682,6 +709,14 @@ await connections[i].DisableAsync(cancellationToken) } } _ = State.TryDisable(); + try + { + await m_activationCoordinator.StopAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + m_logger.LogError(ex, "Failed to stop PubSub activation coordinator."); + } } /// diff --git a/Libraries/Opc.Ua.PubSub/Connections/PubSubConnection.cs b/Libraries/Opc.Ua.PubSub/Connections/PubSubConnection.cs index 79ef5ed671..acfffb234d 100644 --- a/Libraries/Opc.Ua.PubSub/Connections/PubSubConnection.cs +++ b/Libraries/Opc.Ua.PubSub/Connections/PubSubConnection.cs @@ -42,6 +42,7 @@ using Opc.Ua.PubSub.Encoding.Uadp; using Opc.Ua.PubSub.Groups; using Opc.Ua.PubSub.MetaData; +using Opc.Ua.PubSub.Redundancy; using Opc.Ua.PubSub.Scheduling; using Opc.Ua.PubSub.Security; using Opc.Ua.PubSub.StateMachine; @@ -74,6 +75,7 @@ public sealed class PubSubConnection : IPubSubConnection, IAsyncDisposable private readonly IPubSubScheduler m_scheduler; private readonly IDataSetMetaDataRegistry m_metaDataRegistry; private readonly IPubSubDiagnostics m_diagnostics; + private readonly IPubSubActivationCoordinator m_activationCoordinator; private readonly UadpSecurityWrapper? m_securityWrapper; private readonly UadpSecurityWrapOptions m_securityWrapOptions; private readonly MessageSecurityMode m_requiredSecurityMode; @@ -171,6 +173,7 @@ public PubSubConnection( /// /// Optional scheduler used for periodic discovery announcements. /// + /// Optional high-availability activation coordinator. public PubSubConnection( PubSubConnectionDataType configuration, IPubSubTransportFactory transportFactory, @@ -186,7 +189,8 @@ public PubSubConnection( UadpSecurityWrapOptions securityWrapOptions, int maxNetworkMessageSize = 0, MessageSecurityMode requiredSecurityMode = MessageSecurityMode.None, - IPubSubScheduler? scheduler = null) + IPubSubScheduler? scheduler = null, + IPubSubActivationCoordinator? activationCoordinator = null) { if (configuration is null) { @@ -226,6 +230,7 @@ public PubSubConnection( m_readerGroupViews = readerGroups.ToArrayOf(static group => group); m_metaDataRegistry = metaDataRegistry; m_diagnostics = diagnostics; + m_activationCoordinator = activationCoordinator ?? AlwaysActiveCoordinator.Instance; m_telemetry = telemetry; m_timeProvider = timeProvider; m_scheduler = scheduler ?? new PubSubScheduler(telemetry, timeProvider); @@ -247,6 +252,9 @@ public PubSubConnection( foreach (WriterGroup wg in m_writerGroups) { State.AttachChild(wg.State); + wg.ConfigureActivationCoordinator( + BuildWriterGroupComponentId(Name, wg.Name), + m_activationCoordinator); wg.EncodingProfileOverride = ResolveEncoderProfile(); wg.PubSubAddressing = new WriterGroup.PublisherIdHolder { @@ -258,6 +266,9 @@ public PubSubConnection( foreach (ReaderGroup rg in m_readerGroups) { State.AttachChild(rg.State); + rg.ConfigureActivationCoordinator( + BuildReaderGroupComponentId(Name, rg.Name), + m_activationCoordinator); } } @@ -451,6 +462,31 @@ public async ValueTask EnableAsync(CancellationToken cancellationToken = default WriterGroup wg = m_writerGroups[i]; await wg.EnableAsync(cancellationToken).ConfigureAwait(false); } + await ApplyActivationRolesAsync(cancellationToken).ConfigureAwait(false); + } + + internal async ValueTask ApplyActivationRolesAsync(CancellationToken cancellationToken = default) + { + for (int i = 0; i < m_readerGroups.Count; i++) + { + ReaderGroup rg = m_readerGroups[i]; + await rg.ApplyActivationRoleAsync(cancellationToken).ConfigureAwait(false); + } + for (int i = 0; i < m_writerGroups.Count; i++) + { + WriterGroup wg = m_writerGroups[i]; + await wg.ApplyActivationRoleAsync(cancellationToken).ConfigureAwait(false); + } + } + + private static string BuildWriterGroupComponentId(string connectionName, string groupName) + { + return string.Concat("pubsub:writergroup:", connectionName, ":", groupName); + } + + private static string BuildReaderGroupComponentId(string connectionName, string groupName) + { + return string.Concat("pubsub:readergroup:", connectionName, ":", groupName); } /// diff --git a/Libraries/Opc.Ua.PubSub/DependencyInjection/OpcUaPubSubBuilderExtensions.cs b/Libraries/Opc.Ua.PubSub/DependencyInjection/OpcUaPubSubBuilderExtensions.cs index 00f0d21a4c..2c3b26e77e 100644 --- a/Libraries/Opc.Ua.PubSub/DependencyInjection/OpcUaPubSubBuilderExtensions.cs +++ b/Libraries/Opc.Ua.PubSub/DependencyInjection/OpcUaPubSubBuilderExtensions.cs @@ -302,7 +302,9 @@ private static void RegisterCoreServices(IServiceCollection services) securityWrapperResolver: sp.GetRequiredService(), configurationStore: store, - runtimeStateStore: sp.GetRequiredService()); + runtimeStateStore: sp.GetRequiredService(), + activationCoordinator: + sp.GetRequiredService()); }); services.AddSingleton(); diff --git a/Libraries/Opc.Ua.PubSub/Groups/ReaderGroup.cs b/Libraries/Opc.Ua.PubSub/Groups/ReaderGroup.cs index c65c3f70c1..df2a0cfaed 100644 --- a/Libraries/Opc.Ua.PubSub/Groups/ReaderGroup.cs +++ b/Libraries/Opc.Ua.PubSub/Groups/ReaderGroup.cs @@ -35,6 +35,7 @@ using Opc.Ua.PubSub.Diagnostics; using Opc.Ua.PubSub.Encoding; using Opc.Ua.PubSub.Scheduling; +using Opc.Ua.PubSub.Redundancy; using Opc.Ua.PubSub.StateMachine; namespace Opc.Ua.PubSub.Groups @@ -58,6 +59,10 @@ public sealed class ReaderGroup : IReaderGroup, IAsyncDisposable private readonly IPubSubScheduler? m_scheduler; private readonly IPubSubDiagnostics? m_diagnostics; private readonly ITelemetryContext m_telemetry; + private readonly System.Threading.Lock m_gate = new(); + private IPubSubActivationCoordinator m_activationCoordinator = AlwaysActiveCoordinator.Instance; + private string m_componentId = string.Empty; + private bool m_roleChangedSubscribed; private DataSetReaderTimeoutWatcher? m_timeoutWatcher; /// @@ -91,12 +96,16 @@ public ReaderGroup( /// Diagnostics sink for receive-timeout counter increments. When /// no counters are emitted. /// + /// Optional high-availability activation coordinator. + /// Deterministic redundancy component id. public ReaderGroup( ReaderGroupDataType configuration, ArrayOf readers, ITelemetryContext telemetry, IPubSubScheduler? scheduler, - IPubSubDiagnostics? diagnostics) + IPubSubDiagnostics? diagnostics, + IPubSubActivationCoordinator? activationCoordinator = null, + string? componentId = null) { if (configuration is null) { @@ -110,6 +119,9 @@ public ReaderGroup( m_readers = readers; m_dataSetReaders = readers.ToArrayOf(static reader => reader); Name = configuration.Name ?? string.Empty; + ConfigureActivationCoordinator( + componentId ?? string.Concat("pubsub:readergroup::", Name), + activationCoordinator); m_telemetry = telemetry; m_scheduler = scheduler; m_diagnostics = diagnostics; @@ -150,7 +162,7 @@ public async ValueTask DispatchAsync( { throw new ArgumentNullException(nameof(networkMessage)); } - if (State.State == PubSubState.Disabled) + if (State.State != PubSubState.Operational) { return; } @@ -200,6 +212,8 @@ public async ValueTask EnableAsync(CancellationToken cancellationToken = default _ = State.TryResumeCascade(); } } + SubscribeRoleChanges(); + await ApplyActivationRoleAsync(cancellationToken).ConfigureAwait(false); if (m_scheduler is not null && m_diagnostics is not null && m_timeoutWatcher is null) { m_timeoutWatcher = new DataSetReaderTimeoutWatcher( @@ -211,12 +225,119 @@ public async ValueTask EnableAsync(CancellationToken cancellationToken = default } } + + internal void ConfigureActivationCoordinator( + string componentId, + IPubSubActivationCoordinator? activationCoordinator) + { + if (string.IsNullOrEmpty(componentId)) + { + throw new ArgumentException("componentId is required.", nameof(componentId)); + } + + IPubSubActivationCoordinator previous; + bool unsubscribe; + lock (m_gate) + { + previous = m_activationCoordinator; + unsubscribe = m_roleChangedSubscribed; + m_activationCoordinator = activationCoordinator ?? AlwaysActiveCoordinator.Instance; + m_componentId = componentId; + m_roleChangedSubscribed = false; + } + if (unsubscribe) + { + previous.RoleChanged -= OnRoleChanged; + SubscribeRoleChanges(); + } + } + + internal async ValueTask ApplyActivationRoleAsync(CancellationToken cancellationToken = default) + { + IPubSubActivationCoordinator coordinator; + string componentId; + lock (m_gate) + { + coordinator = m_activationCoordinator; + componentId = m_componentId; + } + + PubSubComponentRole role = await coordinator.GetRoleAsync(componentId, cancellationToken) + .ConfigureAwait(false); + ApplyActivationRole(role); + } + + private void SubscribeRoleChanges() + { + IPubSubActivationCoordinator coordinator; + lock (m_gate) + { + if (m_roleChangedSubscribed) + { + return; + } + + coordinator = m_activationCoordinator; + m_roleChangedSubscribed = true; + } + coordinator.RoleChanged += OnRoleChanged; + } + + private void UnsubscribeRoleChanges() + { + IPubSubActivationCoordinator coordinator; + lock (m_gate) + { + if (!m_roleChangedSubscribed) + { + return; + } + + coordinator = m_activationCoordinator; + m_roleChangedSubscribed = false; + } + coordinator.RoleChanged -= OnRoleChanged; + } + + private void OnRoleChanged(object? sender, PubSubRoleChangedEventArgs e) + { + if (!string.Equals(e.ComponentId, m_componentId, StringComparison.Ordinal)) + { + return; + } + + ApplyActivationRole(e.Role); + } + + private void ApplyActivationRole(PubSubComponentRole role) + { + if (role == PubSubComponentRole.Standby) + { + _ = State.TryPause(PubSubStateTransitionReason.ByParent); + return; + } + + if (State.State == PubSubState.Paused) + { + _ = State.TryResume(PubSubStateTransitionReason.ByParent); + } + if (State.State == PubSubState.PreOperational) + { + _ = State.TryMarkOperational(PubSubStateTransitionReason.ByParent); + } + if (State.State == PubSubState.Operational) + { + _ = State.TryResumeCascade(); + } + } + /// /// Disables the reader group and every child reader. /// public async ValueTask DisableAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + UnsubscribeRoleChanges(); DataSetReaderTimeoutWatcher? watcher = m_timeoutWatcher; m_timeoutWatcher = null; if (watcher is not null) diff --git a/Libraries/Opc.Ua.PubSub/Groups/WriterGroup.cs b/Libraries/Opc.Ua.PubSub/Groups/WriterGroup.cs index 60b0d3677a..a9ff99b94f 100644 --- a/Libraries/Opc.Ua.PubSub/Groups/WriterGroup.cs +++ b/Libraries/Opc.Ua.PubSub/Groups/WriterGroup.cs @@ -35,6 +35,7 @@ using Opc.Ua.PubSub.DataSets; using Opc.Ua.PubSub.Encoding; using Opc.Ua.PubSub.Scheduling; +using Opc.Ua.PubSub.Redundancy; using Opc.Ua.PubSub.StateMachine; using JsonDataSetMessageV2 = Opc.Ua.PubSub.Encoding.Json.JsonDataSetMessage; using JsonNetworkMessageV2 = Opc.Ua.PubSub.Encoding.Json.JsonNetworkMessage; @@ -65,6 +66,9 @@ public sealed class WriterGroup : IWriterGroup, IAsyncDisposable private readonly TimeProvider m_timeProvider; private readonly Dictionary m_writerState; private readonly System.Threading.Lock m_gate = new(); + private IPubSubActivationCoordinator m_activationCoordinator = AlwaysActiveCoordinator.Instance; + private string m_componentId = string.Empty; + private bool m_roleChangedSubscribed; private IAsyncDisposable? m_schedule; private long m_lastPublishedTicks; private bool m_disposed; @@ -78,13 +82,17 @@ public sealed class WriterGroup : IWriterGroup, IAsyncDisposable /// Scheduler used to drive the publish loop. /// Telemetry context. /// Clock. + /// Optional high-availability activation coordinator. + /// Deterministic redundancy component id. public WriterGroup( WriterGroupDataType configuration, ArrayOf writers, PubSubSchedule schedule, IPubSubScheduler scheduler, ITelemetryContext telemetry, - TimeProvider timeProvider) + TimeProvider timeProvider, + IPubSubActivationCoordinator? activationCoordinator = null, + string? componentId = null) { if (configuration is null) { @@ -106,6 +114,9 @@ public WriterGroup( m_timeProvider = timeProvider; WriterGroupId = configuration.WriterGroupId; Name = configuration.Name ?? string.Empty; + ConfigureActivationCoordinator( + componentId ?? string.Concat("pubsub:writergroup::", Name), + activationCoordinator); m_logger = telemetry.CreateLogger(); State = new PubSubStateMachine( string.IsNullOrEmpty(Name) ? $"group-{WriterGroupId}" : Name, @@ -167,6 +178,8 @@ public async ValueTask EnableAsync(CancellationToken cancellationToken = default { _ = State.TryResumeCascade(); } + SubscribeRoleChanges(); + await ApplyActivationRoleAsync(cancellationToken).ConfigureAwait(false); m_schedule = await m_scheduler.ScheduleAsync( Schedule, PublishOnceAsync, @@ -180,6 +193,7 @@ public async ValueTask EnableAsync(CancellationToken cancellationToken = default public async ValueTask DisableAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + UnsubscribeRoleChanges(); IAsyncDisposable? schedule; lock (m_gate) { @@ -204,7 +218,7 @@ public async ValueTask PublishOnceAsync(CancellationToken cancellationToken = de { return; } - if (State.State == PubSubState.Disabled) + if (State.State != PubSubState.Operational) { return; } @@ -262,6 +276,112 @@ await PublishSink(networkMessage, cancellationToken) } } + + internal void ConfigureActivationCoordinator( + string componentId, + IPubSubActivationCoordinator? activationCoordinator) + { + if (string.IsNullOrEmpty(componentId)) + { + throw new ArgumentException("componentId is required.", nameof(componentId)); + } + + IPubSubActivationCoordinator previous; + bool unsubscribe; + lock (m_gate) + { + previous = m_activationCoordinator; + unsubscribe = m_roleChangedSubscribed; + m_activationCoordinator = activationCoordinator ?? AlwaysActiveCoordinator.Instance; + m_componentId = componentId; + m_roleChangedSubscribed = false; + } + if (unsubscribe) + { + previous.RoleChanged -= OnRoleChanged; + SubscribeRoleChanges(); + } + } + + internal async ValueTask ApplyActivationRoleAsync(CancellationToken cancellationToken = default) + { + IPubSubActivationCoordinator coordinator; + string componentId; + lock (m_gate) + { + coordinator = m_activationCoordinator; + componentId = m_componentId; + } + + PubSubComponentRole role = await coordinator.GetRoleAsync(componentId, cancellationToken) + .ConfigureAwait(false); + ApplyActivationRole(role); + } + + private void SubscribeRoleChanges() + { + IPubSubActivationCoordinator coordinator; + lock (m_gate) + { + if (m_roleChangedSubscribed) + { + return; + } + + coordinator = m_activationCoordinator; + m_roleChangedSubscribed = true; + } + coordinator.RoleChanged += OnRoleChanged; + } + + private void UnsubscribeRoleChanges() + { + IPubSubActivationCoordinator coordinator; + lock (m_gate) + { + if (!m_roleChangedSubscribed) + { + return; + } + + coordinator = m_activationCoordinator; + m_roleChangedSubscribed = false; + } + coordinator.RoleChanged -= OnRoleChanged; + } + + private void OnRoleChanged(object? sender, PubSubRoleChangedEventArgs e) + { + if (!string.Equals(e.ComponentId, m_componentId, StringComparison.Ordinal)) + { + return; + } + + ApplyActivationRole(e.Role); + } + + private void ApplyActivationRole(PubSubComponentRole role) + { + if (role == PubSubComponentRole.Standby) + { + _ = State.TryPause(PubSubStateTransitionReason.ByParent); + return; + } + + if (State.State == PubSubState.Paused) + { + _ = State.TryResume(PubSubStateTransitionReason.ByParent); + } + if (State.State == PubSubState.PreOperational) + { + _ = State.TryMarkOperational(PubSubStateTransitionReason.ByParent); + } + if (State.State == PubSubState.Operational) + { + _ = State.TryResumeCascade(); + } + } + private async ValueTask BuildDataSetMessageAsync( DataSetWriter writer, CancellationToken cancellationToken) diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaQosMappingTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaQosMappingTests.cs index 7c79ceab17..4c383189ce 100644 --- a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaQosMappingTests.cs +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaQosMappingTests.cs @@ -43,7 +43,7 @@ public sealed class KafkaQosMappingTests [Test] [TestCase(KafkaQualityOfService.BestEffort, KafkaAcks.None, false)] [TestCase(KafkaQualityOfService.AtMostOnce, KafkaAcks.Leader, false)] - [TestCase(KafkaQualityOfService.AtLeastOnce, KafkaAcks.All, false)] + [TestCase(KafkaQualityOfService.AtLeastOnce, KafkaAcks.All, true)] [TestCase(KafkaQualityOfService.ExactlyOnce, KafkaAcks.All, true)] public void QualityOfServiceMapsToKafkaDeliveryGuarantee( KafkaQualityOfService qos, diff --git a/Tests/Opc.Ua.PubSub.Tests/Redundancy/PubSubActivationFailoverTests.cs b/Tests/Opc.Ua.PubSub.Tests/Redundancy/PubSubActivationFailoverTests.cs new file mode 100644 index 0000000000..4ec8cde9e7 --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Tests/Redundancy/PubSubActivationFailoverTests.cs @@ -0,0 +1,439 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Moq; +using NUnit.Framework; +using Opc.Ua.PubSub.DataSets; +using Opc.Ua.PubSub.Encoding; +using Opc.Ua.PubSub.Encoding.Uadp; +using Opc.Ua.PubSub.Groups; +using Opc.Ua.PubSub.Redundancy; +using Opc.Ua.PubSub.Scheduling; +using Opc.Ua.PubSub.StateMachine; +using Opc.Ua.Tests; + +namespace Opc.Ua.PubSub.Tests.Redundancy +{ + /// + /// Covers the Part 14 §9.1.6 activation coordinator failover wiring used + /// by redundant PubSub publishers and subscribers. + /// + [TestFixture] + [TestSpec("9.1.6", Summary = "PubSub HA activation coordinator failover")] + public class PubSubActivationFailoverTests + { + private const string WriterComponentId = "pubsub:writergroup:conn:wg"; + private const string ReaderComponentId = "pubsub:readergroup:conn:rg"; + + [Test] + [TestSpec("9.1.6")] + public async Task LeaseActivationCoordinator_ElectsSingleActiveAndFailsOverOnStopAsync() + { + var clock = new FakeTimeProvider( + new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var store = new InMemoryPubSubLeaseStore(clock); + TimeSpan ttl = TimeSpan.FromSeconds(9); + TimeSpan interval = TimeSpan.FromSeconds(1); + var firstEvents = new List(); + var secondEvents = new List(); + + await using var first = new LeaseActivationCoordinator( + store, + NUnitTelemetryContext.Create(), + ownerId: "publisher-a", + leaseDuration: ttl, + renewInterval: interval, + retryInterval: interval, + timeProvider: clock); + await using var second = new LeaseActivationCoordinator( + store, + NUnitTelemetryContext.Create(), + ownerId: "publisher-b", + leaseDuration: ttl, + renewInterval: interval, + retryInterval: interval, + timeProvider: clock); + first.RoleChanged += (_, e) => firstEvents.Add(e); + second.RoleChanged += (_, e) => secondEvents.Add(e); + + await first.StartAsync().ConfigureAwait(false); + await second.StartAsync().ConfigureAwait(false); + _ = await first.GetRoleAsync(WriterComponentId).ConfigureAwait(false); + _ = await second.GetRoleAsync(WriterComponentId).ConfigureAwait(false); + + await WaitForRolesAsync( + clock, + first, + second, + activeCount: 1).ConfigureAwait(false); + + PubSubComponentRole firstRole = await first.GetRoleAsync(WriterComponentId).ConfigureAwait(false); + LeaseActivationCoordinator active = firstRole == PubSubComponentRole.Active ? first : second; + LeaseActivationCoordinator standby = ReferenceEquals(active, first) ? second : first; + List standbyEvents = ReferenceEquals(standby, first) + ? firstEvents + : secondEvents; + + await active.StopAsync().ConfigureAwait(false); + clock.Advance(interval); + + await WaitForRoleAsync( + clock, + standby, + PubSubComponentRole.Active).ConfigureAwait(false); + + Assert.That( + standbyEvents, + Has.Some.Matches(e => + string.Equals(e.ComponentId, WriterComponentId, StringComparison.Ordinal) + && e.Role == PubSubComponentRole.Active)); + } + + [Test] + [TestSpec("9.1.6")] + public async Task InMemoryLeaseStore_IncrementsFencingTokenOnOwnershipChangeAsync() + { + var clock = new FakeTimeProvider( + new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var store = new InMemoryPubSubLeaseStore(clock); + TimeSpan ttl = TimeSpan.FromSeconds(30); + + PubSubLease? firstLease = await store.TryAcquireAsync( + WriterComponentId, + "owner-a", + ttl).ConfigureAwait(false); + Assert.That(firstLease, Is.Not.Null); + PubSubLease first = firstLease.GetValueOrDefault(); + + PubSubLease? blockedLease = await store.TryAcquireAsync( + WriterComponentId, + "owner-b", + ttl).ConfigureAwait(false); + Assert.That(blockedLease, Is.Null); + + await store.ReleaseAsync(first).ConfigureAwait(false); + PubSubLease? secondLease = await store.TryAcquireAsync( + WriterComponentId, + "owner-b", + ttl).ConfigureAwait(false); + + Assert.That(secondLease, Is.Not.Null); + PubSubLease second = secondLease.GetValueOrDefault(); + Assert.That(second.FencingToken, Is.GreaterThan(first.FencingToken)); + } + + [Test] + [TestSpec("9.1.6")] + public async Task WriterGroup_StandbyRolePausesPublishingUntilActiveAsync() + { + PubSubComponentRole role = PubSubComponentRole.Standby; + Mock coordinator = CreateCoordinatorMock( + WriterComponentId, + () => role); + var captured = new List(); + WriterGroup group = CreateWriterGroup(coordinator.Object, captured); + + await group.EnableAsync().ConfigureAwait(false); + await group.PublishOnceAsync().ConfigureAwait(false); + + Assert.Multiple(() => + { + Assert.That(group.State.State, Is.EqualTo(PubSubState.Paused)); + Assert.That(captured, Is.Empty); + }); + + role = PubSubComponentRole.Active; + coordinator.Raise( + c => c.RoleChanged += null!, + new PubSubRoleChangedEventArgs(WriterComponentId, PubSubComponentRole.Active)); + await group.PublishOnceAsync().ConfigureAwait(false); + + Assert.Multiple(() => + { + Assert.That(group.State.State, Is.EqualTo(PubSubState.Operational)); + Assert.That(captured, Has.Count.EqualTo(1)); + }); + } + + [Test] + [TestSpec("9.1.6")] + public async Task ReaderGroup_StandbyRoleSuppressesDispatchUntilActiveAsync() + { + PubSubComponentRole role = PubSubComponentRole.Standby; + Mock coordinator = CreateCoordinatorMock( + ReaderComponentId, + () => role); + var sink = new CountingSink(); + ReaderGroup group = CreateReaderGroup(coordinator.Object, sink); + PubSubNetworkMessage message = CreateNetworkMessage(); + + await group.EnableAsync().ConfigureAwait(false); + await group.DispatchAsync(message).ConfigureAwait(false); + + Assert.Multiple(() => + { + Assert.That(group.State.State, Is.EqualTo(PubSubState.Paused)); + Assert.That(sink.CallCount, Is.Zero); + }); + + role = PubSubComponentRole.Active; + coordinator.Raise( + c => c.RoleChanged += null!, + new PubSubRoleChangedEventArgs(ReaderComponentId, PubSubComponentRole.Active)); + await group.DispatchAsync(message).ConfigureAwait(false); + + Assert.Multiple(() => + { + Assert.That(group.State.State, Is.EqualTo(PubSubState.Operational)); + Assert.That(sink.CallCount, Is.EqualTo(1)); + }); + } + + private static Mock CreateCoordinatorMock( + string componentId, + Func role) + { + var coordinator = new Mock(MockBehavior.Strict); + coordinator + .Setup(c => c.GetRoleAsync(componentId, It.IsAny())) + .Returns(() => new ValueTask(role())); + coordinator + .Setup(c => c.StartAsync(It.IsAny())) + .Returns(default(ValueTask)); + coordinator + .Setup(c => c.StopAsync(It.IsAny())) + .Returns(default(ValueTask)); + return coordinator; + } + + private static WriterGroup CreateWriterGroup( + IPubSubActivationCoordinator coordinator, + List captured) + { + var dataSetConfig = new PublishedDataSetDataType + { + Name = "pds", + DataSetMetaData = new DataSetMetaDataType + { + Fields = [new FieldMetaData { Name = "value" }] + } + }; + var dataSet = new PublishedDataSet(dataSetConfig, new CountingSource()); + var writer = new DataSetWriter( + new DataSetWriterDataType + { + Name = "writer", + DataSetWriterId = 1, + DataSetName = "pds", + KeyFrameCount = 1 + }, + dataSet, + NUnitTelemetryContext.Create()); + var group = new WriterGroup( + new WriterGroupDataType + { + Name = "wg", + WriterGroupId = 1, + PublishingInterval = 100 + }, + [writer], + new PubSubSchedule( + TimeSpan.FromMilliseconds(100), + TimeSpan.Zero, + TimeSpan.Zero, + TimeSpan.Zero), + NoOpScheduler.Instance, + NUnitTelemetryContext.Create(), + TimeProvider.System, + coordinator, + WriterComponentId) + { + PublishSink = (message, _) => + { + captured.Add(message); + return default; + } + }; + return group; + } + + private static ReaderGroup CreateReaderGroup( + IPubSubActivationCoordinator coordinator, + ISubscribedDataSetSink sink) + { + var reader = new DataSetReader( + new DataSetReaderDataType + { + Name = "reader", + DataSetWriterId = 1 + }, + sink, + NUnitTelemetryContext.Create(), + TimeProvider.System); + return new ReaderGroup( + new ReaderGroupDataType { Name = "rg" }, + [reader], + NUnitTelemetryContext.Create(), + scheduler: null, + diagnostics: null, + coordinator, + ReaderComponentId); + } + + private static UadpNetworkMessage CreateNetworkMessage() + { + return new UadpNetworkMessage + { + DataSetMessages = + [ + new UadpDataSetMessage + { + DataSetWriterId = 1, + Fields = [new DataSetField { Name = "value", Value = new Variant(1) }] + } + ] + }; + } + + private static async Task WaitForRolesAsync( + FakeTimeProvider clock, + LeaseActivationCoordinator first, + LeaseActivationCoordinator second, + int activeCount) + { + await WaitForConditionAsync(clock, async () => + { + int count = 0; + if (await first.GetRoleAsync(WriterComponentId).ConfigureAwait(false) == PubSubComponentRole.Active) + { + count++; + } + if (await second.GetRoleAsync(WriterComponentId).ConfigureAwait(false) == PubSubComponentRole.Active) + { + count++; + } + return count == activeCount; + }).ConfigureAwait(false); + } + + private static async Task WaitForRoleAsync( + FakeTimeProvider clock, + LeaseActivationCoordinator coordinator, + PubSubComponentRole role) + { + await WaitForConditionAsync(clock, async () => + await coordinator.GetRoleAsync(WriterComponentId).ConfigureAwait(false) == role).ConfigureAwait(false); + } + + private static async Task WaitForConditionAsync( + FakeTimeProvider clock, + Func> condition) + { + for (int i = 0; i < 32; i++) + { + if (await condition().ConfigureAwait(false)) + { + return; + } + + await Task.Yield(); + clock.Advance(TimeSpan.FromSeconds(1)); + await Task.Yield(); + } + + Assert.Fail("The expected HA role transition did not occur."); + } + + private sealed class CountingSource : IPublishedDataSetSource + { + private int m_value; + + public DataSetMetaDataType BuildMetaData() + { + return new DataSetMetaDataType + { + Fields = [new FieldMetaData { Name = "value" }] + }; + } + + public ValueTask SampleAsync( + DataSetMetaDataType metaData, + CancellationToken cancellationToken = default) + { + int value = Interlocked.Increment(ref m_value); + return new ValueTask( + new PublishedDataSetSnapshot( + new ConfigurationVersionDataType(), + [new DataSetField { Name = "value", Value = new Variant(value) }], + DateTimeUtc.From(DateTimeOffset.UtcNow))); + } + } + + private sealed class CountingSink : ISubscribedDataSetSink + { + public int CallCount { get; private set; } + + public ValueTask WriteAsync( + IReadOnlyList fields, + CancellationToken cancellationToken = default) + { + CallCount++; + return default; + } + } + + private sealed class NoOpScheduler : IPubSubScheduler + { + public static NoOpScheduler Instance { get; } = new(); + + public ValueTask ScheduleAsync( + PubSubSchedule schedule, + Func action, + CancellationToken cancellationToken = default) + { + return new ValueTask(NoOpHandle.Instance); + } + + private sealed class NoOpHandle : IAsyncDisposable + { + public static NoOpHandle Instance { get; } = new(); + + public ValueTask DisposeAsync() + { + return default; + } + } + } + } +} From 553605e74c5073b3f43bb5308aabe5f147aa8ace Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Fri, 3 Jul 2026 16:14:37 +0200 Subject: [PATCH 09/21] Address PR #3949 Kafka review feedback Inline review comments: - KafkaPubSubTransportFactory: deep-clone Tls/Topics options per connection so the automatic kafkas TLS upgrade no longer mutates the shared default options (added KafkaTlsOptions.Clone / KafkaTopicOptions.Clone). - KafkaQualityOfService: align AtLeastOnce enum XML-doc with the mapping, which enables the idempotent producer (acks=all + enable.idempotence=true). - Docs/PubSubKafka.md: same AtLeastOnce idempotence correction. - NugetREADME: rewrite the NativeAOT note to document the per-TFM client split (net10 uses managed Dekaf = AOT-compatible; other TFMs use Confluent/librdkafka = not AOT-compatible). - IKafkaClientFactory internal member: no change (mirrors the shipped Opc.Ua.PubSub.Mqtt IMqttClientFactory pattern; builds clean on all TFMs). Low-confidence findings also addressed: - InMemoryPubSubLeaseStore.TryRenewAsync now rejects renewal of an expired lease so the owner must reacquire (advancing the fencing token). Added a test. - LeaseActivationCoordinator waits retryInterval after losing a lease before re-acquiring, avoiding a tight retry loop. - WriterGroup/ReaderGroup default componentId no longer emits a double colon (pubsub:writergroup: instead of pubsub:writergroup::). --- Docs/PubSubKafka.md | 2 +- .../KafkaPubSubTransportFactory.cs | 4 +-- .../KafkaQualityOfService.cs | 7 ++-- .../Opc.Ua.PubSub.Kafka/KafkaTlsOptions.cs | 18 ++++++++++ .../Opc.Ua.PubSub.Kafka/KafkaTopicOptions.cs | 13 +++++++ Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md | 12 +++++-- Libraries/Opc.Ua.PubSub/Groups/ReaderGroup.cs | 2 +- Libraries/Opc.Ua.PubSub/Groups/WriterGroup.cs | 2 +- .../Redundancy/InMemoryPubSubLeaseStore.cs | 3 +- .../Redundancy/LeaseActivationCoordinator.cs | 1 + .../PubSubActivationFailoverTests.cs | 36 +++++++++++++++++++ 11 files changed, 88 insertions(+), 12 deletions(-) diff --git a/Docs/PubSubKafka.md b/Docs/PubSubKafka.md index df565cee2a..6c0d1b8581 100644 --- a/Docs/PubSubKafka.md +++ b/Docs/PubSubKafka.md @@ -131,7 +131,7 @@ Kafka topic names should use Kafka-safe characters such as letters, digits, `.`, Every produced Kafka record carries the normative `content-type` record header from Part 14 Annex B.2: `application/opcua+uadp` for `pubsub-kafka-uadp` and `application/json` for `pubsub-kafka-json`. The record key is derived from the PubSubConnection PublisherId when available so records for the same publisher preserve partition ordering. -`KafkaConnectionOptions.DeliveryGuarantee` maps the Part 14 broker QoS intent to Kafka producer settings: `BestEffort` uses `acks=0`, `AtMostOnce` uses `acks=1`, `AtLeastOnce` uses `acks=all` without idempotence, and `ExactlyOnce` uses `acks=all` with producer idempotence enabled. Per-writer `RequestedDeliveryGuarantee` values on broker transport settings override the connection default when specified. +`KafkaConnectionOptions.DeliveryGuarantee` maps the Part 14 broker QoS intent to Kafka producer settings: `BestEffort` uses `acks=0`, `AtMostOnce` uses `acks=1`, `AtLeastOnce` uses `acks=all` with producer idempotence enabled, and `ExactlyOnce` uses `acks=all` with producer idempotence enabled. Per-writer `RequestedDeliveryGuarantee` values on broker transport settings override the connection default when specified. ## SASL and TLS diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaPubSubTransportFactory.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaPubSubTransportFactory.cs index 146d94659b..a85597471f 100644 --- a/Libraries/Opc.Ua.PubSub.Kafka/KafkaPubSubTransportFactory.cs +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaPubSubTransportFactory.cs @@ -212,11 +212,11 @@ private static KafkaConnectionOptions CloneOptionsWithEndpoint( AuthenticationProfileUri = source.AuthenticationProfileUri, ResourceUri = source.ResourceUri, AllowCredentialsOverPlaintext = source.AllowCredentialsOverPlaintext, - Tls = source.Tls, + Tls = source.Tls?.Clone(), DeliveryGuarantee = source.DeliveryGuarantee, AutoOffsetReset = source.AutoOffsetReset, EnableAutoCommit = source.EnableAutoCommit, - Topics = source.Topics, + Topics = source.Topics?.Clone() ?? new KafkaTopicOptions(), ConnectTimeout = source.ConnectTimeout, MessageTimeout = source.MessageTimeout, MaxMessageSize = source.MaxMessageSize diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaQualityOfService.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaQualityOfService.cs index aae53f84c3..095f67fbd4 100644 --- a/Libraries/Opc.Ua.PubSub.Kafka/KafkaQualityOfService.cs +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaQualityOfService.cs @@ -61,9 +61,10 @@ public enum KafkaQualityOfService AtMostOnce, /// - /// The producer waits for all in-sync replicas to acknowledge - /// the write (acks=all) with retries but without - /// idempotence, so duplicates are possible on retry. Maps to + /// The producer waits for all in-sync replicas to acknowledge the + /// write (acks=all) with the idempotent producer enabled + /// (enable.idempotence=true), so retries do not create + /// duplicate records. Maps to /// BrokerTransportQualityOfService.AtLeastOnce. /// AtLeastOnce, diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaTlsOptions.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaTlsOptions.cs index d251407409..3b7af0030a 100644 --- a/Libraries/Opc.Ua.PubSub.Kafka/KafkaTlsOptions.cs +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaTlsOptions.cs @@ -85,5 +85,23 @@ public sealed class KafkaTlsOptions /// librdkafka ssl.key.location property. /// public string? ClientKeyPath { get; set; } + + /// + /// Creates a deep copy of these TLS options so per-connection + /// mutations (for example the automatic kafkas TLS upgrade) + /// never leak back into a shared default options instance. + /// + /// A new, independent . + public KafkaTlsOptions Clone() + { + return new KafkaTlsOptions + { + UseTls = UseTls, + ValidateServerCertificate = ValidateServerCertificate, + CaCertificatePath = CaCertificatePath, + ClientCertificatePath = ClientCertificatePath, + ClientKeyPath = ClientKeyPath + }; + } } } diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaTopicOptions.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaTopicOptions.cs index efb4aa00f9..4d585327dd 100644 --- a/Libraries/Opc.Ua.PubSub.Kafka/KafkaTopicOptions.cs +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaTopicOptions.cs @@ -52,5 +52,18 @@ public sealed class KafkaTopicOptions /// explicit Kafka topic (QueueName). Defaults to opcua. /// public string Prefix { get; set; } = "opcua"; + + /// + /// Creates a deep copy of these topic options so per-connection + /// changes never leak back into a shared default options instance. + /// + /// A new, independent . + public KafkaTopicOptions Clone() + { + return new KafkaTopicOptions + { + Prefix = Prefix + }; + } } } diff --git a/Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md b/Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md index c1cf3b3115..da97ca7abb 100644 --- a/Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md +++ b/Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md @@ -27,9 +27,15 @@ Connection addresses use `kafka://host:9092` (plain/SASL) or `kafkas://host:9093 ## NativeAOT -This transport wraps the native `librdkafka` client (via Confluent.Kafka) and is therefore -**not** NativeAOT/trimming compatible. Use a JIT-compiled host for Kafka deployments; the -other PubSub transports (UDP, Ethernet, MQTT) remain AOT-compatible. +The Kafka client is dual-sourced by target framework: + +- On **net10.0** the transport uses the pure-managed [Dekaf](https://github.com/thomhurst/Dekaf) + client (no native dependency) and is **NativeAOT / trimming compatible**. +- On **net472, net48, netstandard2.1, net8.0, and net9.0** it uses `Confluent.Kafka` + (native `librdkafka`), which is **not** NativeAOT/trimming compatible — use a JIT-compiled + host on those frameworks. + +The other PubSub transports (UDP, Ethernet, MQTT) remain AOT-compatible on all frameworks. ## Additional documentation diff --git a/Libraries/Opc.Ua.PubSub/Groups/ReaderGroup.cs b/Libraries/Opc.Ua.PubSub/Groups/ReaderGroup.cs index df2a0cfaed..98180ff6c0 100644 --- a/Libraries/Opc.Ua.PubSub/Groups/ReaderGroup.cs +++ b/Libraries/Opc.Ua.PubSub/Groups/ReaderGroup.cs @@ -120,7 +120,7 @@ public ReaderGroup( m_dataSetReaders = readers.ToArrayOf(static reader => reader); Name = configuration.Name ?? string.Empty; ConfigureActivationCoordinator( - componentId ?? string.Concat("pubsub:readergroup::", Name), + componentId ?? string.Concat("pubsub:readergroup:", Name), activationCoordinator); m_telemetry = telemetry; m_scheduler = scheduler; diff --git a/Libraries/Opc.Ua.PubSub/Groups/WriterGroup.cs b/Libraries/Opc.Ua.PubSub/Groups/WriterGroup.cs index a9ff99b94f..581e255932 100644 --- a/Libraries/Opc.Ua.PubSub/Groups/WriterGroup.cs +++ b/Libraries/Opc.Ua.PubSub/Groups/WriterGroup.cs @@ -115,7 +115,7 @@ public WriterGroup( WriterGroupId = configuration.WriterGroupId; Name = configuration.Name ?? string.Empty; ConfigureActivationCoordinator( - componentId ?? string.Concat("pubsub:writergroup::", Name), + componentId ?? string.Concat("pubsub:writergroup:", Name), activationCoordinator); m_logger = telemetry.CreateLogger(); State = new PubSubStateMachine( diff --git a/Libraries/Opc.Ua.PubSub/Redundancy/InMemoryPubSubLeaseStore.cs b/Libraries/Opc.Ua.PubSub/Redundancy/InMemoryPubSubLeaseStore.cs index 035a5a123e..4a94b70e08 100644 --- a/Libraries/Opc.Ua.PubSub/Redundancy/InMemoryPubSubLeaseStore.cs +++ b/Libraries/Opc.Ua.PubSub/Redundancy/InMemoryPubSubLeaseStore.cs @@ -107,7 +107,8 @@ public InMemoryPubSubLeaseStore(TimeProvider? timeProvider = null) { if (!m_leases.TryGetValue(lease.LeaseKey, out PubSubLease current) || current.FencingToken != lease.FencingToken - || !string.Equals(current.OwnerId, lease.OwnerId, StringComparison.Ordinal)) + || !string.Equals(current.OwnerId, lease.OwnerId, StringComparison.Ordinal) + || current.ExpiresAt <= now) { return new ValueTask((PubSubLease?)null); } diff --git a/Libraries/Opc.Ua.PubSub/Redundancy/LeaseActivationCoordinator.cs b/Libraries/Opc.Ua.PubSub/Redundancy/LeaseActivationCoordinator.cs index 108340885e..47246a01f3 100644 --- a/Libraries/Opc.Ua.PubSub/Redundancy/LeaseActivationCoordinator.cs +++ b/Libraries/Opc.Ua.PubSub/Redundancy/LeaseActivationCoordinator.cs @@ -280,6 +280,7 @@ private async Task RunAsync() { held = null; SetRole(PubSubComponentRole.Standby); + await DelayAsync(m_owner.m_retryInterval).ConfigureAwait(false); continue; } held = renewed; diff --git a/Tests/Opc.Ua.PubSub.Tests/Redundancy/PubSubActivationFailoverTests.cs b/Tests/Opc.Ua.PubSub.Tests/Redundancy/PubSubActivationFailoverTests.cs index 4ec8cde9e7..989e592048 100644 --- a/Tests/Opc.Ua.PubSub.Tests/Redundancy/PubSubActivationFailoverTests.cs +++ b/Tests/Opc.Ua.PubSub.Tests/Redundancy/PubSubActivationFailoverTests.cs @@ -153,6 +153,42 @@ public async Task InMemoryLeaseStore_IncrementsFencingTokenOnOwnershipChangeAsyn Assert.That(second.FencingToken, Is.GreaterThan(first.FencingToken)); } + [Test] + [TestSpec("9.1.6")] + public async Task InMemoryLeaseStore_RejectsRenewalOfExpiredLeaseAsync() + { + var clock = new FakeTimeProvider( + new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var store = new InMemoryPubSubLeaseStore(clock); + TimeSpan ttl = TimeSpan.FromSeconds(10); + + PubSubLease? acquired = await store.TryAcquireAsync( + WriterComponentId, + "owner-a", + ttl).ConfigureAwait(false); + Assert.That(acquired, Is.Not.Null); + PubSubLease lease = acquired.GetValueOrDefault(); + + PubSubLease? renewedInTime = await store.TryRenewAsync(lease, ttl).ConfigureAwait(false); + Assert.That(renewedInTime, Is.Not.Null); + + clock.Advance(ttl + TimeSpan.FromSeconds(1)); + + PubSubLease? renewedExpired = await store.TryRenewAsync( + renewedInTime.GetValueOrDefault(), + ttl).ConfigureAwait(false); + Assert.That(renewedExpired, Is.Null); + + PubSubLease? reacquired = await store.TryAcquireAsync( + WriterComponentId, + "owner-a", + ttl).ConfigureAwait(false); + Assert.That(reacquired, Is.Not.Null); + Assert.That( + reacquired.GetValueOrDefault().FencingToken, + Is.GreaterThan(lease.FencingToken)); + } + [Test] [TestSpec("9.1.6")] public async Task WriterGroup_StandbyRolePausesPublishingUntilActiveAsync() From 00426db89bfc12b876d84510bdaffc355ffe22b6 Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Fri, 3 Jul 2026 16:44:00 +0200 Subject: [PATCH 10/21] Fold Kafka transport doc into PubSub.md and document PubSub redundancy Addresses PR #3949 review feedback: - Merge Docs/PubSubKafka.md into Docs/PubSub.md as the last transport under "Transports" (Apache Kafka now follows the DTLS transport-status note), condensing the code snippet and pointing at the reference sample for full publisher/subscriber configurations. Delete PubSubKafka.md and repoint the Docs/README.md and NugetREADME.md links to PubSub.md#apache-kafka. - Document PubSub high availability + redundancy in PubSub.md: renamed the HA section, added an "Active/standby activation" subsection covering IPubSubActivationCoordinator, IPubSubLeaseStore, PubSubComponentRole, fencing tokens, AlwaysActiveCoordinator/LeaseActivationCoordinator, and PubSubRedundancyMode. - Align the redundancy docs to #3918's core abstractions (Opc.Ua.Redundancy.ISharedKeyValueStore CompareAndSwapAsync + ILeaderElection, ServiceLevel) and note the intended convergence of the PubSub lease store onto the shared compare-and-swap primitive; cross-reference Docs/HighAvailability.md. - Update the Native AOT section for the Kafka per-TFM client split. --- Docs/PubSub.md | 78 ++++++++-- Docs/PubSubKafka.md | 142 ------------------- Docs/README.md | 2 +- Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md | 4 +- 4 files changed, 73 insertions(+), 153 deletions(-) delete mode 100644 Docs/PubSubKafka.md diff --git a/Docs/PubSub.md b/Docs/PubSub.md index f9ce27f276..8f86c0bd03 100644 --- a/Docs/PubSub.md +++ b/Docs/PubSub.md @@ -21,7 +21,7 @@ - [Security Key Service (SKS)](#security-key-service-sks) - [Server-side address space](#server-side-address-space) - [Binding PubSub to an external OPC UA server (client-session adapters)](#binding-pubsub-to-an-external-opc-ua-server-client-session-adapters) -- [High availability state providers](#high-availability-state-providers) +- [High availability and redundancy](#high-availability-and-redundancy) - [Diagnostics](#diagnostics) - [Native AOT](#native-aot) - [Spec coverage](#spec-coverage) @@ -399,7 +399,7 @@ only makes sense together with the PubSub feature: - `IPubSubBuilder.AddMqttTransport(Action?)` — MQTT 3.1.1 + 5.0 via MQTTnet. - `IPubSubBuilder.AddKafkaTransport(Action?)` — - Apache Kafka broker transport; see [Kafka transport](PubSubKafka.md). + Apache Kafka broker transport; see [Apache Kafka](#apache-kafka). - `IPubSubBuilder.AddEthTransport(Action?)` — Ethernet Layer 2 (`opc.eth://`); chain `.WithPcap()` for the SharpPcap (libpcap / Npcap) backend. @@ -677,16 +677,46 @@ var options = new MqttConnectionOptions }; ``` -### Apache Kafka - -Implemented in `Opc.Ua.PubSub.Kafka`. Wire profiles [`PubSub Kafka UADP`](http://opcfoundation.org/UA-Profile/Transport/pubsub-kafka-uadp) and [`PubSub Kafka JSON`](http://opcfoundation.org/UA-Profile/Transport/pubsub-kafka-json) implement the Part 14 Annex B.2 Kafka broker mapping over `kafka://` and `kafkas://` endpoints. See [Kafka transport](PubSubKafka.md) for DI registration, topic mapping, SASL/TLS configuration, `content-type` record headers, QoS mapping, and NativeAOT notes. - ### DTLS transport status The `opc.dtls://` transport URI is parsed for Part 14 §7.3.2.4 unicast endpoints and wired through the UDP transport factory when `.WithDtls(...)` is registered on the `IUdpTransportBuilder` returned by `AddUdpTransport()`. The DTLS 1.3 handshake is implemented, including ECDHE negotiation, HelloRetryRequest cookies, and certificate authentication. The key schedule/HKDF, AEAD record protection, and anti-replay window are implemented for the registered runtime profiles. The runtime profile registry remains fail-closed: Curve25519 / Curve448 profiles are not registered because the portable .NET BCL does not expose RFC 7748 ECDH APIs, and optional NIST / Brainpool profiles are registered only when the required BCL cipher, HKDF, and ECDH curve probes succeed. +### Apache Kafka + +Implemented in `Opc.Ua.PubSub.Kafka`. Wire profiles [`PubSub Kafka UADP`](http://opcfoundation.org/UA-Profile/Transport/pubsub-kafka-uadp) and [`PubSub Kafka JSON`](http://opcfoundation.org/UA-Profile/Transport/pubsub-kafka-json) implement the [Part 14 Annex B.2](https://reference.opcfoundation.org/specs/OPC-10000-14/v1.05.06/Annex-B.2) Kafka broker mapping. Use `kafka://host[:port][,host...]` for plaintext bootstrap servers and `kafkas://host[:port][,...]` for TLS; the default Kafka port is `9092`. + +Register the transport with `AddKafkaTransport()` (in the `Microsoft.Extensions.DependencyInjection` namespace), which adds both profile factories. Options come from `KafkaConnectionOptions` — supplied by callback or bound from the `OpcUa:PubSub:Kafka` configuration section: + +```csharp +pubsub.AddPublisher() + .AddKafkaTransport(options => + { + options.Endpoint = "kafkas://broker1:9093,broker2:9093"; + options.SecurityProtocol = KafkaSecurityProtocol.SaslSsl; + options.SaslMechanism = KafkaSaslMechanism.ScramSha256; + options.UserName = "pubsub-publisher"; + options.PasswordSecretId = "KafkaPublisherPassword"; // resolved via the secret store + options.DeliveryGuarantee = KafkaQualityOfService.AtLeastOnce; + options.Tls = new KafkaTlsOptions + { + ValidateServerCertificate = true, + CaCertificatePath = "pki/kafka/ca.pem" + }; + }); +``` + +Subscribers set `GroupId` (consumer group) and `AutoOffsetReset` instead of the delivery guarantee. Writer and reader groups use the same fluent `PubSubConfigurationBuilder` shown under [Fluent builder walkthrough](#fluent-builder-walkthrough); the [reference sample](../Applications/ConsoleReferencePubSubClient/README.md) contains complete Kafka publisher and subscriber configurations. + +**Topic mapping.** Kafka topics come from the OPC UA broker transport settings: `BrokerDataSetWriterTransportDataType.QueueName` / `BrokerDataSetReaderTransportDataType.QueueName` select the per-writer/reader data topic, `BrokerWriterGroupTransportDataType.QueueName` is the writer-group fallback, and `MetaDataQueueName` selects the metadata topic. When `MetaDataQueueName` is unset the transport derives a deterministic fallback from `KafkaConnectionOptions.Topics.Prefix`, the encoding, message type, PublisherId, WriterGroupId, and DataSetWriterId (segments joined with `.`). Use Kafka-safe characters (letters, digits, `.`, `_`, `-`). + +**Record headers and delivery guarantees.** Every record carries the normative `content-type` header (Annex B.2): `application/opcua+uadp` for `pubsub-kafka-uadp` and `application/json` for `pubsub-kafka-json`. The record key derives from the PublisherId so a publisher's records keep partition ordering. `KafkaConnectionOptions.DeliveryGuarantee` maps to producer settings: `BestEffort` → `acks=0`; `AtMostOnce` → `acks=1`; `AtLeastOnce` and `ExactlyOnce` → `acks=all` with the idempotent producer enabled. Per-writer `RequestedDeliveryGuarantee` on the broker transport settings overrides the connection default. + +**SASL and TLS.** Use `kafkas://` or `KafkaConnectionOptions.Tls.UseTls` for TLS; `KafkaTlsOptions` carries CA / client-certificate / client-key PEM paths. `SecurityProtocol = KafkaSecurityProtocol.SaslSsl` with `SaslMechanism`, `UserName`, and `PasswordSecretId` enables SASL over TLS, and `PasswordSecretId` is resolved through the OPC UA secret store so configuration never carries a plaintext password. Sending SASL credentials over plaintext `kafka://` is rejected unless `AllowCredentialsOverPlaintext` is explicitly set for local development. + +**NativeAOT.** On `net10.0` the transport uses the pure-managed [Dekaf](https://github.com/thomhurst/Dekaf) client and is NativeAOT/trimming compatible; on `net472`, `net48`, `netstandard2.1`, `net8.0`, and `net9.0` it uses `Confluent.Kafka` (native librdkafka), which is not AOT-compatible. See [Native AOT](#native-aot). + ## Encodings ### UADP — `Opc.Ua.PubSub.Encoding.Uadp` @@ -1363,7 +1393,11 @@ See `Applications\ConsoleReferencePubSubClient` (the `external` mode) for a comp - [Certificates](Certificates.md) - [OPC UA Part 14](https://reference.opcfoundation.org/specs/OPC-10000-14/v1.05.06/) -## High availability state providers +## High availability and redundancy + +Part 14 §9.1.6 HA deployments run several publisher / subscriber instances over a shared configuration and elect a single active owner per component. PubSub splits this into two injectable concerns — externalized **state providers** and **active/standby activation** — each with a zero-overhead in-memory default so single-instance deployments are unaffected. These abstractions align with the server-side OPC 10000-4 §6.6 redundancy model documented in [High availability](HighAvailability.md). + +### State providers Part 14 deployments that run multiple server instances should externalize the state that otherwise lives in one process. The PubSub DI surface provides @@ -1394,6 +1428,30 @@ configuration and per-dataset `ConfigurationVersion`, SKS key changes are mirrored to the security-key store, and component run-state transitions are mirrored to the runtime-state store. +### Active/standby activation + +Redundant sets elect one active owner per component so exactly one instance drives each `WriterGroup` / `ReaderGroup` while the others stand by and take over on failover. Two provider contracts express this: + +- `IPubSubActivationCoordinator` answers, per component, whether this instance is `PubSubComponentRole.Active` (drive the transport) or `PubSubComponentRole.Standby` (wait to take over), and raises `RoleChanged` so the runtime pauses or resumes the component. Components are addressed by a deterministic id — for example `pubsub:writergroup::` — so every replica agrees on the key. +- `IPubSubLeaseStore` is the shared coordination primitive: `TryAcquireAsync` / `TryRenewAsync` / `ReleaseAsync` grant a single owner per lease key and stamp a monotonic **fencing token** on each ownership change so a paused owner cannot resume after its lease expired (renewal of an expired lease is rejected and forces reacquisition, which advances the token). + +The default coordinator is `AlwaysActiveCoordinator`, which reports every component active — non-redundant deployments incur no overhead. Redundant deployments register `LeaseActivationCoordinator` (a background acquire/renew loop over an `IPubSubLeaseStore`); the built-in `InMemoryPubSubLeaseStore` is single-process, and a distributed store backs true cross-instance failover: + +```csharp +services.AddOpcUa() + .AddPubSub() + .WithLeaseStore(distributedLeaseStore) // shared, cross-instance + .WithActivationCoordinator(new LeaseActivationCoordinator( + distributedLeaseStore, + telemetry, + ownerId: Environment.MachineName, + leaseDuration: TimeSpan.FromSeconds(15))); +``` + +`PubSubRedundancyMode` (`None` / `Cold` / `Warm` / `Hot`) selects how much runtime state a standby keeps warm: `Cold` rebuilds from the shared stores on failover, `Warm` keeps the configuration loaded but paused, and `Hot` additionally tracks live sequence / keep-alive state so take-over introduces no gap. + +**Alignment with the core redundancy abstractions.** The server-side redundancy work (Part 4 §6.6, see [High availability](HighAvailability.md)) introduces the core primitives `Opc.Ua.Redundancy.ISharedKeyValueStore` (a `CompareAndSwapAsync` "master-write" primitive) and `Opc.Ua.Redundancy.ILeaderElection` (`IsLeader` / `LeadershipChanged` / `TryAcquireOrRenewAsync`), and drives the OPC UA `ServiceLevel` from leadership. The PubSub `IPubSubLeaseStore` / lease model maps directly onto that compare-and-swap primitive (a lease is a fenced CAS on a per-component key), and `IPubSubActivationCoordinator` mirrors per-component leader election. Once the core abstractions ship, the PubSub lease store is intended to converge onto `ISharedKeyValueStore.CompareAndSwapAsync` so a single shared store and leader-election stack serves both the server address space and PubSub components. + ## Diagnostics `IPubSubDiagnostics` is the per-component counter sink. Every @@ -1422,7 +1480,11 @@ detailed the counters become; configure via ## Native AOT -PubSub is AOT-clean across all four assemblies. +The four core PubSub assemblies (`Opc.Ua.PubSub`, `.Udp`, `.Eth`, `.Mqtt`) are +AOT-clean on every target framework. The `Opc.Ua.PubSub.Kafka` transport is +AOT-clean only on `net10.0` (managed Dekaf client); on the other frameworks it +uses `Confluent.Kafka` / native librdkafka and is not AOT-compatible (see +[Apache Kafka](#apache-kafka)). - **No reflection-based serialization.** Source-generated `IEncodeable` types (Part 14 datatypes) plus hand-written diff --git a/Docs/PubSubKafka.md b/Docs/PubSubKafka.md deleted file mode 100644 index 6c0d1b8581..0000000000 --- a/Docs/PubSubKafka.md +++ /dev/null @@ -1,142 +0,0 @@ -# Apache Kafka PubSub transport - -`Opc.Ua.PubSub.Kafka` implements the OPC UA Part 14 Annex B.2 Apache Kafka transport mapping for broker-based PubSub deployments. It registers two transport profiles: `pubsub-kafka-uadp` (`KafkaProfiles.PubSubKafkaUadpTransport`) for UADP NetworkMessages and `pubsub-kafka-json` (`KafkaProfiles.PubSubKafkaJsonTransport`) for JSON NetworkMessages. - -Use `kafka://host[:port][,host[:port]...]` for plaintext bootstrap servers and `kafkas://host[:port][,...]` for TLS-protected bootstrap servers. The default Kafka port is `9092`, so `kafka://localhost` and `kafka://localhost:9092` resolve to the same bootstrap server list. - -## Dependency injection - -Register the transport on the PubSub builder with `AddKafkaTransport()`. The extension lives in the `Microsoft.Extensions.DependencyInjection` namespace and registers both Kafka profile factories. Options are supplied with `KafkaConnectionOptions`, either by callback or from the `OpcUa:PubSub:Kafka` configuration section. - -Important `KafkaConnectionOptions` fields include `Endpoint` and `BootstrapServers` (bootstrap broker selection), `GroupId` (subscriber consumer group), `SecurityProtocol`, `SaslMechanism`, `UserName`, `PasswordSecretId`, `AuthenticationProfileUri`, `ResourceUri`, `Tls`, `DeliveryGuarantee`, `AutoOffsetReset`, `EnableAutoCommit`, `MaxMessageSize`, and `Topics.Prefix`. `PasswordSecretId` is resolved through the OPC UA secret store so configuration does not carry plaintext passwords. - -```csharp -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Opc.Ua; -using Opc.Ua.PubSub.Configuration; -using Opc.Ua.PubSub.Kafka; - -const string DataTopic = "opcua.json.data.Line1.Simple"; -const string MetadataTopic = "opcua.json.metadata.Line1.Simple"; - -HostApplicationBuilder publisherHost = Host.CreateApplicationBuilder(args); -publisherHost.Services.AddOpcUa().AddPubSub(pubsub => -{ - pubsub.AddPublisher() - .AddKafkaTransport(options => - { - options.Endpoint = "kafkas://broker1.example.com:9093,broker2.example.com:9093"; - options.BootstrapServers = "broker1.example.com:9093,broker2.example.com:9093"; - options.SecurityProtocol = KafkaSecurityProtocol.SaslSsl; - options.SaslMechanism = KafkaSaslMechanism.ScramSha256; - options.UserName = "pubsub-publisher"; - options.PasswordSecretId = "KafkaPublisherPassword"; - options.DeliveryGuarantee = KafkaQualityOfService.AtLeastOnce; - options.Tls = new KafkaTlsOptions - { - ValidateServerCertificate = true, - CaCertificatePath = "pki/kafka/ca.pem" - }; - }) - .AddDataSetSource("Simple", publishedDataSetSource) - .ConfigureApplication(app => app.UseConfiguration(BuildKafkaPublisherConfiguration())); -}); - -HostApplicationBuilder subscriberHost = Host.CreateApplicationBuilder(args); -subscriberHost.Services.AddOpcUa().AddPubSub(pubsub => -{ - pubsub.AddSubscriber() - .AddKafkaTransport(options => - { - options.Endpoint = "kafkas://broker1.example.com:9093,broker2.example.com:9093"; - options.GroupId = "opcua-reference-subscribers"; - options.SecurityProtocol = KafkaSecurityProtocol.SaslSsl; - options.SaslMechanism = KafkaSaslMechanism.ScramSha256; - options.UserName = "pubsub-subscriber"; - options.PasswordSecretId = "KafkaSubscriberPassword"; - options.AutoOffsetReset = KafkaAutoOffsetReset.Latest; - options.Tls = new KafkaTlsOptions - { - ValidateServerCertificate = true, - CaCertificatePath = "pki/kafka/ca.pem" - }; - }) - .AddSubscribedDataSetSink("Reader 1", sp => subscribedDataSetSink) - .ConfigureApplication(app => app.UseConfiguration(BuildKafkaSubscriberConfiguration())); -}); - -static PubSubConfigurationDataType BuildKafkaPublisherConfiguration() -{ - return PubSubConfigurationBuilder.Create() - .AddPublishedDataSet("Simple", dataSet => dataSet - .AddField("Value", (byte)DataTypes.Int32, DataTypeIds.Int32)) - .AddConnection("Kafka Publisher", connection => connection - .WithPublisherId(new Variant((ushort)1)) - .WithTransportProfile(KafkaProfiles.PubSubKafkaJsonTransport) - .WithAddress("kafkas://broker1.example.com:9093,broker2.example.com:9093") - .AddWriterGroup("WriterGroup 1", group => group - .WithWriterGroupId(100) - .WithTransportSettings(new BrokerWriterGroupTransportDataType - { - QueueName = DataTopic - }) - .WithMessageSettings(new JsonWriterGroupMessageDataType - { - NetworkMessageContentMask = (uint)( - JsonNetworkMessageContentMask.NetworkMessageHeader | - JsonNetworkMessageContentMask.DataSetMessageHeader | - JsonNetworkMessageContentMask.PublisherId) - }) - .AddDataSetWriter("Writer 1", writer => writer - .WithDataSetName("Simple") - .WithDataSetWriterId(1) - .WithTransportSettings(new BrokerDataSetWriterTransportDataType - { - QueueName = DataTopic, - MetaDataQueueName = MetadataTopic, - RequestedDeliveryGuarantee = BrokerTransportQualityOfService.AtLeastOnce - })))) - .Build(); -} - -static PubSubConfigurationDataType BuildKafkaSubscriberConfiguration() -{ - return PubSubConfigurationBuilder.Create() - .AddConnection("Kafka Subscriber", connection => connection - .WithPublisherId(new Variant((ushort)1)) - .WithTransportProfile(KafkaProfiles.PubSubKafkaJsonTransport) - .WithAddress("kafkas://broker1.example.com:9093,broker2.example.com:9093") - .AddReaderGroup("ReaderGroup 1", group => group - .AddDataSetReader("Reader 1", reader => reader - .WithFilter(new Variant((ushort)1), 100, 1) - .WithTransportSettings(new BrokerDataSetReaderTransportDataType - { - QueueName = DataTopic, - MetaDataQueueName = MetadataTopic, - RequestedDeliveryGuarantee = BrokerTransportQualityOfService.AtLeastOnce - }) - .WithMirrorSubscribedDataSet("Reader 1")))) - .Build(); -} -``` - -## Topic mapping - -Kafka uses the OPC UA broker transport settings as topic names. `BrokerDataSetWriterTransportDataType.QueueName` and `BrokerDataSetReaderTransportDataType.QueueName` select the data topic for a specific DataSetWriter/DataSetReader. `BrokerWriterGroupTransportDataType.QueueName` is the writer-group fallback for data messages. `MetaDataQueueName` selects the metadata topic; if it is not set, the transport generates a deterministic fallback topic from `KafkaConnectionOptions.Topics.Prefix`, the encoding (`uadp` or `json`), message type, PublisherId, WriterGroupId, and DataSetWriterId. - -Kafka topic names should use Kafka-safe characters such as letters, digits, `.`, `_`, and `-`. The fallback topic builder uses `.` as the segment separator. - -## Record headers and delivery guarantees - -Every produced Kafka record carries the normative `content-type` record header from Part 14 Annex B.2: `application/opcua+uadp` for `pubsub-kafka-uadp` and `application/json` for `pubsub-kafka-json`. The record key is derived from the PubSubConnection PublisherId when available so records for the same publisher preserve partition ordering. - -`KafkaConnectionOptions.DeliveryGuarantee` maps the Part 14 broker QoS intent to Kafka producer settings: `BestEffort` uses `acks=0`, `AtMostOnce` uses `acks=1`, `AtLeastOnce` uses `acks=all` with producer idempotence enabled, and `ExactlyOnce` uses `acks=all` with producer idempotence enabled. Per-writer `RequestedDeliveryGuarantee` values on broker transport settings override the connection default when specified. - -## SASL and TLS - -Use `kafkas://` or set `KafkaConnectionOptions.Tls.UseTls` for TLS. `KafkaTlsOptions` carries CA, client certificate, and client key PEM paths. Use `SecurityProtocol = KafkaSecurityProtocol.SaslSsl` with `SaslMechanism`, `UserName`, and `PasswordSecretId` for SASL over TLS. Sending SASL credentials over plaintext `kafka://` is rejected unless `AllowCredentialsOverPlaintext` is explicitly enabled for local development or a controlled network. - -## NativeAOT note - -On `net10.0`, `Opc.Ua.PubSub.Kafka` uses the pure-managed Dekaf Kafka client and is intended to be NativeAOT-compatible. On `net472`, `net48`, `netstandard2.1`, `net8.0`, and `net9.0`, the library uses Confluent.Kafka, which wraps native librdkafka and is not NativeAOT-compatible. diff --git a/Docs/README.md b/Docs/README.md index 5184becc5f..7dda00c8f0 100644 --- a/Docs/README.md +++ b/Docs/README.md @@ -42,7 +42,7 @@ Here is a list of available documentation for different topics: * [PubSub (Part 14)](PubSub.md) - Publisher/subscriber support library: architecture, fluent builder, transports (UDP / MQTT 3.1.1 + 5.0 / Kafka / Ethernet Layer 2), encodings (UADP / JSON), security, and server-side address space. * [Migration sub-doc](migrate/2.0.x/pubsub.md) - 1.5.378 → 2.0 breaking API, transport, JSON, and field-encoding changes, plus the compatibility matrix. * [Ethernet transport](PubSub.md#transports) - Layer 2 PubSub (`opc.eth://`, EtherType `0xB62C`, 802.1Q VLAN) with native AF_PACKET / BPF, SharpPcap, and in-memory backends. - * [Kafka transport](PubSubKafka.md) - Apache Kafka broker transport (`kafka://`, `kafkas://`) for UADP and JSON PubSub profiles with SASL/TLS and NativeAOT support on `net10.0`. + * [Kafka transport](PubSub.md#apache-kafka) - Apache Kafka broker transport (`kafka://`, `kafkas://`) for UADP and JSON PubSub profiles with SASL/TLS and NativeAOT support on `net10.0`. * [External server adapter](PubSub.md#binding-pubsub-to-an-external-opc-ua-server-client-session-adapters) - Bind PubSub publishers, subscribers, and Action responders to an external OPC UA server through `ManagedSession`. * [Dependency Injection extensions](DependencyInjection.md) - `AddPubSub`, `AddPubSubPublisher`, `AddPubSubSubscriber`, `AddPubSubSecurityKeyServiceClient/Server`, `AddPubSubAddressSpace`. * [Profiles](Profiles.md#pubsub-transports) - Datagram-v2, SKS pull / push, AES-128/256-CTR security facets. diff --git a/Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md b/Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md index da97ca7abb..06995b135f 100644 --- a/Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md +++ b/Libraries/Opc.Ua.PubSub.Kafka/NugetREADME.md @@ -39,6 +39,6 @@ The other PubSub transports (UDP, Ethernet, MQTT) remain AOT-compatible on all f ## Additional documentation -See the [PubSub Kafka documentation](https://github.com/OPCFoundation/UA-.NETStandard/blob/master/Docs/PubSubKafka.md) -and the [PubSub documentation](https://github.com/OPCFoundation/UA-.NETStandard/blob/master/Docs/PubSub.md) +See the [Apache Kafka transport documentation](https://github.com/OPCFoundation/UA-.NETStandard/blob/master/Docs/PubSub.md#apache-kafka) +in the [PubSub documentation](https://github.com/OPCFoundation/UA-.NETStandard/blob/master/Docs/PubSub.md) for transports, encodings, security, high availability, and the fluent / DI API. From 695d7e0332ea307c0dcfef43b9148c45cb7e87a5 Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Sat, 4 Jul 2026 02:10:47 +0200 Subject: [PATCH 11/21] Fix Kafka Docker integration test endpoint construction (CI) GetBootstrapAddress() returns a scheme-qualified URI (e.g. PLAINTEXT://127.0.0.1:49158); the test prepended kafka:// verbatim, producing a malformed kafka://PLAINTEXT://... endpoint that KafkaEndpointParser rejects with 'empty port component'. This only surfaced on CI runners where Docker is available (the test skips locally). Extract the host:port authority from the bootstrap URI so the kafka:// endpoint is well-formed. --- .../KafkaIntegrationDockerTests.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs index 7af20b809b..93b27029a9 100644 --- a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs @@ -80,9 +80,12 @@ public async Task OneTimeTearDownAsync() public async Task RealBrokerRoundTripsPayloadAsync() { Assert.That(m_container, Is.Not.Null); - string bootstrapAddress = m_container!.GetBootstrapAddress(); + // GetBootstrapAddress() returns a scheme-qualified URI such as + // "PLAINTEXT://127.0.0.1:49158"; extract the host:port authority so + // the kafka:// endpoint is well-formed. + var bootstrapUri = new Uri(m_container!.GetBootstrapAddress()); string topic = "opcua-json-data-" + Guid.NewGuid().ToString("N"); - string url = "kafka://" + bootstrapAddress; + string url = $"kafka://{bootstrapUri.Host}:{bootstrapUri.Port}"; IConfigurationRoot configuration = new ConfigurationBuilder() .AddInMemoryCollection(new System.Collections.Generic.Dictionary { From 09fb99ef84c36c2ad912b012e49e77aad1c86ac1 Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Sat, 4 Jul 2026 08:04:22 +0200 Subject: [PATCH 12/21] Use a Kafka 4.0 broker for the Kafka Docker integration test (CI) On net10.0 the transport uses the Dekaf client, whose consumer requires a Kafka 4.0+ broker (KIP-848 ConsumerGroupHeartbeat API). The test used confluentinc/cp-kafka:7.5.12 (Kafka 3.5), so the real-broker round trip threw BrokerVersionException and the subscriber never received the frame. Switch to apache/kafka:4.0.0 (KRaft, new consumer group protocol enabled by default); the Confluent client on other TFMs uses the classic protocol, also supported. --- .../KafkaIntegrationDockerTests.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs index 93b27029a9..4acdb6f76e 100644 --- a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs @@ -58,7 +58,11 @@ public async Task OneTimeSetUpAsync() { try { - m_container = new KafkaBuilder("confluentinc/cp-kafka:7.5.12").Build(); + // Dekaf (the net10.0 client) requires a Kafka 4.0+ broker for the + // KIP-848 ConsumerGroupHeartbeat API; the Apache image runs KRaft and + // enables the new consumer group protocol by default. The Confluent + // client used on other TFMs falls back to the classic protocol. + m_container = new KafkaBuilder("apache/kafka:4.0.0").Build(); await m_container.StartAsync().ConfigureAwait(false); } catch (Exception ex) From 52d243a69d47f9cdaa7c68d04ce7f12f94937486 Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Sat, 4 Jul 2026 17:38:35 +0200 Subject: [PATCH 13/21] Make Kafka Docker round-trip test resilient to consumer-assignment race (CI) With the apache/kafka:4.0.0 broker the Dekaf consumer now connects, but the test produced a single record immediately after opening the subscriber, before the consumer had joined the group and received its partition assignment, so the subscriber never observed the record (frame was null after 20s). Resend the same topic/payload every 2s until a record is observed or a 30s deadline elapses; a genuine no-delivery still fails the assertion. --- .../KafkaIntegrationDockerTests.cs | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs index 4acdb6f76e..46b53bb7f9 100644 --- a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs @@ -128,11 +128,26 @@ public async Task RealBrokerRoundTripsPayloadAsync() await subscriber.OpenAsync(CancellationToken.None).ConfigureAwait(false); await publisher.OpenAsync(CancellationToken.None).ConfigureAwait(false); byte[] payload = [0x10, 0x20, 0x30, 0x40]; - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); Task receiveTask = KafkaTestHelper.ReceiveOneAsync(subscriber, cts.Token); - await publisher.SendAsync(payload, topic).ConfigureAwait(false); - PubSubTransportFrame? frame = await receiveTask.ConfigureAwait(false); + // The consumer may still be joining the group and receiving its + // partition assignment when the first record is produced, so resend + // periodically until the subscriber observes a record or the deadline + // elapses. Every resend carries the same topic and payload, so which + // copy is observed does not affect the assertions. + PubSubTransportFrame? frame = null; + while (frame is null && !cts.IsCancellationRequested) + { + await publisher.SendAsync(payload, topic).ConfigureAwait(false); + Task finished = await Task.WhenAny( + receiveTask, + Task.Delay(TimeSpan.FromSeconds(2))).ConfigureAwait(false); + if (finished == receiveTask) + { + frame = await receiveTask.ConfigureAwait(false); + } + } Assert.That(frame, Is.Not.Null); Assert.That(frame!.Value.Topic, Is.EqualTo(topic)); From 888c4622f3b4877aa8c415a79f11f5221a00ec1f Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Sun, 5 Jul 2026 17:32:43 +0200 Subject: [PATCH 14/21] Fix Kafka Docker round-trip test: create topic before subscribing (CI) Root-caused locally with Docker: the net10 Dekaf consumer (KIP-848 protocol) does not pick up a topic that is created *after* it has subscribed, so the subscriber-then-produce ordering never delivered the record (frame was null). A standalone Dekaf repro confirmed produce-then-subscribe delivers immediately while subscribe-then-produce never does. Open the publisher and produce one record first so the topic exists before the subscriber subscribes; it then reads from the earliest offset. Full Kafka suite: 83/83 pass locally on net10 with a real apache/kafka:4.0.0 broker. --- .../KafkaIntegrationDockerTests.cs | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs index 46b53bb7f9..c2a24bd87d 100644 --- a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaIntegrationDockerTests.cs @@ -125,21 +125,26 @@ public async Task RealBrokerRoundTripsPayloadAsync() kafkaSubscriber.Subscriptions.Add(topic); } - await subscriber.OpenAsync(CancellationToken.None).ConfigureAwait(false); - await publisher.OpenAsync(CancellationToken.None).ConfigureAwait(false); byte[] payload = [0x10, 0x20, 0x30, 0x40]; + + // Open the publisher and produce one record before the subscriber + // subscribes so the topic already exists: the Dekaf consumer (net10, + // KIP-848 protocol) does not pick up a topic that is created after it + // has subscribed. The subscriber reads from the earliest offset once + // it is assigned the partition. + await publisher.OpenAsync(CancellationToken.None).ConfigureAwait(false); + await publisher.SendAsync(payload, topic).ConfigureAwait(false); + await subscriber.OpenAsync(CancellationToken.None).ConfigureAwait(false); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); Task receiveTask = KafkaTestHelper.ReceiveOneAsync(subscriber, cts.Token); - // The consumer may still be joining the group and receiving its - // partition assignment when the first record is produced, so resend - // periodically until the subscriber observes a record or the deadline - // elapses. Every resend carries the same topic and payload, so which - // copy is observed does not affect the assertions. + // Resend periodically in case partition assignment lags; every resend + // carries the same topic and payload, so which copy is observed does not + // affect the assertions. PubSubTransportFrame? frame = null; while (frame is null && !cts.IsCancellationRequested) { - await publisher.SendAsync(payload, topic).ConfigureAwait(false); Task finished = await Task.WhenAny( receiveTask, Task.Delay(TimeSpan.FromSeconds(2))).ConfigureAwait(false); @@ -147,6 +152,10 @@ public async Task RealBrokerRoundTripsPayloadAsync() { frame = await receiveTask.ConfigureAwait(false); } + else + { + await publisher.SendAsync(payload, topic).ConfigureAwait(false); + } } Assert.That(frame, Is.Not.Null); From b9ff3d1ab050d9be3738714002c3348032fa744f Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Mon, 6 Jul 2026 15:03:07 +0200 Subject: [PATCH 15/21] Update Dekaf to 1.0.0, consolidate Kafka package entries, bump packages (review feedback) - Update Dekaf 0.0.1-ci.2047 -> 1.0.0 (stable). Dekaf 1.0.0 has breaking API changes: ProducerMessage.Create(topic,key,value,headers) was removed (construct ProducerMessage via object initializer with a settable Topic) and IKafkaConsumer.Wakeup() was removed (the consume loop already exits on token cancellation and CloseAsync). Adapter updated accordingly. - Move the Confluent.Kafka and Dekaf PackageVersion entries into the main alphabetical package list and drop their separate commented ItemGroups. - Bump non-pinned packages to latest where it does not cross a TFM/major-version compatibility boundary: AspNetCore 2.3.10->2.3.11, Extensions.Http.Resilience / TimeProvider.Testing 10.6.0->10.7.0, System.Reflection.Metadata 10.0.6->10.0.9, SourceLink 10.0.102->10.0.300, NET.Test.Sdk 18.6.0->18.7.0, SharpFuzz 2.2.0->2.3.0, SourceGenerator.Foundations 2.0.14->2.0.16, System.CommandLine 2.0.8->2.0.9, Nerdbank.GitVersioning 3.9.50->3.10.85, TUnit 1.51.0->1.58.0. Validated locally with Docker: Kafka tests 83/83 on net10 (real apache/kafka:4.0.0 broker) and the Kafka library builds clean on net10 and net48. --- Directory.Packages.props | 53 ++++++++----------- .../Internal/DekafKafkaClientAdapter.cs | 13 ++--- 2 files changed, 28 insertions(+), 38 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 3dd0c17cb0..0ce8f76d5c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -12,7 +12,9 @@ + + @@ -21,14 +23,14 @@ - - - - - - - - + + + + + + + + @@ -46,22 +48,22 @@ - + - + - - - - + + + + - - + + - - - - - - - \ No newline at end of file diff --git a/Libraries/Opc.Ua.PubSub.Kafka/Internal/DekafKafkaClientAdapter.cs b/Libraries/Opc.Ua.PubSub.Kafka/Internal/DekafKafkaClientAdapter.cs index 57edaa1010..8f4ae8dcee 100644 --- a/Libraries/Opc.Ua.PubSub.Kafka/Internal/DekafKafkaClientAdapter.cs +++ b/Libraries/Opc.Ua.PubSub.Kafka/Internal/DekafKafkaClientAdapter.cs @@ -254,11 +254,13 @@ public async ValueTask ProduceAsync(KafkaMessage message, CancellationToken ct) Headers headers = CreateHeaders(message); byte[] key = message.Key.IsEmpty ? null! : message.Key.ToArray(); byte[] value = message.Value.IsEmpty ? Array.Empty() : message.Value.ToArray(); - ProducerMessage record = ProducerMessage.Create( - message.Topic, - key, - value, - headers); + var record = new ProducerMessage + { + Topic = message.Topic, + Key = key, + Value = value, + Headers = headers + }; await m_clientGate.WaitAsync(ct).ConfigureAwait(false); try @@ -318,7 +320,6 @@ private async ValueTask ShutdownAsync(bool raiseEvent) try { loopCts.Cancel(); - consumer?.Wakeup(); } catch (Exception ex) { From 6a942cc744cd9d69d45834b45da88ab7642a50ff Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Mon, 6 Jul 2026 16:30:27 +0200 Subject: [PATCH 16/21] Suppress Dekaf 1.0.0 assembly-level AOT rollup warning in the AOT test (CI) Updating Dekaf to 1.0.0 regressed the NativeAOT publish of Opc.Ua.Aot.Tests: Dekaf.dll now emits an assembly-level IL3053 ('produced AOT analysis warnings') rollup that the pre-release did not, and with warnings-as-errors the ilc publish failed (aot-ubuntu-latest / aot-windows-latest). Dekaf reaches the AOT test transitively via ConsoleReferencePubSubClient -> Opc.Ua.PubSub.Kafka. Extend the existing third-party rollup suppression (which already covers the IL2104 trim rollup) to also cover IL3053. This is the referenced-assembly rollup only; our own assemblies remain AOT-analyzed individually per-project via IsAotCompatible on net8.0+, and the native publish itself succeeds (ilc returns 0). Validated locally: dotnet publish -r win-x64 of Opc.Ua.Aot.Tests succeeds with 0 warnings / 0 errors. --- Tests/Opc.Ua.Aot.Tests/Opc.Ua.Aot.Tests.csproj | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Tests/Opc.Ua.Aot.Tests/Opc.Ua.Aot.Tests.csproj b/Tests/Opc.Ua.Aot.Tests/Opc.Ua.Aot.Tests.csproj index 65904b2a35..07ecfab900 100644 --- a/Tests/Opc.Ua.Aot.Tests/Opc.Ua.Aot.Tests.csproj +++ b/Tests/Opc.Ua.Aot.Tests/Opc.Ua.Aot.Tests.csproj @@ -6,8 +6,12 @@ true enable disable - - $(NoWarn);IL2104 + + $(NoWarn);IL2104;IL3053 From cbc965f0b63b8f6ed4a94549bd7056708affdff7 Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Mon, 6 Jul 2026 16:38:42 +0200 Subject: [PATCH 17/21] Promote transcoded DataSet fields into Kafka record headers KafkaBrokerTransport now implements IPubSubHeaderTransport (added in #3956): the transcoding pipeline's promoted fields are surfaced as Kafka record headers. Refactors SendAsync into a shared ProduceInternalAsync and adds the header-aware overload mapping ArrayOf -> KafkaMessage.Headers (content-type stays adapter-set). Adds KafkaHeaderTransportTests and generalizes the Docs/PubSub.md promotion section to broker headers (MQTT + Kafka). 0-warning build and tests pass on net10.0 (Dekaf) and net48 (Confluent); full Kafka suite 87 passed. --- Docs/PubSub.md | 6 +- .../KafkaBrokerTransport.cs | 38 +++++- .../KafkaHeaderTransportTests.cs | 113 ++++++++++++++++++ 3 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaHeaderTransportTests.cs diff --git a/Docs/PubSub.md b/Docs/PubSub.md index 69bf0541f1..b7941110ba 100644 --- a/Docs/PubSub.md +++ b/Docs/PubSub.md @@ -1000,9 +1000,9 @@ services.AddOpcUa() Editing the `Routes` array — for example through a reloadable JSON file — changes only the affected bridges: adding a route starts a new bridge, removing a route disposes its bridge, and changing a route's fields tears down and rebuilds just that route while every other route continues to run. -### Promoting fields to MQTT user properties +### Promoting fields to broker headers -Selected DataSet fields can be promoted into transport message properties on the target side. On MQTT this maps each promoted field to an MQTT 5.0 User Property, so downstream consumers can filter on the value without decoding the payload — the broker-layer analogue of the Part 14 [PromotedFields](https://reference.opcfoundation.org/specs/OPC-10000-14/v1.05.06/7.2.3) concept. Promotion is *copy* semantics: the promoted fields remain in the encoded payload and are additionally surfaced as properties. Field values are formatted as invariant strings. +Selected DataSet fields can be promoted into transport message headers on the target side — MQTT 5.0 User Properties or Kafka record headers — so downstream consumers can filter on the value without decoding the payload, the broker-layer analogue of the Part 14 [PromotedFields](https://reference.opcfoundation.org/specs/OPC-10000-14/v1.05.06/7.2.3) concept. Promotion is *copy* semantics: the promoted fields remain in the encoded payload and are additionally surfaced as headers. Field values are formatted as invariant strings. Configure promotion fluently with `PromoteFields(params string[])` and, optionally, `WithPromotedFieldPrefix(string)`, or declaratively with the `PromoteFields` and `PromotedFieldPrefix` route options. @@ -1015,7 +1015,7 @@ pubsub.AddTranscodingBridge(bridge => bridge .ToTopic("plant/line1/telemetry")); ``` -Promotion is delivered through the `IPubSubHeaderTransport` capability. Transports that implement it — the MQTT broker transport — attach the promoted properties to each published message; datagram transports without a header channel ignore them. On MQTT 3.1.1, which has no User Property support, the properties are silently dropped. +Promotion is delivered through the `IPubSubHeaderTransport` capability. Transports that implement it — the MQTT broker transport (as MQTT 5.0 User Properties) and the Kafka broker transport (as record headers) — attach the promoted properties to each published message; transports without a header channel (datagram transports) ignore them. On MQTT 3.1.1, which has no User Property support, the properties are silently dropped. ### Standalone primitives diff --git a/Libraries/Opc.Ua.PubSub.Kafka/KafkaBrokerTransport.cs b/Libraries/Opc.Ua.PubSub.Kafka/KafkaBrokerTransport.cs index 6b7311f9c0..64288a672b 100644 --- a/Libraries/Opc.Ua.PubSub.Kafka/KafkaBrokerTransport.cs +++ b/Libraries/Opc.Ua.PubSub.Kafka/KafkaBrokerTransport.cs @@ -73,7 +73,8 @@ namespace Opc.Ua.PubSub.Kafka /// publisher preserve ordering within a partition. /// /// - public sealed class KafkaBrokerTransport : IPubSubTransport, IPubSubTopicProvider + public sealed class KafkaBrokerTransport + : IPubSubTransport, IPubSubTopicProvider, IPubSubHeaderTransport { private readonly PubSubConnectionDataType m_connection; private readonly KafkaEndpoint m_endpoint; @@ -326,10 +327,41 @@ public async ValueTask CloseAsync(CancellationToken cancellationToken = default) } /// - public async ValueTask SendAsync( + public ValueTask SendAsync( ReadOnlyMemory payload, string? topic = null, CancellationToken cancellationToken = default) + => ProduceInternalAsync(payload, topic, null, cancellationToken); + + /// + public ValueTask SendAsync( + ReadOnlyMemory payload, + string? topic, + ArrayOf properties, + CancellationToken cancellationToken = default) + => ProduceInternalAsync(payload, topic, ToHeaders(properties), cancellationToken); + + private static Dictionary? ToHeaders( + ArrayOf properties) + { + if (properties.Count == 0) + { + return null; + } + var headers = new Dictionary(properties.Count, StringComparer.Ordinal); + for (int i = 0; i < properties.Count; i++) + { + PubSubMessageProperty property = properties[i]; + headers[property.Name] = property.Value; + } + return headers; + } + + private async ValueTask ProduceInternalAsync( + ReadOnlyMemory payload, + string? topic, + IReadOnlyDictionary? headers, + CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); if (string.IsNullOrEmpty(topic)) @@ -361,7 +393,7 @@ public async ValueTask SendAsync( m_partitionKey, payload, contentType, - Headers: null); + headers); await adapter.ProduceAsync(message, cancellationToken).ConfigureAwait(false); m_diagnostics?.Increment(PubSubDiagnosticsCounterKind.SentNetworkMessages, 1); diff --git a/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaHeaderTransportTests.cs b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaHeaderTransportTests.cs new file mode 100644 index 0000000000..85e8d7884e --- /dev/null +++ b/Tests/Opc.Ua.PubSub.Kafka.Tests/KafkaHeaderTransportTests.cs @@ -0,0 +1,113 @@ +/* ======================================================================== + * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.PubSub.Tests; +using Opc.Ua.PubSub.Transports; + +namespace Opc.Ua.PubSub.Kafka.Tests +{ + /// + /// Tests for promoting DataSet fields into Kafka record headers through + /// the capability on + /// . + /// + [TestFixture] + [Category("Unit")] + [TestSpec("B.2", Summary = "Kafka field promotion to record headers")] + [CancelAfter(10000)] + public sealed class KafkaHeaderTransportTests + { + [Test] + public async Task KafkaBrokerTransport_ImplementsHeaderCapability() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + Assert.That(transport, Is.InstanceOf()); + } + + [Test] + public async Task SendAsync_WithProperties_EmitsRecordHeaders() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + + PubSubMessageProperty[] properties = + [ + new("Temperature", "21.5"), + new("Unit", "C") + ]; + + await transport.SendAsync(new byte[] { 1, 2, 3 }, KafkaTestHelper.JsonTopic, properties) + .ConfigureAwait(false); + + KafkaMessage message = KafkaTestHelper.FirstProduced(factory.Adapter); + Assert.That(message.Headers, Is.Not.Null); + Assert.That(message.Headers!, Has.Count.EqualTo(2)); + Assert.Multiple(() => + { + Assert.That(message.Headers!["Temperature"], Is.EqualTo("21.5")); + Assert.That(message.Headers!["Unit"], Is.EqualTo("C")); + }); + } + + [Test] + public async Task SendAsync_NoProperties_EmitsNoRecordHeaders() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + + await transport + .SendAsync(new byte[] { 1 }, KafkaTestHelper.JsonTopic, + default(ArrayOf)) + .ConfigureAwait(false); + + KafkaMessage message = KafkaTestHelper.FirstProduced(factory.Adapter); + Assert.That(message.Headers, Is.Null); + } + + [Test] + public async Task PlainSendAsync_EmitsNoRecordHeaders() + { + var factory = new FakeKafkaClientFactory(); + await using KafkaBrokerTransport transport = KafkaTestHelper.NewTransport(factory); + await transport.OpenAsync(CancellationToken.None).ConfigureAwait(false); + + await transport.SendAsync(new byte[] { 1 }, KafkaTestHelper.JsonTopic) + .ConfigureAwait(false); + + KafkaMessage message = KafkaTestHelper.FirstProduced(factory.Adapter); + Assert.That(message.Headers, Is.Null); + } + } +} From b86e2cea93e897b50bf71c533663002f33541296 Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Mon, 6 Jul 2026 22:58:00 +0200 Subject: [PATCH 18/21] Revert SourceGenerator.Foundations 2.0.16 -> 2.0.14 (broken analyzer, CI) SourceGenerator.Foundations 2.0.16 (bumped as part of the 'update packages to latest' review request) ships a broken Roslyn analyzer: the compiler fails with CS8032 'An instance of analyzer SGF.Analyzer.SourceGeneratorAnalyzer cannot be created ... TargetInvocationException' when building Tools/Opc.Ua.SourceGeneration, failing the solution build on all TFMs (UA_Debug_net10_0, build-linux-all-tfm). Revert to the proven-good 2.0.14; latest is not usable here. --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 0ce8f76d5c..b8944d45fa 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -97,7 +97,7 @@ - + From 8b2983b2e505834b389e64678ba997788dafe173 Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Tue, 7 Jul 2026 04:24:17 +0200 Subject: [PATCH 19/21] Deterministic wait in PubSub failover tests (fix net48 flake, CI) LeaseActivationCoordinator_ElectsSingleActiveAndFailsOverOnStopAsync flaked on the net48 Windows CI runner ('The expected HA role transition did not occur'). The coordinator loops run on background threads driven by the FakeTimeProvider; WaitForConditionAsync used Task.Yield after advancing the clock, which does not reliably give those threads CPU to observe the advance and apply the role change on contended agents. Advance the clock then await a short real Task.Delay (with a wider iteration margin) so the background loops reliably progress. Assertion unchanged. Verified 3x on net48 locally (~200ms each). --- .../Redundancy/PubSubActivationFailoverTests.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Tests/Opc.Ua.PubSub.Tests/Redundancy/PubSubActivationFailoverTests.cs b/Tests/Opc.Ua.PubSub.Tests/Redundancy/PubSubActivationFailoverTests.cs index 989e592048..735ad829f7 100644 --- a/Tests/Opc.Ua.PubSub.Tests/Redundancy/PubSubActivationFailoverTests.cs +++ b/Tests/Opc.Ua.PubSub.Tests/Redundancy/PubSubActivationFailoverTests.cs @@ -396,16 +396,21 @@ private static async Task WaitForConditionAsync( FakeTimeProvider clock, Func> condition) { - for (int i = 0; i < 32; i++) + for (int i = 0; i < 80; i++) { if (await condition().ConfigureAwait(false)) { return; } - await Task.Yield(); clock.Advance(TimeSpan.FromSeconds(1)); - await Task.Yield(); + + // The coordinator loops run on background threads driven by the + // fake clock; a short real delay reliably yields CPU so they can + // observe the advance and apply the resulting role change before + // the next check. Task.Yield alone starves them on contended + // CI agents (observed flaky on the net48 Windows runner). + await Task.Delay(25).ConfigureAwait(false); } Assert.Fail("The expected HA role transition did not occur."); From a32121a011483b505cc381ea312b6eadd99040cf Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Tue, 7 Jul 2026 07:46:28 +0200 Subject: [PATCH 20/21] Update JsonSchema.Net to 9.2.2 and migrate schema tests (review feedback) Per review feedback, update JsonSchema.Net 7.4.0 -> 9.2.2. 9.2.2 has breaking changes that required migrating the schema-validation tests: - JsonSchema.Evaluate now takes a JsonElement (was JsonNode) -> convert via JsonSerializer.SerializeToElement(instance). - Schemas are registered in a global SchemaRegistry keyed by \ and rebuilding the same \ throws 'Overwriting registered schemas is not permitted'; build each schema with a fresh local registry (new BuildOptions { SchemaRegistry = new SchemaRegistry() }). Validated locally on net10: Opc.Ua.Core.Schema.Tests 92/92 and Opc.Ua.PubSub.Schema.Tests 23/23 pass. --- Directory.Packages.props | 2 +- .../SchemaValidationIntegrationTests.cs | 7 +++++-- .../PubSubRealMessageValidationTests.cs | 6 ++++-- .../PubSubSchemaValidationIntegrationTests.cs | 7 +++++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index b8944d45fa..a3e8a91a58 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -17,7 +17,7 @@ - + diff --git a/Tests/Opc.Ua.Core.Schema.Tests/SchemaValidationIntegrationTests.cs b/Tests/Opc.Ua.Core.Schema.Tests/SchemaValidationIntegrationTests.cs index d492fcefc9..c6f5574055 100644 --- a/Tests/Opc.Ua.Core.Schema.Tests/SchemaValidationIntegrationTests.cs +++ b/Tests/Opc.Ua.Core.Schema.Tests/SchemaValidationIntegrationTests.cs @@ -27,6 +27,7 @@ * http://opcfoundation.org/License/MIT/1.00/ * ======================================================================*/ +using System.Text.Json; using System.Text.Json.Nodes; using Json.Schema; using NUnit.Framework; @@ -161,9 +162,11 @@ private static JsonNode EncodeEncodeable(T value) private static EvaluationResults Evaluate(IUaSchema schema, JsonNode instance) { - var jsonSchema = JsonSchema.FromText(schema.ToSchemaString()); + var jsonSchema = JsonSchema.FromText( + schema.ToSchemaString(), + new BuildOptions { SchemaRegistry = new SchemaRegistry() }); return jsonSchema.Evaluate( - instance, + JsonSerializer.SerializeToElement(instance), new EvaluationOptions { OutputFormat = OutputFormat.List }); } } diff --git a/Tests/Opc.Ua.PubSub.Schema.Tests/PubSubRealMessageValidationTests.cs b/Tests/Opc.Ua.PubSub.Schema.Tests/PubSubRealMessageValidationTests.cs index 7e87bb3900..90d4b16075 100644 --- a/Tests/Opc.Ua.PubSub.Schema.Tests/PubSubRealMessageValidationTests.cs +++ b/Tests/Opc.Ua.PubSub.Schema.Tests/PubSubRealMessageValidationTests.cs @@ -218,9 +218,11 @@ private static DataSetMetaDataType CreateMetaData() private static EvaluationResults Evaluate(UaSchema schema, JsonNode instance) { - var jsonSchema = JsonSchema.FromText(schema.ToSchemaString()); + var jsonSchema = JsonSchema.FromText( + schema.ToSchemaString(), + new BuildOptions { SchemaRegistry = new SchemaRegistry() }); return jsonSchema.Evaluate( - instance, + JsonSerializer.SerializeToElement(instance), new EvaluationOptions { OutputFormat = OutputFormat.List }); } diff --git a/Tests/Opc.Ua.PubSub.Schema.Tests/PubSubSchemaValidationIntegrationTests.cs b/Tests/Opc.Ua.PubSub.Schema.Tests/PubSubSchemaValidationIntegrationTests.cs index 8f0d86f5a0..2eeb3c3a65 100644 --- a/Tests/Opc.Ua.PubSub.Schema.Tests/PubSubSchemaValidationIntegrationTests.cs +++ b/Tests/Opc.Ua.PubSub.Schema.Tests/PubSubSchemaValidationIntegrationTests.cs @@ -27,6 +27,7 @@ * http://opcfoundation.org/License/MIT/1.00/ * ======================================================================*/ +using System.Text.Json; using System.Text.Json.Nodes; using Json.Schema; using NUnit.Framework; @@ -143,9 +144,11 @@ private static DataSetMetaDataType CreateMetaData() private static EvaluationResults Evaluate(UaSchema schema, JsonNode instance) { - var jsonSchema = JsonSchema.FromText(schema.ToSchemaString()); + var jsonSchema = JsonSchema.FromText( + schema.ToSchemaString(), + new BuildOptions { SchemaRegistry = new SchemaRegistry() }); return jsonSchema.Evaluate( - instance, + JsonSerializer.SerializeToElement(instance), new EvaluationOptions { OutputFormat = OutputFormat.List }); } } From b8e3eedbe85fb22fbd2e8818d6630ebdaf30b6c5 Mon Sep 17 00:00:00 2001 From: Marc Schier Date: Tue, 7 Jul 2026 09:57:22 +0200 Subject: [PATCH 21/21] Bump Dekaf to 1.2.0 (net10 client); keep the dual-source Kafka split Dekaf 1.2.0 now ships a netstandard2.0 asset, but its consumer is non-functional at runtime off net10.0: validated against a real apache/kafka:4.0.0 broker, the producer works (record lands at offset 0) but the consumer receives nothing on net8.0 (both the Docker integration test and a minimal standalone Dekaf-only repro time out). So the transport keeps the dual-source split - Dekaf on net10.0, Confluent.Kafka on the other TFMs - to preserve a working subscriber everywhere, and only advances the net10.0 Dekaf client to 1.2.0. Kafka tests 87/87 on net10.0 (incl. the real-broker Docker round trip); library builds clean on net10.0 and net48. --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 5a2e39d26d..843c4514ee 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -14,7 +14,7 @@ - +