diff --git a/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs b/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs index e6eee49d..5cd3d135 100644 --- a/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs +++ b/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs @@ -47,7 +47,7 @@ public LdAiCompletionConfig BuildCompletionConfig( return BuildCompletionFromDefault(key, defaultValue, mergedVars, trackerFactory, interpolate); } - var (enabled, variationKey, version, mode) = ParseMeta(ldValue); + var (enabled, variationKey, version, mode, modelKey, modelVersion) = ParseMeta(ldValue); if (mode != LdAiCompletionConfig.Mode) { @@ -70,6 +70,8 @@ public LdAiCompletionConfig BuildCompletionConfig( enabled, variationKey, version, + modelKey, + modelVersion, messages, tools, judgeConfiguration, @@ -95,6 +97,8 @@ private LdAiCompletionConfig BuildCompletionFromDefault( defaultValue.Enabled ?? true, variationKey: "", version: 1, + modelKey: null, + modelVersion: 1, messages, tools: ImmutableDictionary.Empty, defaultValue.JudgeConfiguration, @@ -123,7 +127,7 @@ public LdAiAgentConfig BuildAgentConfig( return BuildAgentFromDefault(key, defaultValue, mergedVars, trackerFactory, interpolate); } - var (enabled, variationKey, version, mode) = ParseMeta(ldValue); + var (enabled, variationKey, version, mode, modelKey, modelVersion) = ParseMeta(ldValue); if (mode != LdAiAgentConfig.Mode) { @@ -146,6 +150,8 @@ public LdAiAgentConfig BuildAgentConfig( enabled, variationKey, version, + modelKey, + modelVersion, instructions, tools, model, @@ -169,6 +175,8 @@ internal LdAiAgentConfig BuildAgentFromDefault( defaultValue.Enabled ?? true, variationKey: "", version: 1, + modelKey: null, + modelVersion: 1, instructions, tools: ImmutableDictionary.Empty, defaultValue.Model, @@ -196,7 +204,7 @@ public LdAiJudgeConfig BuildJudgeConfig( return BuildJudgeFromDefault(key, defaultValue, mergedVars, trackerFactory, interpolate); } - var (enabled, variationKey, version, mode) = ParseMeta(ldValue); + var (enabled, variationKey, version, mode, modelKey, modelVersion) = ParseMeta(ldValue); if (mode != LdAiJudgeConfig.Mode) { @@ -218,6 +226,8 @@ public LdAiJudgeConfig BuildJudgeConfig( enabled, variationKey, version, + modelKey, + modelVersion, messages, evaluationMetricKey, model, @@ -240,6 +250,8 @@ private LdAiJudgeConfig BuildJudgeFromDefault( defaultValue.Enabled ?? true, variationKey: "", version: 1, + modelKey: null, + modelVersion: 1, messages, defaultValue.EvaluationMetricKey, defaultValue.Model, @@ -305,10 +317,17 @@ private Func TrackerFactoryFor(Context context, context, cfg.Model?.Name, cfg.Provider?.Name, + cfg.ModelKey, + cfg.ModelVersion, graphKey); } - private static (bool Enabled, string VariationKey, int Version, string Mode) ParseMeta(LdValue value) + // _ldMeta carries modelKey/modelVersion alongside the variation-level fields (rather than + // the model object) so modelVersion can't be mistaken for the underlying LLM's own version. + private readonly record struct Meta( + bool Enabled, string VariationKey, int Version, string Mode, string ModelKey, int ModelVersion); + + private static Meta ParseMeta(LdValue value) { var meta = value.Get("_ldMeta"); var enabled = meta.Get("enabled").AsBool; @@ -318,7 +337,10 @@ private static (bool Enabled, string VariationKey, int Version, string Mode) Par // Default to the completion mode when _ldMeta.mode is missing or non-string: legacy // flags predate the mode tag and were always served as completion configs. var mode = meta.Get("mode").AsString ?? LdAiCompletionConfig.Mode; - return (enabled, variationKey, version, mode); + var modelKey = meta.Get("modelKey").AsString; + var modelVersionValue = meta.Get("modelVersion"); + var modelVersion = modelVersionValue.IsNull ? 1 : modelVersionValue.AsInt; + return new Meta(enabled, variationKey, version, mode, modelKey, modelVersion); } private static LdAiConfigTypes.ModelConfig ParseModel(LdValue modelValue) diff --git a/pkgs/sdk/server-ai/src/Config/LdAiAgentConfig.cs b/pkgs/sdk/server-ai/src/Config/LdAiAgentConfig.cs index 152f021f..e009ff75 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiAgentConfig.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiAgentConfig.cs @@ -41,13 +41,15 @@ internal LdAiAgentConfig( bool enabled, string variationKey, int version, + string modelKey, + int modelVersion, string instructions, IReadOnlyDictionary tools, LdAiConfigTypes.ModelConfig model, LdAiConfigTypes.ProviderConfig provider, LdAiConfigTypes.JudgeConfiguration judgeConfiguration, Func trackerFactory) - : base(key, enabled, variationKey, version, model, provider, trackerFactory) + : base(key, enabled, variationKey, version, modelKey, modelVersion, model, provider, trackerFactory) { Instructions = instructions; Tools = tools?.ToImmutableDictionary() ?? ImmutableDictionary.Empty; diff --git a/pkgs/sdk/server-ai/src/Config/LdAiCompletionConfig.cs b/pkgs/sdk/server-ai/src/Config/LdAiCompletionConfig.cs index 92b48da7..9faef003 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiCompletionConfig.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiCompletionConfig.cs @@ -39,11 +39,12 @@ public sealed class LdAiCompletionConfig : LdAiConfig public LdAiConfigTypes.JudgeConfiguration JudgeConfiguration { get; } internal LdAiCompletionConfig(string key, bool enabled, string variationKey, int version, + string modelKey, int modelVersion, IEnumerable messages, IReadOnlyDictionary tools, LdAiConfigTypes.JudgeConfiguration judgeConfiguration, LdAiConfigTypes.ModelConfig model, LdAiConfigTypes.ProviderConfig provider, Func trackerFactory) - : base(key, enabled, variationKey, version, model, provider, trackerFactory) + : base(key, enabled, variationKey, version, modelKey, modelVersion, model, provider, trackerFactory) { Messages = messages?.ToList() ?? new List(); Tools = tools ?? ImmutableDictionary.Empty; diff --git a/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs b/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs index 6a7bee19..3d9e109b 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs @@ -40,11 +40,23 @@ public abstract class LdAiConfig /// internal int Version { get; } + /// + /// This field meant for internal LaunchDarkly usage. Not exposed on so + /// it can't be mistaken for the underlying LLM's own version (e.g. "GPT-5.4"). + /// + internal string ModelKey { get; } + + /// + /// This field meant for internal LaunchDarkly usage. + /// + internal int ModelVersion { get; } + /// /// Factory that produces a tracker for the config. The factory is mode-agnostic — it /// operates only on the shared fields (, , - /// , , ), so the - /// same tracker class serves all config modes. + /// , , , + /// , ), so the same tracker class serves + /// all config modes. /// private readonly Func _trackerFactory; @@ -64,6 +76,8 @@ private protected LdAiConfig( bool enabled, string variationKey, int version, + string modelKey, + int modelVersion, LdAiConfigTypes.ModelConfig model, LdAiConfigTypes.ProviderConfig provider, Func trackerFactory) @@ -72,6 +86,8 @@ private protected LdAiConfig( Enabled = enabled; VariationKey = variationKey ?? ""; Version = version; + ModelKey = modelKey; + ModelVersion = modelVersion; Model = model ?? new LdAiConfigTypes.ModelConfig("", new Dictionary(), new Dictionary()); Provider = provider ?? new LdAiConfigTypes.ProviderConfig(""); _trackerFactory = trackerFactory ?? throw new ArgumentNullException(nameof(trackerFactory)); diff --git a/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs b/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs index a4278ab7..af8875c0 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs @@ -71,7 +71,10 @@ public sealed record ModelConfig /// public readonly IReadOnlyDictionary Custom; - internal ModelConfig(string name, IReadOnlyDictionary parameters, IReadOnlyDictionary custom) + internal ModelConfig( + string name, + IReadOnlyDictionary parameters, + IReadOnlyDictionary custom) { Name = name; // Materialize into an ImmutableDictionary so a consumer that downcasts to diff --git a/pkgs/sdk/server-ai/src/Config/LdAiJudgeConfig.cs b/pkgs/sdk/server-ai/src/Config/LdAiJudgeConfig.cs index de32cb9d..4371d359 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiJudgeConfig.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiJudgeConfig.cs @@ -36,12 +36,14 @@ internal LdAiJudgeConfig( bool enabled, string variationKey, int version, + string modelKey, + int modelVersion, IEnumerable messages, string evaluationMetricKey, LdAiConfigTypes.ModelConfig model, LdAiConfigTypes.ProviderConfig provider, Func trackerFactory) - : base(key, enabled, variationKey, version, model, provider, trackerFactory) + : base(key, enabled, variationKey, version, modelKey, modelVersion, model, provider, trackerFactory) { Messages = messages?.ToList() ?? new List(); EvaluationMetricKey = evaluationMetricKey; diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 70a2182e..8ad6721e 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -76,7 +76,7 @@ public class LdAiConfigTracker : ILdAiConfigTracker /// internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string configKey, string variationKey, int version, Context context, string modelName, string providerName, - string graphKey = null) + string modelKey = null, int modelVersion = 1, string graphKey = null) { _client = client ?? throw new ArgumentNullException(nameof(client)); _configKey = configKey ?? throw new ArgumentNullException(nameof(configKey)); @@ -96,6 +96,7 @@ internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string conf { "version", LdValue.Of(_version) }, { "modelName", LdValue.Of(_modelName) }, { "providerName", LdValue.Of(_providerName) }, + { "modelVersion", LdValue.Of(modelVersion) }, }; if (!string.IsNullOrEmpty(_graphKey)) { @@ -105,6 +106,10 @@ internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string conf { trackDataBuilder.Add("variationKey", LdValue.Of(_variationKey)); } + if (!string.IsNullOrEmpty(modelKey)) + { + trackDataBuilder.Add("modelKey", LdValue.Of(modelKey)); + } _trackData = LdValue.ObjectFrom(trackDataBuilder); _resumptionToken = new Lazy(BuildResumptionToken); @@ -118,6 +123,7 @@ private string BuildResumptionToken() // Utf8JsonWriter gives stable key ordering and avoids the runtime cost of // anonymous-type reflection. The wire format omits empty optional fields so that // resumption tokens round-trip exactly for configs that never carried them. + // modelName, providerName, modelKey, and modelVersion are not included. using var stream = new MemoryStream(); using (var writer = new Utf8JsonWriter(stream)) { @@ -426,7 +432,8 @@ private LdValue MergeTrackData(string key, LdValue value) /// process or at a later time. /// /// The reconstructed tracker will have empty model and provider names, as these are not - /// included in the resumption token. + /// included in the resumption token. modelKey and modelVersion are also not included; + /// modelVersion defaults to 1 on reconstructed trackers. /// /// the resumption token obtained from /// the LaunchDarkly client @@ -466,7 +473,7 @@ public static LdAiConfigTracker FromResumptionToken(string token, ILaunchDarklyC } return new LdAiConfigTracker(client, payload.RunId, payload.ConfigKey, - payload.VariationKey, payload.Version ?? 1, context, "", "", payload.GraphKey); + payload.VariationKey, payload.Version ?? 1, context, "", "", graphKey: payload.GraphKey); } private class ResumptionPayload diff --git a/pkgs/sdk/server-ai/test/AgentGraphDefinitionTest.cs b/pkgs/sdk/server-ai/test/AgentGraphDefinitionTest.cs index 54bb8d5f..67bfc821 100644 --- a/pkgs/sdk/server-ai/test/AgentGraphDefinitionTest.cs +++ b/pkgs/sdk/server-ai/test/AgentGraphDefinitionTest.cs @@ -23,6 +23,8 @@ private static LdAiAgentConfig MakeAgentConfig(string key, bool enabled = true) enabled: enabled, variationKey: "v1", version: 1, + modelKey: null, + modelVersion: 1, instructions: null, tools: new Dictionary(), model: null, diff --git a/pkgs/sdk/server-ai/test/ConfigFactoryParserTest.cs b/pkgs/sdk/server-ai/test/ConfigFactoryParserTest.cs index 004cebce..ae434f27 100644 --- a/pkgs/sdk/server-ai/test/ConfigFactoryParserTest.cs +++ b/pkgs/sdk/server-ai/test/ConfigFactoryParserTest.cs @@ -1,5 +1,7 @@ using System.Collections.Generic; using LaunchDarkly.Sdk.Server.Ai.Config; +using LaunchDarkly.Sdk.Server.Ai.Interfaces; +using Moq; using Xunit; namespace LaunchDarkly.Sdk.Server.Ai; @@ -204,4 +206,89 @@ public void ParseAllFields_WithNoNewFields_ReturnsNullOrEmptyWithoutError() Assert.Null(judgeConfig); Assert.Null(evaluationMetricKey); } + + [Fact] + public void CreateTrackerDefaultsModelVersionAndOmitsModelKeyWhenAbsentFromFlag() + { + // modelKey/modelVersion are intentionally not exposed on Model (they'd read as + // properties of the LLM itself, e.g. a version like "5.4"); the only place they + // surface is the tracker's stamped event data, mirroring variationKey/version. + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + mockClient.Setup(x => + x.JsonVariation("model-config-no-version", It.IsAny(), It.IsAny())) + .Returns(LdValue.ObjectFrom(new Dictionary + { + ["_ldMeta"] = LdValue.ObjectFrom(new Dictionary + { + ["enabled"] = LdValue.Of(true), + ["variationKey"] = LdValue.Of("v1"), + ["version"] = LdValue.Of(1) + }), + ["model"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("gpt-4") + }), + ["provider"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("openai") + }), + ["messages"] = LdValue.ArrayOf() + })); + + var client = new LdAiClient(mockClient.Object); + var context = Context.New("user-key"); + var config = client.CompletionConfig("model-config-no-version", context); + var tracker = config.CreateTracker(); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => + d.Get("modelKey").IsNull && + d.Get("modelVersion").AsInt == 1), + 1.0f), Times.Once); + } + + [Fact] + public void CreateTrackerStampsModelKeyAndVersionOnTrackData() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + mockClient.Setup(x => + x.JsonVariation("my-config-key", It.IsAny(), It.IsAny())) + .Returns(LdValue.ObjectFrom(new Dictionary + { + ["_ldMeta"] = LdValue.ObjectFrom(new Dictionary + { + ["enabled"] = LdValue.Of(true), + ["variationKey"] = LdValue.Of("var-abc"), + ["version"] = LdValue.Of(7), + ["modelKey"] = LdValue.Of("my-model"), + ["modelVersion"] = LdValue.Of(2) + }), + ["model"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("gpt-4") + }), + ["provider"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("openai") + }), + ["messages"] = LdValue.ArrayOf() + })); + + var client = new LdAiClient(mockClient.Object); + var context = Context.New("user-key"); + var config = client.CompletionConfig("my-config-key", context); + var tracker = config.CreateTracker(); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => + d.Get("modelKey").AsString == "my-model" && + d.Get("modelVersion").AsInt == 2), + 1.0f), Times.Once); + } } diff --git a/pkgs/sdk/server-ai/test/LdAiAgentGraphConfigTest.cs b/pkgs/sdk/server-ai/test/LdAiAgentGraphConfigTest.cs index 7e47c846..b5ba1550 100644 --- a/pkgs/sdk/server-ai/test/LdAiAgentGraphConfigTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiAgentGraphConfigTest.cs @@ -24,7 +24,7 @@ private static LdAiConfigTracker MakeTracker(ILaunchDarklyClient client, Context string graphKey = null) { return new LdAiConfigTracker(client, Guid.NewGuid().ToString(), "config-key", - "v1", 1, context, "model", "provider", graphKey); + "v1", 1, context, "model", "provider", graphKey: graphKey); } // Test 1: Tracker created with graphKey → events include graphKey in data diff --git a/pkgs/sdk/server-ai/test/LdAiClientTest.cs b/pkgs/sdk/server-ai/test/LdAiClientTest.cs index dc4034f1..7f133f27 100644 --- a/pkgs/sdk/server-ai/test/LdAiClientTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiClientTest.cs @@ -668,7 +668,9 @@ public void CreateTrackerFromResumptionTokenSetsEmptyModelAndProvider() d.Get("variationKey").AsString == "var-1" && d.Get("version").AsInt == 2 && d.Get("modelName").AsString == "" && - d.Get("providerName").AsString == ""), + d.Get("providerName").AsString == "" && + d.Get("modelVersion").AsInt == 1 && + d.Get("modelKey").IsNull), 1.0f), Times.Once); } diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 0bd7793a..0ecc7a24 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -25,6 +25,8 @@ private static LdAiCompletionConfig BuildConfig(ILaunchDarklyClient client, stri enabled: false, variationKey: "", version: 1, + modelKey: null, + modelVersion: 1, messages: new List(), tools: new Dictionary(), judgeConfiguration: null, @@ -686,6 +688,84 @@ public void ResumptionTokenContainsExpectedFields() // modelName and providerName should NOT be in the token Assert.False(doc.RootElement.TryGetProperty("modelName", out _)); Assert.False(doc.RootElement.TryGetProperty("providerName", out _)); + Assert.False(doc.RootElement.TryGetProperty("modelKey", out _)); + Assert.False(doc.RootElement.TryGetProperty("modelVersion", out _)); + } + + [Fact] + public void TrackDataIncludesModelVersionByDefault() + { + var mockClient = new Mock(); + var context = Context.New("key"); + + var tracker = new LdAiConfigTracker(mockClient.Object, "test-run-id", + "config-key", "variation-key", 3, context, "fakeModel", "fakeProvider"); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => d.Get("modelVersion").AsInt == 1), 1.0f), Times.Once); + } + + [Fact] + public void TrackDataIncludesModelKeyWhenSet() + { + var mockClient = new Mock(); + var context = Context.New("key"); + + var tracker = new LdAiConfigTracker(mockClient.Object, "test-run-id", + "config-key", "variation-key", 3, context, "fakeModel", "fakeProvider", + modelKey: "my-model", modelVersion: 2); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => + d.Get("modelKey").AsString == "my-model" && + d.Get("modelVersion").AsInt == 2), + 1.0f), Times.Once); + } + + [Fact] + public void TrackDataOmitsModelKeyWhenEmpty() + { + var mockClient = new Mock(); + var context = Context.New("key"); + + var tracker = new LdAiConfigTracker(mockClient.Object, "test-run-id", + "config-key", "variation-key", 3, context, "fakeModel", "fakeProvider", + modelKey: "", modelVersion: 3); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => + d.Get("modelVersion").AsInt == 3 && + d.Get("modelKey").IsNull), + 1.0f), Times.Once); + } + + [Fact] + public void FromResumptionTokenDefaultsModelVersionAndOmitsModelKey() + { + var mockClient = new Mock(); + var context = Context.New("key"); + + var payload = JsonSerializer.Serialize(new + { + runId = "test-run-id", + configKey = "test-key", + variationKey = "var-1", + version = 2, + }); + var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(payload)); + var token = base64.Replace('+', '-').Replace('/', '_').TrimEnd('='); + + var tracker = LdAiConfigTracker.FromResumptionToken(token, mockClient.Object, context); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => + d.Get("modelVersion").AsInt == 1 && + d.Get("modelKey").IsNull), + 1.0f), Times.Once); } [Fact]