From eeb92a55a92818b9d6f36b701758336cefbf0654 Mon Sep 17 00:00:00 2001 From: esrakartalOpt Date: Tue, 21 Jul 2026 15:42:40 -0500 Subject: [PATCH 1/9] [AI-FSSDK] [FSSDK-12735] Add holdout exclusion logic for Targeted Delivery rules --- OptimizelySwiftSDK.xcodeproj/project.pbxproj | 6 + Sources/Data Model/Holdout.swift | 9 +- .../DefaultDecisionService.swift | 71 ++++- Sources/Utils/LogMessage.swift | 4 +- ...rviceTests_ExcludeTargetedDeliveries.swift | 249 ++++++++++++++++++ 5 files changed, 326 insertions(+), 13 deletions(-) create mode 100644 Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift diff --git a/OptimizelySwiftSDK.xcodeproj/project.pbxproj b/OptimizelySwiftSDK.xcodeproj/project.pbxproj index 7aba56bd..f36163d9 100644 --- a/OptimizelySwiftSDK.xcodeproj/project.pbxproj +++ b/OptimizelySwiftSDK.xcodeproj/project.pbxproj @@ -2121,6 +2121,8 @@ 98AC985F2DBA6721001405DD /* OptimizelyUserContextTests_Decide_With_Holdouts_Reasons.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98AC985D2DBA6721001405DD /* OptimizelyUserContextTests_Decide_With_Holdouts_Reasons.swift */; }; 98C2DF242F900669003F2443 /* DecisionServiceTests_LocalHoldouts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98C2DF232F900669003F2443 /* DecisionServiceTests_LocalHoldouts.swift */; }; 98C2DF252F900669003F2443 /* DecisionServiceTests_LocalHoldouts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98C2DF232F900669003F2443 /* DecisionServiceTests_LocalHoldouts.swift */; }; + E793710B634841BF89AA931E /* DecisionServiceTests_ExcludeTargetedDeliveries.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6EA2AD71FDC644278FE20412 /* DecisionServiceTests_ExcludeTargetedDeliveries.swift */; }; + BC46ADF752734A65854A6676 /* DecisionServiceTests_ExcludeTargetedDeliveries.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6EA2AD71FDC644278FE20412 /* DecisionServiceTests_ExcludeTargetedDeliveries.swift */; }; 98D5AE842DBB91C0000D5844 /* OptimizelyUserContextTests_Decide_Holdouts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98D5AE832DBB91C0000D5844 /* OptimizelyUserContextTests_Decide_Holdouts.swift */; }; 98D5AE852DBB91C0000D5844 /* OptimizelyUserContextTests_Decide_Holdouts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98D5AE832DBB91C0000D5844 /* OptimizelyUserContextTests_Decide_Holdouts.swift */; }; 98F28A1D2E01940500A86546 /* Cmab.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98F28A1C2E01940500A86546 /* Cmab.swift */; }; @@ -2633,6 +2635,7 @@ 98AC98482DB8FC29001405DD /* DecisionServiceTests_Holdouts.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DecisionServiceTests_Holdouts.swift; sourceTree = ""; }; 98AC985D2DBA6721001405DD /* OptimizelyUserContextTests_Decide_With_Holdouts_Reasons.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OptimizelyUserContextTests_Decide_With_Holdouts_Reasons.swift; sourceTree = ""; }; 98C2DF232F900669003F2443 /* DecisionServiceTests_LocalHoldouts.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DecisionServiceTests_LocalHoldouts.swift; sourceTree = ""; }; + 6EA2AD71FDC644278FE20412 /* DecisionServiceTests_ExcludeTargetedDeliveries.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DecisionServiceTests_ExcludeTargetedDeliveries.swift; sourceTree = ""; }; 98D5AE832DBB91C0000D5844 /* OptimizelyUserContextTests_Decide_Holdouts.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OptimizelyUserContextTests_Decide_Holdouts.swift; sourceTree = ""; }; 98F28A1C2E01940500A86546 /* Cmab.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Cmab.swift; sourceTree = ""; }; 98F28A2D2E01968000A86546 /* CmabTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CmabTests.swift; sourceTree = ""; }; @@ -3171,6 +3174,7 @@ 6E75198B22C5211100B2B157 /* NotificationCenterTests.swift */, 84861810286D0B8900B7F41B /* OdpEventManagerTests.swift */, 98C2DF232F900669003F2443 /* DecisionServiceTests_LocalHoldouts.swift */, + 6EA2AD71FDC644278FE20412 /* DecisionServiceTests_ExcludeTargetedDeliveries.swift */, 8486180E286D0B8900B7F41B /* OdpManagerTests.swift */, 8486180D286D0B8900B7F41B /* OdpSegmentManagerTests.swift */, 98F28A512E02E81500A86546 /* CMABClientTests.swift */, @@ -5138,6 +5142,7 @@ 6E75187922C520D400B2B157 /* Variation.swift in Sources */, 6E75191522C520D500B2B157 /* BackgroundingCallbacks.swift in Sources */, 98C2DF242F900669003F2443 /* DecisionServiceTests_LocalHoldouts.swift in Sources */, + E793710B634841BF89AA931E /* DecisionServiceTests_ExcludeTargetedDeliveries.swift in Sources */, 6E75195D22C520D500B2B157 /* OPTBucketer.swift in Sources */, 6E9B117622C5487100C22D81 /* DatafileHandlerTests.swift in Sources */, 84E2E97F2855875E001114AB /* OdpEventManager.swift in Sources */, @@ -5443,6 +5448,7 @@ 6E7516FD22C520D400B2B157 /* OptimizelyLogLevel.swift in Sources */, 6E75187322C520D400B2B157 /* Variation.swift in Sources */, 98C2DF252F900669003F2443 /* DecisionServiceTests_LocalHoldouts.swift in Sources */, + BC46ADF752734A65854A6676 /* DecisionServiceTests_ExcludeTargetedDeliveries.swift in Sources */, 6E7517E322C520D400B2B157 /* DefaultDecisionService.swift in Sources */, 6E75179922C520D400B2B157 /* DataStoreQueueStackImpl+Extension.swift in Sources */, 6E9B115C22C5486E00C22D81 /* DatafileHandlerTests.swift in Sources */, diff --git a/Sources/Data Model/Holdout.swift b/Sources/Data Model/Holdout.swift index b6da58cb..9e037f48 100644 --- a/Sources/Data Model/Holdout.swift +++ b/Sources/Data Model/Holdout.swift @@ -1,5 +1,5 @@ // -// Copyright 2022, Optimizely, Inc. and contributors +// Copyright 2022, 2026, Optimizely, Inc. and contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -32,9 +32,10 @@ struct Holdout: Codable, ExperimentCore { var audienceIds: [String] var audienceConditions: ConditionHolder? var includedRules: [String]? + var excludeTargetedDeliveries: Bool enum CodingKeys: String, CodingKey { - case id, key, status, variations, trafficAllocation, audienceIds, audienceConditions, includedRules + case id, key, status, variations, trafficAllocation, audienceIds, audienceConditions, includedRules, excludeTargetedDeliveries } var variationsMap: [String: OptimizelyVariation] = [:] @@ -55,6 +56,7 @@ struct Holdout: Codable, ExperimentCore { audienceConditions = try container.decodeIfPresent(ConditionHolder.self, forKey: .audienceConditions) includedRules = try container.decodeIfPresent([String].self, forKey: .includedRules) + excludeTargetedDeliveries = try container.decodeIfPresent(Bool.self, forKey: .excludeTargetedDeliveries) ?? false } } @@ -67,7 +69,8 @@ extension Holdout: Equatable { lhs.trafficAllocation == rhs.trafficAllocation && lhs.audienceIds == rhs.audienceIds && lhs.audienceConditions == rhs.audienceConditions && - lhs.includedRules == rhs.includedRules + lhs.includedRules == rhs.includedRules && + lhs.excludeTargetedDeliveries == rhs.excludeTargetedDeliveries } } diff --git a/Sources/Implementation/DefaultDecisionService.swift b/Sources/Implementation/DefaultDecisionService.swift index f2210635..d655df6f 100644 --- a/Sources/Implementation/DefaultDecisionService.swift +++ b/Sources/Implementation/DefaultDecisionService.swift @@ -1,5 +1,5 @@ // -// Copyright 2019-2022, Optimizely, Inc. and contributors +// Copyright 2019-2022, 2026, Optimizely, Inc. and contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -400,6 +400,8 @@ class DefaultDecisionService: OPTDecisionService { options: [OptimizelyDecideOption]? = nil) -> DecisionResponse { let reasons = DecisionReasons(options: options) + var storedHoldoutDecision: FeatureDecision? + let holdouts = config.getGlobalHoldouts() for holdout in holdouts { let holdoutDecision = getVariationForHoldout(config: config, @@ -410,21 +412,62 @@ class DefaultDecisionService: OPTDecisionService { reasons.merge(holdoutDecision.reasons) if let variation = holdoutDecision.result { let featureDecision = FeatureDecision(experiment: holdout, variation: variation, source: Constants.DecisionSource.holdout.rawValue) - return DecisionResponse(result: featureDecision, reasons: reasons) + if holdout.excludeTargetedDeliveries { + storedHoldoutDecision = featureDecision + } else { + return DecisionResponse(result: featureDecision, reasons: reasons) + } + break } } - + + if let storedHoldout = storedHoldoutDecision { + let experimentIds = featureFlag.experimentIds + for experimentId in experimentIds { + if let experiment = config.getExperiment(id: experimentId) { + if experiment.type == .targetedDelivery { + let decisionResponse = getVariationFromExperimentRule(config: config, + flagKey: featureFlag.key, + rule: experiment, + user: user, + userProfileTracker: userProfileTracker, + isAsync: isAsync, + options: options) + reasons.merge(decisionResponse.reasons) + if let result = decisionResponse.result { + if result.cmabError { + let featureDecision = FeatureDecision(experiment: experiment, variation: nil, source: Constants.DecisionSource.featureTest.rawValue, error: true) + return DecisionResponse(result: featureDecision, reasons: reasons) + } else if let variation = result.variation { + if let holdout = result.holdout { + let featureDecision = FeatureDecision(experiment: holdout, variation: variation, source: Constants.DecisionSource.holdout.rawValue, cmabUUID: result.cmabUUID) + return DecisionResponse(result: featureDecision, reasons: reasons) + } else { + let featureDecision = FeatureDecision(experiment: experiment, variation: variation, source: Constants.DecisionSource.featureTest.rawValue, cmabUUID: result.cmabUUID) + return DecisionResponse(result: featureDecision, reasons: reasons) + } + } + } + } else { + return DecisionResponse(result: storedHoldout, reasons: reasons) + } + } + } + + return DecisionResponse(result: storedHoldout, reasons: reasons) + } + let flagExpDecision = getVariationForFeatureExperiments(config: config, featureFlag: featureFlag, user: user, userProfileTracker: userProfileTracker, isAsync: isAsync, options: options) - + reasons.merge(flagExpDecision.reasons) - + if let decision = flagExpDecision.result { return DecisionResponse(result: decision, reasons: reasons) } - + let rolloutDecision = getVariationForFeatureRollout(config: config, featureFlag: featureFlag, user: user, options: options) reasons.merge(rolloutDecision.reasons) - + if let decision = rolloutDecision.result { return DecisionResponse(result: decision, reasons: reasons) } else { @@ -660,6 +703,12 @@ class DefaultDecisionService: OPTDecisionService { // check local holdouts targeting this rule let localHoldouts = config.getHoldoutsForRule(ruleId: rule.id) for holdout in localHoldouts { + if holdout.excludeTargetedDeliveries && rule.type == .targetedDelivery { + let info = LogMessage.holdoutExcludesTargetedDelivery(holdout.key, rule.key) + logger.d(info) + reasons.addInfo(info) + continue + } let holdoutDecision = getVariationForHoldout(config: config, flagKey: flagKey, holdout: holdout, @@ -667,7 +716,6 @@ class DefaultDecisionService: OPTDecisionService { options: options) reasons.merge(holdoutDecision.reasons) if let variation = holdoutDecision.result { - // User is in holdout — return holdout variation immediately, skip this rule let variationDecision = VariationDecision(variation: variation, holdout: holdout) return DecisionResponse(result: variationDecision, reasons: reasons) } @@ -718,6 +766,12 @@ class DefaultDecisionService: OPTDecisionService { // check local holdouts targeting this delivery rule let localHoldouts = config.getHoldoutsForRule(ruleId: rule.id) for holdout in localHoldouts { + if holdout.excludeTargetedDeliveries && rule.type == .targetedDelivery { + let info = LogMessage.holdoutExcludesTargetedDelivery(holdout.key, rule.key) + logger.d(info) + reasons.addInfo(info) + continue + } let holdoutDecision = getVariationForHoldout(config: config, flagKey: flagKey, holdout: holdout, @@ -725,7 +779,6 @@ class DefaultDecisionService: OPTDecisionService { options: options) reasons.merge(holdoutDecision.reasons) if let variation = holdoutDecision.result { - // User is in holdout — return holdout variation with holdout info let decision = DeliveryRuleDecision(variation: variation, skipToEveryoneElse: skipToEveryoneElse, holdout: holdout) return DecisionResponse(result: decision, reasons: reasons) } diff --git a/Sources/Utils/LogMessage.swift b/Sources/Utils/LogMessage.swift index 9b0dfee8..05745b54 100644 --- a/Sources/Utils/LogMessage.swift +++ b/Sources/Utils/LogMessage.swift @@ -1,5 +1,5 @@ // -// Copyright 2019-2021, 2023 Optimizely, Inc. and contributors +// Copyright 2019-2021, 2023, 2026 Optimizely, Inc. and contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -79,6 +79,7 @@ enum LogMessage { case cmabFetchSuccess(_ variationId: String, _ cmabUUID: String, _expKey: String) case cmabFetchFailed(_ expKey: String) case cmabNotSupportedInSyncMode + case holdoutExcludesTargetedDelivery(_ holdoutKey: String, _ ruleKey: String) } extension LogMessage: CustomStringConvertible { @@ -152,6 +153,7 @@ extension LogMessage: CustomStringConvertible { case .cmabFetchSuccess(let variationId, let cmabUUID, let expKey): message = "Successfully fetched CMAB decision, variationId: \(variationId), cmabUUID: \(cmabUUID) for experiment \(expKey)." case .cmabFetchFailed(let key): message = "Failed to fetch CMAB data for experiment \(key)." case .cmabNotSupportedInSyncMode: message = "CMAB is not supported in sync mode." + case .holdoutExcludesTargetedDelivery(let holdoutKey, let ruleKey): message = "Holdout (\(holdoutKey)) excludes targeted deliveries, skipping holdout for rule (\(ruleKey))." } return message diff --git a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift new file mode 100644 index 00000000..7efd6344 --- /dev/null +++ b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift @@ -0,0 +1,249 @@ +// +// Copyright 2026, Optimizely, Inc. and contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest + +class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { + + var optimizely: OptimizelyClient! + var config: ProjectConfig! + + let userId = "test_user" + let flagKey = "feature_1" + let experimentRuleId = "10390977673" + let deliveryRuleId = "3332020515" + + var sampleHoldout: [String: Any] { + return [ + "status": "Running", + "id": "9999900010", + "key": "holdout_test_key", + "trafficAllocation": [ + ["entityId": "9999900020", "endOfRange": 5000] + ], + "audienceIds": [], + "variations": [ + [ + "variables": [], + "id": "9999900020", + "key": "holdout_variation_key", + "featureEnabled": false + ] + ] + ] + } + + override func setUp() { + super.setUp() + optimizely = OTUtils.createOptimizely(datafileName: "decide_datafile", + clearUserProfileService: true) + config = optimizely.config! + } + + // MARK: - excludeTargetedDeliveries = false (default) + + func testExcludeTargetedDeliveriesFalse_HoldoutApplesToTDRule() { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = [experimentRuleId] + holdout.excludeTargetedDeliveries = false + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) + + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let user = optimizely.createUserContext(userId: userId) + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.variationKey, "holdout_variation_key") + XCTAssertFalse(decision.enabled) + } + + // MARK: - excludeTargetedDeliveries = true with TD rule + + func testExcludeTargetedDeliveriesTrue_TDRule_HoldoutSkipped() { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = [experimentRuleId] + holdout.excludeTargetedDeliveries = true + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) + + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let user = optimizely.createUserContext(userId: userId) + let decision = user.decide(key: flagKey) + + XCTAssertNotEqual(decision.variationKey, "holdout_variation_key") + } + + // MARK: - excludeTargetedDeliveries = true with AB rule + + func testExcludeTargetedDeliveriesTrue_ABRule_HoldoutStillApplies() { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = [experimentRuleId] + holdout.excludeTargetedDeliveries = true + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) + + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .ab + config.project.experiments = [experiment] + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let user = optimizely.createUserContext(userId: userId) + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.variationKey, "holdout_variation_key") + XCTAssertFalse(decision.enabled) + } + + // MARK: - Backward compatibility (field missing from JSON) + + func testExcludeTargetedDeliveries_MissingFromJSON_DefaultsFalse() { + var holdoutData = sampleHoldout + holdoutData.removeValue(forKey: "excludeTargetedDeliveries") + holdoutData["includedRules"] = [experimentRuleId] + let holdout = try! OTUtils.model(from: holdoutData) as Holdout + + XCTAssertFalse(holdout.excludeTargetedDeliveries) + } + + func testExcludeTargetedDeliveries_MissingFromJSON_HoldoutAppliesNormally() { + var holdoutData = sampleHoldout + holdoutData.removeValue(forKey: "excludeTargetedDeliveries") + holdoutData["includedRules"] = [experimentRuleId] + let holdout = try! OTUtils.model(from: holdoutData) as Holdout + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) + + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let user = optimizely.createUserContext(userId: userId) + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.variationKey, "holdout_variation_key") + XCTAssertFalse(decision.enabled) + } + + // MARK: - Global holdout with excludeTargetedDeliveries true and TD rule + + func testGlobalHoldout_ExcludeTargetedDeliveriesTrue_TDRule_HoldoutSkippedForTD() { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = nil + holdout.excludeTargetedDeliveries = true + config.holdoutConfig = HoldoutConfig(globalHoldouts: [holdout], localHoldouts: []) + + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let user = optimizely.createUserContext(userId: userId) + let decision = user.decide(key: flagKey) + + XCTAssertNotEqual(decision.variationKey, "holdout_variation_key") + } + + // MARK: - Local holdout with excludeTargetedDeliveries true and TD delivery rule + + func testLocalHoldout_ExcludeTargetedDeliveriesTrue_DeliveryTDRule_HoldoutSkipped() { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = [deliveryRuleId] + holdout.excludeTargetedDeliveries = true + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) + + if var rollout = config.getRollout(id: config.getFeatureFlag(key: flagKey)!.rolloutId) { + if rollout.experiments.count > 0 { + rollout.experiments[0].type = .targetedDelivery + config.project.rollouts = [rollout] + } + } + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let user = optimizely.createUserContext(userId: userId) + let decision = user.decide(key: flagKey) + + XCTAssertNotEqual(decision.variationKey, "holdout_variation_key") + } + + // MARK: - Forced decision beats 100% traffic local holdout + + func testForcedDecisionBeats100PercentLocalHoldout_ExcludeTargetedDeliveriesTrue() { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = [experimentRuleId] + holdout.excludeTargetedDeliveries = true + holdout.trafficAllocation = [TrafficAllocation(entityId: "9999900020", endOfRange: 10000)] + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let user = optimizely.createUserContext(userId: userId) + + let context = OptimizelyDecisionContext(flagKey: flagKey, ruleKey: nil) + let forcedDecision = OptimizelyForcedDecision(variationKey: "a") + user.setForcedDecision(context: context, decision: forcedDecision) + + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.variationKey, "a", "Forced decision should take precedence over 100% traffic local holdout") + } + + func testForcedDecisionBeats100PercentLocalHoldout_ExcludeTargetedDeliveriesFalse() { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = [experimentRuleId] + holdout.excludeTargetedDeliveries = false + holdout.trafficAllocation = [TrafficAllocation(entityId: "9999900020", endOfRange: 10000)] + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let user = optimizely.createUserContext(userId: userId) + + let context = OptimizelyDecisionContext(flagKey: flagKey, ruleKey: nil) + let forcedDecision = OptimizelyForcedDecision(variationKey: "a") + user.setForcedDecision(context: context, decision: forcedDecision) + + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.variationKey, "a", "Forced decision should take precedence over 100% traffic local holdout") + } +} From 1d6f7714b38ec920f7d6d4739ab15ccd7883bfe3 Mon Sep 17 00:00:00 2001 From: esrakartalOpt Date: Wed, 22 Jul 2026 14:31:03 -0500 Subject: [PATCH 2/9] [AI-FSSDK] [FSSDK-12735] Fix holdout exclusion for global-only scope, null TD fallback, and holdout event dispatch --- .../DefaultDecisionService.swift | 21 ++--- .../OptimizelyClient+Decide.swift | 14 ++- ...rviceTests_ExcludeTargetedDeliveries.swift | 88 ++++++++++++++++++- 3 files changed, 103 insertions(+), 20 deletions(-) diff --git a/Sources/Implementation/DefaultDecisionService.swift b/Sources/Implementation/DefaultDecisionService.swift index d655df6f..7b3d9dee 100644 --- a/Sources/Implementation/DefaultDecisionService.swift +++ b/Sources/Implementation/DefaultDecisionService.swift @@ -22,6 +22,7 @@ struct FeatureDecision { let source: String var cmabUUID: String? var error = false + var holdoutToSend: (experiment: ExperimentCore, variation: Variation)? = nil } struct VariationDecision { @@ -443,7 +444,11 @@ class DefaultDecisionService: OPTDecisionService { let featureDecision = FeatureDecision(experiment: holdout, variation: variation, source: Constants.DecisionSource.holdout.rawValue, cmabUUID: result.cmabUUID) return DecisionResponse(result: featureDecision, reasons: reasons) } else { - let featureDecision = FeatureDecision(experiment: experiment, variation: variation, source: Constants.DecisionSource.featureTest.rawValue, cmabUUID: result.cmabUUID) + var featureDecision = FeatureDecision(experiment: experiment, variation: variation, source: Constants.DecisionSource.featureTest.rawValue, cmabUUID: result.cmabUUID) + if let holdoutExperiment = storedHoldout.experiment, + let holdoutVariation = storedHoldout.variation { + featureDecision.holdoutToSend = (experiment: holdoutExperiment, variation: holdoutVariation) + } return DecisionResponse(result: featureDecision, reasons: reasons) } } @@ -454,7 +459,7 @@ class DefaultDecisionService: OPTDecisionService { } } - return DecisionResponse(result: storedHoldout, reasons: reasons) + return DecisionResponse(result: nil, reasons: reasons) } let flagExpDecision = getVariationForFeatureExperiments(config: config, featureFlag: featureFlag, user: user, userProfileTracker: userProfileTracker, isAsync: isAsync, options: options) @@ -703,12 +708,6 @@ class DefaultDecisionService: OPTDecisionService { // check local holdouts targeting this rule let localHoldouts = config.getHoldoutsForRule(ruleId: rule.id) for holdout in localHoldouts { - if holdout.excludeTargetedDeliveries && rule.type == .targetedDelivery { - let info = LogMessage.holdoutExcludesTargetedDelivery(holdout.key, rule.key) - logger.d(info) - reasons.addInfo(info) - continue - } let holdoutDecision = getVariationForHoldout(config: config, flagKey: flagKey, holdout: holdout, @@ -766,12 +765,6 @@ class DefaultDecisionService: OPTDecisionService { // check local holdouts targeting this delivery rule let localHoldouts = config.getHoldoutsForRule(ruleId: rule.id) for holdout in localHoldouts { - if holdout.excludeTargetedDeliveries && rule.type == .targetedDelivery { - let info = LogMessage.holdoutExcludesTargetedDelivery(holdout.key, rule.key) - logger.d(info) - reasons.addInfo(info) - continue - } let holdoutDecision = getVariationForHoldout(config: config, flagKey: flagKey, holdout: holdout, diff --git a/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift b/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift index fad9141e..a479bec2 100644 --- a/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift +++ b/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift @@ -1,5 +1,5 @@ // -// Copyright 2021-2022, Optimizely, Inc. and contributors +// Copyright 2021-2022, 2026, Optimizely, Inc. and contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -324,8 +324,18 @@ extension OptimizelyClient { cmabUUID: flagDecision?.cmabUUID) decisionEventDispatched = true } + if let holdoutInfo = flagDecision?.holdoutToSend { + sendImpressionEvent(experiment: holdoutInfo.experiment, + variation: holdoutInfo.variation, + userId: userId, + attributes: attributes, + flagKey: feature.key, + ruleType: Constants.DecisionSource.holdout.rawValue, + enabled: false, + cmabUUID: nil) + } } - + var variableMap = [String: Any]() if !allOptions.contains(.excludeVariables) { let decisionResponse = getDecisionVariableMap(feature: feature, diff --git a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift index 7efd6344..82154189 100644 --- a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift +++ b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift @@ -78,7 +78,7 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { // MARK: - excludeTargetedDeliveries = true with TD rule - func testExcludeTargetedDeliveriesTrue_TDRule_HoldoutSkipped() { + func testExcludeTargetedDeliveriesTrue_TDRule_LocalHoldoutStillApplies() { var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout holdout.includedRules = [experimentRuleId] holdout.excludeTargetedDeliveries = true @@ -95,7 +95,8 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { let user = optimizely.createUserContext(userId: userId) let decision = user.decide(key: flagKey) - XCTAssertNotEqual(decision.variationKey, "holdout_variation_key") + XCTAssertEqual(decision.variationKey, "holdout_variation_key") + XCTAssertFalse(decision.enabled) } // MARK: - excludeTargetedDeliveries = true with AB rule @@ -178,7 +179,7 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { // MARK: - Local holdout with excludeTargetedDeliveries true and TD delivery rule - func testLocalHoldout_ExcludeTargetedDeliveriesTrue_DeliveryTDRule_HoldoutSkipped() { + func testLocalHoldout_ExcludeTargetedDeliveriesTrue_DeliveryTDRule_HoldoutStillApplies() { var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout holdout.includedRules = [deliveryRuleId] holdout.excludeTargetedDeliveries = true @@ -198,7 +199,8 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { let user = optimizely.createUserContext(userId: userId) let decision = user.decide(key: flagKey) - XCTAssertNotEqual(decision.variationKey, "holdout_variation_key") + XCTAssertEqual(decision.variationKey, "holdout_variation_key") + XCTAssertFalse(decision.enabled) } // MARK: - Forced decision beats 100% traffic local holdout @@ -246,4 +248,82 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { XCTAssertEqual(decision.variationKey, "a", "Forced decision should take precedence over 100% traffic local holdout") } + + // MARK: - Global holdout excludeTrue, TD returns null => returns null + + func testGlobalHoldout_ExcludeTrue_TDReturnsNull_ReturnsNull() { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = nil + holdout.excludeTargetedDeliveries = true + config.holdoutConfig = HoldoutConfig(globalHoldouts: [holdout], localHoldouts: []) + + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let user = optimizely.createUserContext(userId: userId) + let decision = user.decide(key: flagKey) + + XCTAssertNil(decision.variationKey) + } + + // MARK: - Local holdout with excludeTrue still applies + + func testLocalHoldout_ExcludeTrue_StillApplies() { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = [experimentRuleId] + holdout.excludeTargetedDeliveries = true + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) + + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let user = optimizely.createUserContext(userId: userId) + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.variationKey, "holdout_variation_key") + XCTAssertFalse(decision.enabled) + } + + // MARK: - Global holdout excludeTrue, TD matches => holdoutToSend populated + + func testGlobalHoldout_ExcludeTrue_HoldoutEventSent() { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = nil + holdout.excludeTargetedDeliveries = true + config.holdoutConfig = HoldoutConfig(globalHoldouts: [holdout], localHoldouts: []) + + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + experiment.audienceIds = [] + experiment.audienceConditions = nil + config.project.experiments = [experiment] + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let user = optimizely.createUserContext(userId: userId) + let featureFlag = config.getFeatureFlag(key: flagKey)! + let response = mockDecisionService.getDecisionForFlag(config: config, + featureFlag: featureFlag, + user: user, + isAsync: false, + options: nil) + + XCTAssertNotNil(response.result) + XCTAssertEqual(response.result?.source, Constants.DecisionSource.featureTest.rawValue) + XCTAssertNotNil(response.result?.holdoutToSend) + XCTAssertEqual(response.result?.holdoutToSend?.experiment.key, "holdout_test_key") + XCTAssertEqual(response.result?.holdoutToSend?.variation.key, "holdout_variation_key") + } } From 17e621c958e58225bf302539d7c57178bc63686f Mon Sep 17 00:00:00 2001 From: esrakartalOpt Date: Wed, 22 Jul 2026 16:43:22 -0500 Subject: [PATCH 3/9] [AI-FSSDK] [FSSDK-12735] Fix holdout decisionEventDispatched and add excludeTargetedDeliveries reason --- .../DefaultDecisionService.swift | 2 + .../OptimizelyClient+Decide.swift | 1 + Sources/Utils/LogMessage.swift | 2 + ...rviceTests_ExcludeTargetedDeliveries.swift | 56 +++++++++++++++++++ 4 files changed, 61 insertions(+) diff --git a/Sources/Implementation/DefaultDecisionService.swift b/Sources/Implementation/DefaultDecisionService.swift index 7b3d9dee..e45a59ce 100644 --- a/Sources/Implementation/DefaultDecisionService.swift +++ b/Sources/Implementation/DefaultDecisionService.swift @@ -415,6 +415,8 @@ class DefaultDecisionService: OPTDecisionService { let featureDecision = FeatureDecision(experiment: holdout, variation: variation, source: Constants.DecisionSource.holdout.rawValue) if holdout.excludeTargetedDeliveries { storedHoldoutDecision = featureDecision + let info = LogMessage.holdoutExcludeTargetedDeliveriesEnabled(holdout.key) + reasons.addInfo(info) } else { return DecisionResponse(result: featureDecision, reasons: reasons) } diff --git a/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift b/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift index a479bec2..3794c3f8 100644 --- a/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift +++ b/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift @@ -333,6 +333,7 @@ extension OptimizelyClient { ruleType: Constants.DecisionSource.holdout.rawValue, enabled: false, cmabUUID: nil) + decisionEventDispatched = true } } diff --git a/Sources/Utils/LogMessage.swift b/Sources/Utils/LogMessage.swift index 05745b54..aa1720f6 100644 --- a/Sources/Utils/LogMessage.swift +++ b/Sources/Utils/LogMessage.swift @@ -80,6 +80,7 @@ enum LogMessage { case cmabFetchFailed(_ expKey: String) case cmabNotSupportedInSyncMode case holdoutExcludesTargetedDelivery(_ holdoutKey: String, _ ruleKey: String) + case holdoutExcludeTargetedDeliveriesEnabled(_ holdoutKey: String) } extension LogMessage: CustomStringConvertible { @@ -154,6 +155,7 @@ extension LogMessage: CustomStringConvertible { case .cmabFetchFailed(let key): message = "Failed to fetch CMAB data for experiment \(key)." case .cmabNotSupportedInSyncMode: message = "CMAB is not supported in sync mode." case .holdoutExcludesTargetedDelivery(let holdoutKey, let ruleKey): message = "Holdout (\(holdoutKey)) excludes targeted deliveries, skipping holdout for rule (\(ruleKey))." + case .holdoutExcludeTargetedDeliveriesEnabled(let holdoutKey): message = "Holdout '\(holdoutKey)' has excludeTargetedDeliveries enabled, continuing to rollout evaluation." } return message diff --git a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift index 82154189..63f930a7 100644 --- a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift +++ b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift @@ -294,6 +294,62 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { XCTAssertFalse(decision.enabled) } + // MARK: - Decision reason for excludeTargetedDeliveries bypass + + func testGlobalHoldout_ExcludeTrue_DecisionReasonPresent() { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = nil + holdout.excludeTargetedDeliveries = true + config.holdoutConfig = HoldoutConfig(globalHoldouts: [holdout], localHoldouts: []) + + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let user = optimizely.createUserContext(userId: userId) + let decision = user.decide(key: flagKey, options: [.includeReasons]) + + let reasons = decision.reasons + let expectedReason = "Holdout 'holdout_test_key' has excludeTargetedDeliveries enabled, continuing to rollout evaluation." + XCTAssertTrue(reasons.contains(expectedReason), + "Decision reasons should contain excludeTargetedDeliveries bypass reason. Got: \(reasons)") + } + + // MARK: - decisionEventDispatched for holdout impression + + func testGlobalHoldout_ExcludeTrue_TDMatch_DecisionEventDispatched() { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = nil + holdout.excludeTargetedDeliveries = true + config.holdoutConfig = HoldoutConfig(globalHoldouts: [holdout], localHoldouts: []) + + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + experiment.audienceIds = [] + experiment.audienceConditions = nil + config.project.experiments = [experiment] + + let mockBucketer = MockBucketer(mockBucketValue: 2500) + let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) + optimizely.decisionService = mockDecisionService + + let exp = expectation(description: "decision notification") + let user = optimizely.createUserContext(userId: userId) + + optimizely.notificationCenter?.addDecisionNotificationListener { (_, _, _, decisionInfo) in + let dispatched = decisionInfo[Constants.DecisionInfoKeys.decisionEventDispatched] as? Bool + XCTAssertEqual(dispatched, true, "decisionEventDispatched should be true when holdout impression is sent") + exp.fulfill() + } + + _ = user.decide(key: flagKey) + wait(for: [exp], timeout: 1) + } + // MARK: - Global holdout excludeTrue, TD matches => holdoutToSend populated func testGlobalHoldout_ExcludeTrue_HoldoutEventSent() { From 86fd075366595d7977a84e136bf1b18f77b0ee84 Mon Sep 17 00:00:00 2001 From: Muzahidul Islam Date: Tue, 28 Jul 2026 20:23:58 +0600 Subject: [PATCH 4/9] [AI-FSSDK] [FSSDK-12735] Implement excludeTargetedDeliveries for global holdouts - Refactor getDecisionForFlag: skip experiment rules when global holdout has excludeTargetedDeliveries=true, always evaluate rollout (delivery) rules - Attach holdoutDecision to rollout result so holdout impression fires alongside the rollout impression; attach to empty decision when no rollout matches so the holdout impression still fires - Add HoldoutDecision struct replacing the anonymous holdoutToSend tuple; failable init?(from:) absorbs the unwrapping logic previously spread across call sites and the holdoutTuple helper - Add userBucketedIntoRollout / userNotBucketedIntoRollout log messages matching Java SDK parity - Refactor getDecisionForFlag and extract getTargetedDeliveryExperiments on ProjectConfig - Fix DecisionServiceTests_ExcludeTargetedDeliveries: correct test setup ordering (project mutations before holdoutConfig assignment to prevent didSet from wiping the custom holdout), update assertions to match correct spec (experiments skipped, delivery/rollout rules always evaluated) Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../DefaultDecisionService.swift | 84 +++++------- .../OptimizelyClient+Decide.swift | 2 +- Sources/Utils/LogMessage.swift | 4 + ...rviceTests_ExcludeTargetedDeliveries.swift | 120 ++++++++++-------- 4 files changed, 108 insertions(+), 102 deletions(-) diff --git a/Sources/Implementation/DefaultDecisionService.swift b/Sources/Implementation/DefaultDecisionService.swift index e45a59ce..5cb71497 100644 --- a/Sources/Implementation/DefaultDecisionService.swift +++ b/Sources/Implementation/DefaultDecisionService.swift @@ -16,13 +16,25 @@ import Foundation +struct HoldoutDecision { + let experiment: ExperimentCore + let variation: Variation + + init?(from featureDecision: FeatureDecision?) { + guard let experiment = featureDecision?.experiment, + let variation = featureDecision?.variation else { return nil } + self.experiment = experiment + self.variation = variation + } +} + struct FeatureDecision { var experiment: ExperimentCore? let variation: Variation? let source: String var cmabUUID: String? var error = false - var holdoutToSend: (experiment: ExperimentCore, variation: Variation)? = nil + var holdoutDecision: HoldoutDecision? = nil } struct VariationDecision { @@ -423,61 +435,35 @@ class DefaultDecisionService: OPTDecisionService { break } } + + if storedHoldoutDecision == nil { + let flagExpDecision = getVariationForFeatureExperiments(config: config, featureFlag: featureFlag, user: user, userProfileTracker: userProfileTracker, isAsync: isAsync, options: options) - if let storedHoldout = storedHoldoutDecision { - let experimentIds = featureFlag.experimentIds - for experimentId in experimentIds { - if let experiment = config.getExperiment(id: experimentId) { - if experiment.type == .targetedDelivery { - let decisionResponse = getVariationFromExperimentRule(config: config, - flagKey: featureFlag.key, - rule: experiment, - user: user, - userProfileTracker: userProfileTracker, - isAsync: isAsync, - options: options) - reasons.merge(decisionResponse.reasons) - if let result = decisionResponse.result { - if result.cmabError { - let featureDecision = FeatureDecision(experiment: experiment, variation: nil, source: Constants.DecisionSource.featureTest.rawValue, error: true) - return DecisionResponse(result: featureDecision, reasons: reasons) - } else if let variation = result.variation { - if let holdout = result.holdout { - let featureDecision = FeatureDecision(experiment: holdout, variation: variation, source: Constants.DecisionSource.holdout.rawValue, cmabUUID: result.cmabUUID) - return DecisionResponse(result: featureDecision, reasons: reasons) - } else { - var featureDecision = FeatureDecision(experiment: experiment, variation: variation, source: Constants.DecisionSource.featureTest.rawValue, cmabUUID: result.cmabUUID) - if let holdoutExperiment = storedHoldout.experiment, - let holdoutVariation = storedHoldout.variation { - featureDecision.holdoutToSend = (experiment: holdoutExperiment, variation: holdoutVariation) - } - return DecisionResponse(result: featureDecision, reasons: reasons) - } - } - } - } else { - return DecisionResponse(result: storedHoldout, reasons: reasons) - } - } - } - - return DecisionResponse(result: nil, reasons: reasons) - } - - let flagExpDecision = getVariationForFeatureExperiments(config: config, featureFlag: featureFlag, user: user, userProfileTracker: userProfileTracker, isAsync: isAsync, options: options) - - reasons.merge(flagExpDecision.reasons) + reasons.merge(flagExpDecision.reasons) - if let decision = flagExpDecision.result { - return DecisionResponse(result: decision, reasons: reasons) + if let decision = flagExpDecision.result { + return DecisionResponse(result: decision, reasons: reasons) + } } - + let rolloutDecision = getVariationForFeatureRollout(config: config, featureFlag: featureFlag, user: user, options: options) reasons.merge(rolloutDecision.reasons) - if let decision = rolloutDecision.result { + if var decision: FeatureDecision = rolloutDecision.result { + decision.holdoutDecision = HoldoutDecision(from: storedHoldoutDecision) + let info = LogMessage.userBucketedIntoRollout(user.userId, featureFlag.key) + logger.i(info) + reasons.addInfo(info) return DecisionResponse(result: decision, reasons: reasons) } else { + let info = LogMessage.userNotBucketedIntoRollout(user.userId, featureFlag.key) + logger.i(info) + reasons.addInfo(info) + if let holdout = HoldoutDecision(from: storedHoldoutDecision) { + var emptyDecision = FeatureDecision(experiment: nil, variation: nil, source: Constants.DecisionSource.rollout.rawValue) + emptyDecision.holdoutDecision = holdout + return DecisionResponse(result: emptyDecision, reasons: reasons) + } return DecisionResponse(result: nil, reasons: reasons) } } @@ -896,7 +882,7 @@ class DefaultDecisionService: OPTDecisionService { } // MARK: - Utilities - + /// Retrieves the bucketing ID for a user, defaulting to user ID unless overridden in attributes. /// - Parameters: /// - userId: The user's ID. diff --git a/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift b/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift index 3794c3f8..ee6cbc0d 100644 --- a/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift +++ b/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift @@ -324,7 +324,7 @@ extension OptimizelyClient { cmabUUID: flagDecision?.cmabUUID) decisionEventDispatched = true } - if let holdoutInfo = flagDecision?.holdoutToSend { + if let holdoutInfo = flagDecision?.holdoutDecision { sendImpressionEvent(experiment: holdoutInfo.experiment, variation: holdoutInfo.variation, userId: userId, diff --git a/Sources/Utils/LogMessage.swift b/Sources/Utils/LogMessage.swift index aa1720f6..a406839b 100644 --- a/Sources/Utils/LogMessage.swift +++ b/Sources/Utils/LogMessage.swift @@ -81,6 +81,8 @@ enum LogMessage { case cmabNotSupportedInSyncMode case holdoutExcludesTargetedDelivery(_ holdoutKey: String, _ ruleKey: String) case holdoutExcludeTargetedDeliveriesEnabled(_ holdoutKey: String) + case userBucketedIntoRollout(_ userId: String, _ flagKey: String) + case userNotBucketedIntoRollout(_ userId: String, _ flagKey: String) } extension LogMessage: CustomStringConvertible { @@ -156,6 +158,8 @@ extension LogMessage: CustomStringConvertible { case .cmabNotSupportedInSyncMode: message = "CMAB is not supported in sync mode." case .holdoutExcludesTargetedDelivery(let holdoutKey, let ruleKey): message = "Holdout (\(holdoutKey)) excludes targeted deliveries, skipping holdout for rule (\(ruleKey))." case .holdoutExcludeTargetedDeliveriesEnabled(let holdoutKey): message = "Holdout '\(holdoutKey)' has excludeTargetedDeliveries enabled, continuing to rollout evaluation." + case .userBucketedIntoRollout(let userId, let flagKey): message = "The user \"\(userId)\" was bucketed into a rollout for feature flag \"\(flagKey)\"." + case .userNotBucketedIntoRollout(let userId, let flagKey): message = "The user \"\(userId)\" was not bucketed into a rollout for feature flag \"\(flagKey)\"." } return message diff --git a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift index 63f930a7..676d7c81 100644 --- a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift +++ b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift @@ -56,15 +56,15 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { // MARK: - excludeTargetedDeliveries = false (default) func testExcludeTargetedDeliveriesFalse_HoldoutApplesToTDRule() { + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout holdout.includedRules = [experimentRuleId] holdout.excludeTargetedDeliveries = false config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) - var experiment = config.getExperiment(id: experimentRuleId)! - experiment.type = .targetedDelivery - config.project.experiments = [experiment] - let mockBucketer = MockBucketer(mockBucketValue: 2500) let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) optimizely.decisionService = mockDecisionService @@ -79,15 +79,15 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { // MARK: - excludeTargetedDeliveries = true with TD rule func testExcludeTargetedDeliveriesTrue_TDRule_LocalHoldoutStillApplies() { + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout holdout.includedRules = [experimentRuleId] holdout.excludeTargetedDeliveries = true config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) - var experiment = config.getExperiment(id: experimentRuleId)! - experiment.type = .targetedDelivery - config.project.experiments = [experiment] - let mockBucketer = MockBucketer(mockBucketValue: 2500) let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) optimizely.decisionService = mockDecisionService @@ -102,15 +102,15 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { // MARK: - excludeTargetedDeliveries = true with AB rule func testExcludeTargetedDeliveriesTrue_ABRule_HoldoutStillApplies() { + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .ab + config.project.experiments = [experiment] + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout holdout.includedRules = [experimentRuleId] holdout.excludeTargetedDeliveries = true config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) - var experiment = config.getExperiment(id: experimentRuleId)! - experiment.type = .ab - config.project.experiments = [experiment] - let mockBucketer = MockBucketer(mockBucketValue: 2500) let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) optimizely.decisionService = mockDecisionService @@ -134,16 +134,16 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { } func testExcludeTargetedDeliveries_MissingFromJSON_HoldoutAppliesNormally() { + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + var holdoutData = sampleHoldout holdoutData.removeValue(forKey: "excludeTargetedDeliveries") holdoutData["includedRules"] = [experimentRuleId] let holdout = try! OTUtils.model(from: holdoutData) as Holdout config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) - var experiment = config.getExperiment(id: experimentRuleId)! - experiment.type = .targetedDelivery - config.project.experiments = [experiment] - let mockBucketer = MockBucketer(mockBucketValue: 2500) let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) optimizely.decisionService = mockDecisionService @@ -180,11 +180,6 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { // MARK: - Local holdout with excludeTargetedDeliveries true and TD delivery rule func testLocalHoldout_ExcludeTargetedDeliveriesTrue_DeliveryTDRule_HoldoutStillApplies() { - var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout - holdout.includedRules = [deliveryRuleId] - holdout.excludeTargetedDeliveries = true - config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) - if var rollout = config.getRollout(id: config.getFeatureFlag(key: flagKey)!.rolloutId) { if rollout.experiments.count > 0 { rollout.experiments[0].type = .targetedDelivery @@ -192,6 +187,11 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { } } + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.includedRules = [deliveryRuleId] + holdout.excludeTargetedDeliveries = true + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) + let mockBucketer = MockBucketer(mockBucketValue: 2500) let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) optimizely.decisionService = mockDecisionService @@ -251,38 +251,52 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { // MARK: - Global holdout excludeTrue, TD returns null => returns null - func testGlobalHoldout_ExcludeTrue_TDReturnsNull_ReturnsNull() { + // When excludeTargetedDeliveries=true, experiment rules are skipped and delivery + // (rollout) rules are always evaluated. The Everyone Else rule matches so a + // rollout variation is returned and the holdout is queued for impression. + func testGlobalHoldout_ExcludeTrue_DeliveryRulesEvaluated_HoldoutQueued() { + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout holdout.includedRules = nil holdout.excludeTargetedDeliveries = true config.holdoutConfig = HoldoutConfig(globalHoldouts: [holdout], localHoldouts: []) - var experiment = config.getExperiment(id: experimentRuleId)! - experiment.type = .targetedDelivery - config.project.experiments = [experiment] - let mockBucketer = MockBucketer(mockBucketValue: 2500) let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) optimizely.decisionService = mockDecisionService let user = optimizely.createUserContext(userId: userId) - let decision = user.decide(key: flagKey) - - XCTAssertNil(decision.variationKey) + let featureFlag = config.getFeatureFlag(key: flagKey)! + let response = mockDecisionService.getDecisionForFlag(config: config, + featureFlag: featureFlag, + user: user, + isAsync: false, + options: nil) + + // Delivery (rollout) rules are evaluated — Everyone Else matches + XCTAssertNotNil(response.result?.variation) + XCTAssertEqual(response.result?.source, Constants.DecisionSource.rollout.rawValue) + // Holdout is queued so its impression fires via holdoutDecision + XCTAssertNotNil(response.result?.holdoutDecision) + XCTAssertEqual(response.result?.holdoutDecision?.experiment.key, "holdout_test_key") + XCTAssertEqual(response.result?.holdoutDecision?.variation.key, "holdout_variation_key") } // MARK: - Local holdout with excludeTrue still applies func testLocalHoldout_ExcludeTrue_StillApplies() { + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout holdout.includedRules = [experimentRuleId] holdout.excludeTargetedDeliveries = true config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) - var experiment = config.getExperiment(id: experimentRuleId)! - experiment.type = .targetedDelivery - config.project.experiments = [experiment] - let mockBucketer = MockBucketer(mockBucketValue: 2500) let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) optimizely.decisionService = mockDecisionService @@ -297,15 +311,15 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { // MARK: - Decision reason for excludeTargetedDeliveries bypass func testGlobalHoldout_ExcludeTrue_DecisionReasonPresent() { + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout holdout.includedRules = nil holdout.excludeTargetedDeliveries = true config.holdoutConfig = HoldoutConfig(globalHoldouts: [holdout], localHoldouts: []) - var experiment = config.getExperiment(id: experimentRuleId)! - experiment.type = .targetedDelivery - config.project.experiments = [experiment] - let mockBucketer = MockBucketer(mockBucketValue: 2500) let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) optimizely.decisionService = mockDecisionService @@ -350,20 +364,21 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { wait(for: [exp], timeout: 1) } - // MARK: - Global holdout excludeTrue, TD matches => holdoutToSend populated + // MARK: - Global holdout excludeTrue, TD matches => holdoutDecision populated + // When excludeTargetedDeliveries=true, experiment rules are skipped and the + // decision comes from the delivery (rollout) path. The holdout is queued via + // holdoutDecision so its impression fires alongside the rollout impression. func testGlobalHoldout_ExcludeTrue_HoldoutEventSent() { + var experiment = config.getExperiment(id: experimentRuleId)! + experiment.type = .targetedDelivery + config.project.experiments = [experiment] + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout holdout.includedRules = nil holdout.excludeTargetedDeliveries = true config.holdoutConfig = HoldoutConfig(globalHoldouts: [holdout], localHoldouts: []) - var experiment = config.getExperiment(id: experimentRuleId)! - experiment.type = .targetedDelivery - experiment.audienceIds = [] - experiment.audienceConditions = nil - config.project.experiments = [experiment] - let mockBucketer = MockBucketer(mockBucketValue: 2500) let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) optimizely.decisionService = mockDecisionService @@ -371,15 +386,16 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { let user = optimizely.createUserContext(userId: userId) let featureFlag = config.getFeatureFlag(key: flagKey)! let response = mockDecisionService.getDecisionForFlag(config: config, - featureFlag: featureFlag, - user: user, - isAsync: false, - options: nil) + featureFlag: featureFlag, + user: user, + isAsync: false, + options: nil) XCTAssertNotNil(response.result) - XCTAssertEqual(response.result?.source, Constants.DecisionSource.featureTest.rawValue) - XCTAssertNotNil(response.result?.holdoutToSend) - XCTAssertEqual(response.result?.holdoutToSend?.experiment.key, "holdout_test_key") - XCTAssertEqual(response.result?.holdoutToSend?.variation.key, "holdout_variation_key") + // Experiments are skipped; decision comes from delivery (rollout) rules + XCTAssertEqual(response.result?.source, Constants.DecisionSource.rollout.rawValue) + XCTAssertNotNil(response.result?.holdoutDecision) + XCTAssertEqual(response.result?.holdoutDecision?.experiment.key, "holdout_test_key") + XCTAssertEqual(response.result?.holdoutDecision?.variation.key, "holdout_variation_key") } } From dd05055d8e8cf2ded192fb354c89d789a0fba6b0 Mon Sep 17 00:00:00 2001 From: esrakartalOpt Date: Wed, 29 Jul 2026 11:14:20 -0500 Subject: [PATCH 5/9] [FSSDK-12735] Re-trigger FSC tests after datafile updates From c1eab05b978e13400e9fa0af88bbc490bac1c2bc Mon Sep 17 00:00:00 2001 From: Muzahidul Islam <129880873+muzahidul-opti@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:03:29 +0600 Subject: [PATCH 6/9] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6167dfe..c378004a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Optimizely Swift SDK Changelog ## 5.4.2 -July 9, 2026 +July 9th, 2026 ### Fixes From da04aa15b11e0f715038dfab9621a861f915a5cc Mon Sep 17 00:00:00 2001 From: Muzahidul Islam <129880873+muzahidul-opti@users.noreply.github.com> Date: Fri, 7 Aug 2026 09:53:25 +0600 Subject: [PATCH 7/9] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c378004a..56d75eab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ June 24, 2026 ### Fixes -- normalize campaign_id, variation_id, and entity_id on decision events ([#642](https://github.com/optimizely/swift-sdk/pull/642)) +- Normalize campaign_id, variation_id, and entity_id on decision events ([#642](https://github.com/optimizely/swift-sdk/pull/642)) ## 5.4.0 June 22, 2026 From be17129ebe2d8c46a71f20f0f4e2773042de38b8 Mon Sep 17 00:00:00 2001 From: Muzahidul Islam Date: Fri, 7 Aug 2026 18:40:21 +0600 Subject: [PATCH 8/9] [FSSDK-12735] Fix ETD reason message to use double quotes around holdout key FSC asserts the reason string with double quotes ("holdoutKey") but the SDK was emitting single quotes ('holdoutKey'), causing the subset match to fail. Update the unit test to match the corrected format. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- Sources/Utils/LogMessage.swift | 2 +- .../DecisionServiceTests_ExcludeTargetedDeliveries.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Utils/LogMessage.swift b/Sources/Utils/LogMessage.swift index a406839b..3d32a7bb 100644 --- a/Sources/Utils/LogMessage.swift +++ b/Sources/Utils/LogMessage.swift @@ -157,7 +157,7 @@ extension LogMessage: CustomStringConvertible { case .cmabFetchFailed(let key): message = "Failed to fetch CMAB data for experiment \(key)." case .cmabNotSupportedInSyncMode: message = "CMAB is not supported in sync mode." case .holdoutExcludesTargetedDelivery(let holdoutKey, let ruleKey): message = "Holdout (\(holdoutKey)) excludes targeted deliveries, skipping holdout for rule (\(ruleKey))." - case .holdoutExcludeTargetedDeliveriesEnabled(let holdoutKey): message = "Holdout '\(holdoutKey)' has excludeTargetedDeliveries enabled, continuing to rollout evaluation." + case .holdoutExcludeTargetedDeliveriesEnabled(let holdoutKey): message = "Holdout \"\(holdoutKey)\" has excludeTargetedDeliveries enabled, continuing to rollout evaluation." case .userBucketedIntoRollout(let userId, let flagKey): message = "The user \"\(userId)\" was bucketed into a rollout for feature flag \"\(flagKey)\"." case .userNotBucketedIntoRollout(let userId, let flagKey): message = "The user \"\(userId)\" was not bucketed into a rollout for feature flag \"\(flagKey)\"." } diff --git a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift index 676d7c81..273ac5a3 100644 --- a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift +++ b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift @@ -328,7 +328,7 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { let decision = user.decide(key: flagKey, options: [.includeReasons]) let reasons = decision.reasons - let expectedReason = "Holdout 'holdout_test_key' has excludeTargetedDeliveries enabled, continuing to rollout evaluation." + let expectedReason = "Holdout \"holdout_test_key\" has excludeTargetedDeliveries enabled, continuing to rollout evaluation." XCTAssertTrue(reasons.contains(expectedReason), "Decision reasons should contain excludeTargetedDeliveries bypass reason. Got: \(reasons)") } From 06984575ee63124937c28a688d1a821123ede95b Mon Sep 17 00:00:00 2001 From: Muzahidul Islam <129880873+muzahidul-opti@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:44:58 +0600 Subject: [PATCH 9/9] Trigger FSC --- .../DecisionServiceTests_ExcludeTargetedDeliveries.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift index 273ac5a3..bae26ac4 100644 --- a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift +++ b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift @@ -48,8 +48,7 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { override func setUp() { super.setUp() - optimizely = OTUtils.createOptimizely(datafileName: "decide_datafile", - clearUserProfileService: true) + optimizely = OTUtils.createOptimizely(datafileName: "decide_datafile", clearUserProfileService: true) config = optimizely.config! }