Skip to content

Commit e523cde

Browse files
jaeoptclaude
jaeopt
and
claude
committed
[FSSDK-12262] Exclude CMAB from UserProfileService
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 88b0644 commit e523cde

File tree

2 files changed

+281
-4
lines changed

2 files changed

+281
-4
lines changed

‎optimizely/decision_service.py‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -389,13 +389,15 @@ def get_variation(
389389
1. Check if the experiment is running.
390390
2. Check if the user is forced into a variation via the forced variation map.
391391
3. Check if the user is whitelisted into a variation for the experiment.
392-
4. If user profile tracking is enabled and not ignored, check for a stored variation.
392+
4. If user profile tracking is enabled and not ignored, and the experiment is NOT a CMAB
393+
experiment, check for a stored variation. CMAB experiments are excluded from UPS
394+
because UPS sticky bucketing contradicts CMAB's dynamic nature.
393395
5. Evaluate audience conditions to determine if the user qualifies for the experiment.
394396
6. For CMAB experiments:
395397
a. Check if the user is in the CMAB traffic allocation.
396398
b. If so, fetch the CMAB decision and assign the corresponding variation and cmab_uuid.
397399
7. For non-CMAB experiments, bucket the user into a variation.
398-
8. If a variation is assigned, optionally update the user profile.
400+
8. If a variation is assigned, optionally update the user profile (excluded for CMAB).
399401
400402
Args:
401403
project_config: Instance of ProjectConfig.
@@ -457,7 +459,11 @@ def get_variation(
457459
}
458460

459461
# Check to see if user has a decision available for the given experiment
460-
if user_profile_tracker is not None and not ignore_user_profile:
462+
# CMAB experiments are excluded from UPS because UPS maintains decisions
463+
# across the experiment lifetime without considering TTL or user attributes,
464+
# which contradicts CMAB's dynamic nature.
465+
is_cmab_experiment = bool(experiment.cmab)
466+
if user_profile_tracker is not None and not ignore_user_profile and not is_cmab_experiment:
461467
variation = self.get_stored_variation(project_config, experiment, user_profile_tracker.get_user_profile())
462468
if variation:
463469
message = f'Returning previously activated variation ID "{variation}" of experiment ' \
@@ -473,6 +479,11 @@ def get_variation(
473479
else:
474480
self.logger.warning('User profile has invalid format.')
475481

482+
if is_cmab_experiment and user_profile_tracker is not None and not ignore_user_profile:
483+
message = f'Skipping user profile service for CMAB experiment "{experiment.key}".'
484+
self.logger.debug(message)
485+
decide_reasons.append(message)
486+
476487
# Check audience conditions
477488
audience_conditions = experiment.get_audience_conditions_or_ids()
478489
user_meets_audience_conditions, reasons_received = audience_helper.does_user_meet_audience_conditions(
@@ -529,7 +540,8 @@ def get_variation(
529540
self.logger.info(message)
530541
decide_reasons.append(message)
531542
# Store this new decision and return the variation for the user
532-
if user_profile_tracker is not None and not ignore_user_profile:
543+
# CMAB experiments are excluded from UPS saves to preserve dynamic behavior
544+
if user_profile_tracker is not None and not ignore_user_profile and not is_cmab_experiment:
533545
try:
534546
user_profile_tracker.update_user_profile(experiment, variation)
535547
except:

‎tests/test_decision_service.py‎

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1074,6 +1074,271 @@ def test_get_variation_cmab_experiment_with_whitelisted_variation(self):
10741074
mock_bucket.assert_not_called()
10751075
mock_cmab_decision.assert_not_called()
10761076

1077+
def test_get_variation_cmab_experiment_skips_ups_lookup(self):
1078+
"""Test that get_variation skips UPS lookup for CMAB experiments even when UPS has a stored decision."""
1079+
1080+
user = optimizely_user_context.OptimizelyUserContext(
1081+
optimizely_client=None,
1082+
logger=None,
1083+
user_id="test_user",
1084+
user_attributes={}
1085+
)
1086+
1087+
# Create a CMAB experiment
1088+
cmab_experiment = entities.Experiment(
1089+
'111150',
1090+
'cmab_experiment',
1091+
'Running',
1092+
'111150',
1093+
[],
1094+
{},
1095+
[
1096+
entities.Variation('111151', 'variation_1'),
1097+
entities.Variation('111152', 'variation_2')
1098+
],
1099+
[
1100+
{'entityId': '111151', 'endOfRange': 5000},
1101+
{'entityId': '111152', 'endOfRange': 10000}
1102+
],
1103+
cmab={'trafficAllocation': 5000}
1104+
)
1105+
1106+
# Set up a user profile tracker WITH a stored decision
1107+
user_profile_service = user_profile.UserProfileService()
1108+
user_profile_tracker = user_profile.UserProfileTracker(user.user_id, user_profile_service)
1109+
# Manually store a variation for this experiment in the user profile
1110+
user_profile_tracker.user_profile.save_variation_for_experiment('111150', '111152')
1111+
1112+
with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \
1113+
mock.patch('optimizely.helpers.audience.does_user_meet_audience_conditions', return_value=[True, []]), \
1114+
mock.patch.object(self.decision_service.bucketer, 'bucket_to_entity_id',
1115+
return_value=['$', []]), \
1116+
mock.patch.object(self.decision_service, 'cmab_service') as mock_cmab_service, \
1117+
mock.patch.object(self.project_config, 'get_variation_from_id',
1118+
return_value=entities.Variation('111151', 'variation_1')), \
1119+
mock.patch(
1120+
"optimizely.decision_service.DecisionService.get_stored_variation"
1121+
) as mock_get_stored_variation, \
1122+
mock.patch.object(self.decision_service, 'logger') as mock_logger:
1123+
1124+
# Configure CMAB service to return a decision
1125+
mock_cmab_service.get_decision.return_value = (
1126+
{
1127+
'variation_id': '111151',
1128+
'cmab_uuid': 'test-cmab-uuid-456'
1129+
},
1130+
[]
1131+
)
1132+
1133+
variation_result = self.decision_service.get_variation(
1134+
self.project_config,
1135+
cmab_experiment,
1136+
user,
1137+
user_profile_tracker
1138+
)
1139+
variation = variation_result['variation']
1140+
cmab_uuid = variation_result['cmab_uuid']
1141+
reasons = variation_result['reasons']
1142+
1143+
# UPS lookup should NOT have been called for CMAB experiment
1144+
mock_get_stored_variation.assert_not_called()
1145+
1146+
# CMAB service should have been called instead
1147+
mock_cmab_service.get_decision.assert_called_once()
1148+
1149+
# Should return the CMAB decision, not the stored UPS decision
1150+
self.assertEqual(entities.Variation('111151', 'variation_1'), variation)
1151+
self.assertEqual('test-cmab-uuid-456', cmab_uuid)
1152+
1153+
# Should include decision reason about skipping UPS
1154+
self.assertIn(
1155+
'Skipping user profile service for CMAB experiment "cmab_experiment".',
1156+
reasons
1157+
)
1158+
1159+
# Should log the UPS skip
1160+
mock_logger.debug.assert_any_call(
1161+
'Skipping user profile service for CMAB experiment "cmab_experiment".'
1162+
)
1163+
1164+
def test_get_variation_cmab_experiment_skips_ups_save(self):
1165+
"""Test that get_variation does not save to UPS for CMAB experiments."""
1166+
1167+
user = optimizely_user_context.OptimizelyUserContext(
1168+
optimizely_client=None,
1169+
logger=None,
1170+
user_id="test_user",
1171+
user_attributes={}
1172+
)
1173+
1174+
# Create a CMAB experiment
1175+
cmab_experiment = entities.Experiment(
1176+
'111150',
1177+
'cmab_experiment',
1178+
'Running',
1179+
'111150',
1180+
[],
1181+
{},
1182+
[
1183+
entities.Variation('111151', 'variation_1'),
1184+
entities.Variation('111152', 'variation_2')
1185+
],
1186+
[
1187+
{'entityId': '111151', 'endOfRange': 5000},
1188+
{'entityId': '111152', 'endOfRange': 10000}
1189+
],
1190+
cmab={'trafficAllocation': 5000}
1191+
)
1192+
1193+
user_profile_service = user_profile.UserProfileService()
1194+
user_profile_tracker = user_profile.UserProfileTracker(user.user_id, user_profile_service)
1195+
1196+
with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \
1197+
mock.patch('optimizely.helpers.audience.does_user_meet_audience_conditions', return_value=[True, []]), \
1198+
mock.patch.object(self.decision_service.bucketer, 'bucket_to_entity_id',
1199+
return_value=['$', []]), \
1200+
mock.patch.object(self.decision_service, 'cmab_service') as mock_cmab_service, \
1201+
mock.patch.object(self.project_config, 'get_variation_from_id',
1202+
return_value=entities.Variation('111151', 'variation_1')), \
1203+
mock.patch.object(user_profile_tracker, 'update_user_profile') as mock_update_profile, \
1204+
mock.patch.object(self.decision_service, 'logger'):
1205+
1206+
mock_cmab_service.get_decision.return_value = (
1207+
{
1208+
'variation_id': '111151',
1209+
'cmab_uuid': 'test-cmab-uuid-789'
1210+
},
1211+
[]
1212+
)
1213+
1214+
variation_result = self.decision_service.get_variation(
1215+
self.project_config,
1216+
cmab_experiment,
1217+
user,
1218+
user_profile_tracker
1219+
)
1220+
variation = variation_result['variation']
1221+
1222+
# Variation should be returned from CMAB
1223+
self.assertEqual(entities.Variation('111151', 'variation_1'), variation)
1224+
1225+
# UPS should NOT have been updated for CMAB experiment
1226+
mock_update_profile.assert_not_called()
1227+
1228+
def test_get_variation_non_cmab_experiment_still_uses_ups(self):
1229+
"""Test that non-CMAB experiments still use UPS for lookup and save."""
1230+
1231+
user = optimizely_user_context.OptimizelyUserContext(
1232+
optimizely_client=None,
1233+
logger=None,
1234+
user_id="test_user",
1235+
user_attributes={}
1236+
)
1237+
1238+
experiment = self.project_config.get_experiment_from_key("test_experiment")
1239+
1240+
user_profile_service = user_profile.UserProfileService()
1241+
user_profile_tracker = user_profile.UserProfileTracker(user.user_id, user_profile_service)
1242+
1243+
stored_variation = entities.Variation("111128", "control")
1244+
1245+
with mock.patch(
1246+
"optimizely.decision_service.DecisionService.get_whitelisted_variation",
1247+
return_value=[None, []],
1248+
), mock.patch(
1249+
"optimizely.decision_service.DecisionService.get_stored_variation",
1250+
return_value=stored_variation,
1251+
) as mock_get_stored_variation:
1252+
variation_result = self.decision_service.get_variation(
1253+
self.project_config, experiment, user, user_profile_tracker
1254+
)
1255+
variation = variation_result['variation']
1256+
reasons = variation_result['reasons']
1257+
1258+
# UPS lookup should have been called for non-CMAB experiment
1259+
mock_get_stored_variation.assert_called_once()
1260+
1261+
# Should return the stored variation
1262+
self.assertEqual(stored_variation, variation)
1263+
1264+
# Should NOT include the CMAB UPS skip reason
1265+
for reason in reasons:
1266+
self.assertNotIn('Skipping user profile service for CMAB experiment', reason)
1267+
1268+
def test_get_variation_cmab_experiment_ups_exclusion_with_ignore_ups_option(self):
1269+
"""Test that when IGNORE_USER_PROFILE_SERVICE option is set with a CMAB experiment,
1270+
UPS is skipped and no CMAB-specific skip reason is added (since UPS was already ignored)."""
1271+
1272+
from optimizely.decision.optimizely_decide_option import OptimizelyDecideOption
1273+
1274+
user = optimizely_user_context.OptimizelyUserContext(
1275+
optimizely_client=None,
1276+
logger=None,
1277+
user_id="test_user",
1278+
user_attributes={}
1279+
)
1280+
1281+
cmab_experiment = entities.Experiment(
1282+
'111150',
1283+
'cmab_experiment',
1284+
'Running',
1285+
'111150',
1286+
[],
1287+
{},
1288+
[
1289+
entities.Variation('111151', 'variation_1'),
1290+
entities.Variation('111152', 'variation_2')
1291+
],
1292+
[
1293+
{'entityId': '111151', 'endOfRange': 5000},
1294+
{'entityId': '111152', 'endOfRange': 10000}
1295+
],
1296+
cmab={'trafficAllocation': 5000}
1297+
)
1298+
1299+
user_profile_service = user_profile.UserProfileService()
1300+
user_profile_tracker = user_profile.UserProfileTracker(user.user_id, user_profile_service)
1301+
1302+
with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \
1303+
mock.patch('optimizely.helpers.audience.does_user_meet_audience_conditions', return_value=[True, []]), \
1304+
mock.patch.object(self.decision_service.bucketer, 'bucket_to_entity_id',
1305+
return_value=['$', []]), \
1306+
mock.patch.object(self.decision_service, 'cmab_service') as mock_cmab_service, \
1307+
mock.patch.object(self.project_config, 'get_variation_from_id',
1308+
return_value=entities.Variation('111151', 'variation_1')), \
1309+
mock.patch(
1310+
"optimizely.decision_service.DecisionService.get_stored_variation"
1311+
) as mock_get_stored_variation, \
1312+
mock.patch.object(user_profile_tracker, 'update_user_profile') as mock_update_profile, \
1313+
mock.patch.object(self.decision_service, 'logger'):
1314+
1315+
mock_cmab_service.get_decision.return_value = (
1316+
{
1317+
'variation_id': '111151',
1318+
'cmab_uuid': 'test-cmab-uuid'
1319+
},
1320+
[]
1321+
)
1322+
1323+
variation_result = self.decision_service.get_variation(
1324+
self.project_config,
1325+
cmab_experiment,
1326+
user,
1327+
user_profile_tracker,
1328+
options=[OptimizelyDecideOption.IGNORE_USER_PROFILE_SERVICE]
1329+
)
1330+
reasons = variation_result['reasons']
1331+
1332+
# UPS lookup should NOT have been called
1333+
mock_get_stored_variation.assert_not_called()
1334+
1335+
# UPS save should NOT have been called
1336+
mock_update_profile.assert_not_called()
1337+
1338+
# CMAB-specific skip message should NOT appear since UPS was already ignored via option
1339+
for reason in reasons:
1340+
self.assertNotIn('Skipping user profile service for CMAB experiment', reason)
1341+
10771342

10781343
class FeatureFlagDecisionTests(base.BaseTest):
10791344
def setUp(self):

0 commit comments

Comments
 (0)