Skip to content

Commit 1e4f5a6

Browse files
jaeoptjaeopt
committed
[AI-FSSDK] [FSSDK-12262] Exclude CMAB from UserProfileService
1 parent 88b0644 commit 1e4f5a6

File tree

2 files changed

+192
-2
lines changed

2 files changed

+192
-2
lines changed

‎optimizely/decision_service.py‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -457,7 +457,18 @@ def get_variation(
457457
}
458458

459459
# Check to see if user has a decision available for the given experiment
460-
if user_profile_tracker is not None and not ignore_user_profile:
460+
# CMAB experiments are excluded from UPS because UPS maintains decisions
461+
# across the experiment lifetime without considering TTL or user attributes,
462+
# which contradicts CMAB's dynamic nature.
463+
if experiment.cmab:
464+
if user_profile_tracker is not None and not ignore_user_profile:
465+
message = (
466+
f'Skipping user profile service lookup for CMAB experiment "{experiment.key}". '
467+
'CMAB decisions are excluded from UPS.'
468+
)
469+
self.logger.debug(message)
470+
decide_reasons.append(message)
471+
elif user_profile_tracker is not None and not ignore_user_profile:
461472
variation = self.get_stored_variation(project_config, experiment, user_profile_tracker.get_user_profile())
462473
if variation:
463474
message = f'Returning previously activated variation ID "{variation}" of experiment ' \
@@ -529,7 +540,8 @@ def get_variation(
529540
self.logger.info(message)
530541
decide_reasons.append(message)
531542
# Store this new decision and return the variation for the user
532-
if user_profile_tracker is not None and not ignore_user_profile:
543+
# CMAB experiments are excluded from UPS to preserve dynamic decision-making
544+
if user_profile_tracker is not None and not ignore_user_profile and not experiment.cmab:
533545
try:
534546
user_profile_tracker.update_user_profile(experiment, variation)
535547
except:

‎tests/test_decision_service.py‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1074,6 +1074,184 @@ def test_get_variation_cmab_experiment_with_whitelisted_variation(self):
10741074
mock_bucket.assert_not_called()
10751075
mock_cmab_decision.assert_not_called()
10761076

1077+
def test_get_variation_cmab_experiment_skips_user_profile_lookup(self):
1078+
"""Test that get_variation skips UPS lookup for CMAB experiments even when
1079+
user_profile_tracker is provided, and adds appropriate decision reason."""
1080+
1081+
user = optimizely_user_context.OptimizelyUserContext(
1082+
optimizely_client=None,
1083+
logger=None,
1084+
user_id="test_user",
1085+
user_attributes={}
1086+
)
1087+
1088+
cmab_experiment = entities.Experiment(
1089+
'111150',
1090+
'cmab_experiment',
1091+
'Running',
1092+
'111150',
1093+
[],
1094+
{},
1095+
[
1096+
entities.Variation('111151', 'variation_1'),
1097+
entities.Variation('111152', 'variation_2')
1098+
],
1099+
[
1100+
{'entityId': '111151', 'endOfRange': 5000},
1101+
{'entityId': '111152', 'endOfRange': 10000}
1102+
],
1103+
cmab={'trafficAllocation': 5000}
1104+
)
1105+
1106+
user_profile_service = user_profile.UserProfileService()
1107+
user_profile_tracker = user_profile.UserProfileTracker(user.user_id, user_profile_service)
1108+
1109+
with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \
1110+
mock.patch('optimizely.helpers.audience.does_user_meet_audience_conditions', return_value=[True, []]), \
1111+
mock.patch.object(self.decision_service.bucketer, 'bucket_to_entity_id',
1112+
return_value=['$', []]), \
1113+
mock.patch.object(self.decision_service, 'cmab_service') as mock_cmab_service, \
1114+
mock.patch.object(self.project_config, 'get_variation_from_id',
1115+
return_value=entities.Variation('111151', 'variation_1')), \
1116+
mock.patch(
1117+
'optimizely.decision_service.DecisionService.get_stored_variation'
1118+
) as mock_get_stored_variation, \
1119+
mock.patch.object(self.decision_service, 'logger') as mock_logger:
1120+
1121+
mock_cmab_service.get_decision.return_value = (
1122+
{
1123+
'variation_id': '111151',
1124+
'cmab_uuid': 'test-cmab-uuid-123'
1125+
},
1126+
[]
1127+
)
1128+
1129+
variation_result = self.decision_service.get_variation(
1130+
self.project_config,
1131+
cmab_experiment,
1132+
user,
1133+
user_profile_tracker
1134+
)
1135+
1136+
# Verify get_stored_variation was NOT called for CMAB experiment
1137+
mock_get_stored_variation.assert_not_called()
1138+
1139+
# Verify the decision reason includes UPS exclusion message
1140+
reasons = variation_result['reasons']
1141+
self.assertTrue(
1142+
any('Skipping user profile service lookup for CMAB experiment' in r for r in reasons),
1143+
'Expected UPS exclusion reason in decide_reasons'
1144+
)
1145+
1146+
# Verify variation is still returned correctly
1147+
self.assertEqual(entities.Variation('111151', 'variation_1'), variation_result['variation'])
1148+
self.assertEqual('test-cmab-uuid-123', variation_result['cmab_uuid'])
1149+
self.assertStrictFalse(variation_result['error'])
1150+
1151+
def test_get_variation_cmab_experiment_skips_user_profile_save(self):
1152+
"""Test that get_variation does NOT save to UPS for CMAB experiments."""
1153+
1154+
user = optimizely_user_context.OptimizelyUserContext(
1155+
optimizely_client=None,
1156+
logger=None,
1157+
user_id="test_user",
1158+
user_attributes={}
1159+
)
1160+
1161+
cmab_experiment = entities.Experiment(
1162+
'111150',
1163+
'cmab_experiment',
1164+
'Running',
1165+
'111150',
1166+
[],
1167+
{},
1168+
[
1169+
entities.Variation('111151', 'variation_1'),
1170+
entities.Variation('111152', 'variation_2')
1171+
],
1172+
[
1173+
{'entityId': '111151', 'endOfRange': 5000},
1174+
{'entityId': '111152', 'endOfRange': 10000}
1175+
],
1176+
cmab={'trafficAllocation': 5000}
1177+
)
1178+
1179+
user_profile_service = user_profile.UserProfileService()
1180+
user_profile_tracker = user_profile.UserProfileTracker(user.user_id, user_profile_service)
1181+
1182+
with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \
1183+
mock.patch('optimizely.helpers.audience.does_user_meet_audience_conditions', return_value=[True, []]), \
1184+
mock.patch.object(self.decision_service.bucketer, 'bucket_to_entity_id',
1185+
return_value=['$', []]), \
1186+
mock.patch.object(self.decision_service, 'cmab_service') as mock_cmab_service, \
1187+
mock.patch.object(self.project_config, 'get_variation_from_id',
1188+
return_value=entities.Variation('111151', 'variation_1')), \
1189+
mock.patch.object(user_profile_tracker, 'update_user_profile') as mock_update_profile, \
1190+
mock.patch.object(self.decision_service, 'logger'):
1191+
1192+
mock_cmab_service.get_decision.return_value = (
1193+
{
1194+
'variation_id': '111151',
1195+
'cmab_uuid': 'test-cmab-uuid-123'
1196+
},
1197+
[]
1198+
)
1199+
1200+
variation_result = self.decision_service.get_variation(
1201+
self.project_config,
1202+
cmab_experiment,
1203+
user,
1204+
user_profile_tracker
1205+
)
1206+
1207+
# Verify update_user_profile was NOT called for CMAB experiment
1208+
mock_update_profile.assert_not_called()
1209+
1210+
# Verify variation is still returned correctly
1211+
self.assertEqual(entities.Variation('111151', 'variation_1'), variation_result['variation'])
1212+
self.assertEqual('test-cmab-uuid-123', variation_result['cmab_uuid'])
1213+
1214+
def test_get_variation_non_cmab_experiment_still_uses_user_profile(self):
1215+
"""Test that non-CMAB experiments still use UPS for lookup and save."""
1216+
1217+
user = optimizely_user_context.OptimizelyUserContext(
1218+
optimizely_client=None,
1219+
logger=None,
1220+
user_id="test_user",
1221+
user_attributes={}
1222+
)
1223+
user_profile_service = user_profile.UserProfileService()
1224+
user_profile_tracker = user_profile.UserProfileTracker(user.user_id, user_profile_service)
1225+
experiment = self.project_config.get_experiment_from_key("test_experiment")
1226+
1227+
stored_variation = entities.Variation("111129", "variation")
1228+
1229+
with mock.patch.object(self.decision_service, 'logger'), \
1230+
mock.patch(
1231+
'optimizely.decision_service.DecisionService.get_forced_variation',
1232+
return_value=[None, []]
1233+
), \
1234+
mock.patch(
1235+
'optimizely.decision_service.DecisionService.get_whitelisted_variation',
1236+
return_value=[None, []]
1237+
), \
1238+
mock.patch(
1239+
'optimizely.decision_service.DecisionService.get_stored_variation',
1240+
return_value=stored_variation
1241+
) as mock_get_stored_variation:
1242+
1243+
variation_result = self.decision_service.get_variation(
1244+
self.project_config, experiment, user, user_profile_tracker
1245+
)
1246+
1247+
# Verify get_stored_variation WAS called for non-CMAB experiment
1248+
mock_get_stored_variation.assert_called_once_with(
1249+
self.project_config, experiment, user_profile_tracker.get_user_profile()
1250+
)
1251+
1252+
# Verify stored variation is returned
1253+
self.assertEqual(stored_variation, variation_result['variation'])
1254+
10771255

10781256
class FeatureFlagDecisionTests(base.BaseTest):
10791257
def setUp(self):

0 commit comments

Comments
 (0)