diff --git a/tests/acceptance/datafile.py b/tests/acceptance/datafile.py index 33c28012..dea13ac7 100644 --- a/tests/acceptance/datafile.py +++ b/tests/acceptance/datafile.py @@ -1,465 +1,220 @@ -datafile = { - "accountId": "10845721364", - "anonymizeIP": True, - "attributes": [ - { - "id": "16921322086", - "key": "attr_1" - } - ], - "audiences": [ - { - "conditions": "[\"and\", [\"or\", [\"or\", {\"match\": \"exact\", \"name\": \"attr_1\", \"type\": \"custom_attribute\", \"value\": \"hola\"}]]]", - "id": "16902921321", - "name": "Audience1" - }, - { - "conditions": "[\"or\", {\"match\": \"exact\", \"name\": \"$opt_dummy_attribute\", \"type\": \"custom_attribute\", \"value\": \"$opt_dummy_value\"}]", - "id": "$opt_dummy_audience", - "name": "Optimizely-Generated Audience for Backwards Compatibility" - } - ], - "botFiltering": False, - "environmentKey": "production", - "events": [ - { - "experimentIds": [ - "16911963060", - "16910084756" - ], - "id": "16911532385", - "key": "myevent" - } - ], - "experiments": [ - { - "audienceConditions": [ - "or", - "16902921321" - ], - "audienceIds": [ - "16902921321" - ], - "forcedVariations": {}, - "id": "16910084756", - "key": "feature_2_test", - "layerId": "16933431472", - "status": "Running", - "trafficAllocation": [ - { - "endOfRange": 5000, - "entityId": "16925360560" - }, - { - "endOfRange": 10000, - "entityId": "16925360560" - } - ], - "variations": [ - { - "featureEnabled": True, - "id": "16925360560", - "key": "variation_1", - "variables": [] - }, - { - "featureEnabled": True, - "id": "16915611472", - "key": "variation_2", - "variables": [] - } - ] - }, - { - "audienceConditions": [ - "or", - "16902921321" - ], - "audienceIds": [ - "16902921321" - ], - "forcedVariations": {}, - "id": "16911963060", - "key": "ab_test1", - "layerId": "16916031507", - "status": "Running", - "trafficAllocation": [ - { - "endOfRange": 1000, - "entityId": "16905941566" - }, - { - "endOfRange": 5000, - "entityId": "16905941566" - }, - { - "endOfRange": 8000, - "entityId": "16905941566" - }, - { - "endOfRange": 9000, - "entityId": "16905941566" - }, - { - "endOfRange": 10000, - "entityId": "16905941566" - } - ], - "variations": [ - { - "featureEnabled": True, - "id": "16905941566", - "key": "variation_1", - "variables": [] - }, - { - "featureEnabled": True, - "id": "16927770169", - "key": "variation_2", - "variables": [] - } - ] - } - ], - "featureFlags": [ - { - "experimentIds": [], - "id": "16907463855", - "key": "feature_3", - "rolloutId": "16909553406", - "variables": [] - }, - { - "experimentIds": [], - "id": "16912161768", - "key": "feature_4", - "rolloutId": "16943340293", - "variables": [] - }, - { - "experimentIds": [], - "id": "16923312421", - "key": "feature_5", - "rolloutId": "16917103311", - "variables": [] - }, - { - "experimentIds": [], - "id": "16925981047", - "key": "feature_1", - "rolloutId": "16928980969", - "variables": [ - { - "defaultValue": "hello", - "id": "16916052157", - "key": "str_var", - "type": "string" - }, - { - "defaultValue": "5.6", - "id": "16923002469", - "key": "double_var", - "type": "double" - }, - { - "defaultValue": "true", - "id": "16932993089", - "key": "bool_var", - "type": "boolean" - }, - { - "defaultValue": "1", - "id": "16937161477", - "key": "int_var", - "type": "integer" - } - ] - }, - { - "experimentIds": [ - "16910084756" - ], - "id": "16928980973", - "key": "feature_2", - "rolloutId": "16917900798", - "variables": [] - }, - { - "experimentIds": [ - "16911963060" - ], - "id": "147680", - "key": "GkbzTurBWXr8EtNGZj2j6e", - "rolloutId": "rollout-147680-16935023792", - "variables": [] - } - ], - "groups": [], - "integrations": [], - "projectId": "16931203314", - "revision": "137", - "rollouts": [ - { - "experiments": [ - { - "audienceConditions": [], - "audienceIds": [], - "forcedVariations": {}, - "id": "default-16909553406", - "key": "default-16909553406", - "layerId": "16909553406", - "status": "Running", - "trafficAllocation": [ - { - "endOfRange": 10000, - "entityId": "471185" - } - ], - "variations": [ - { - "featureEnabled": False, - "id": "471185", - "key": "off", - "variables": [] - } - ] - } - ], - "id": "16909553406" - }, - { - "experiments": [ - { - "audienceConditions": [], - "audienceIds": [], - "forcedVariations": {}, - "id": "default-16943340293", - "key": "default-16943340293", - "layerId": "16943340293", - "status": "Running", - "trafficAllocation": [ - { - "endOfRange": 10000, - "entityId": "16925940659" - } - ], - "variations": [ - { - "featureEnabled": True, - "id": "16925940659", - "key": "16925940659", - "variables": [] - } - ] - } - ], - "id": "16943340293" - }, - { - "experiments": [ - { - "audienceConditions": [], - "audienceIds": [], - "forcedVariations": {}, - "id": "default-16917103311", - "key": "default-16917103311", - "layerId": "16917103311", - "status": "Running", - "trafficAllocation": [ - { - "endOfRange": 10000, - "entityId": "16927890136" - } - ], - "variations": [ - { - "featureEnabled": True, - "id": "16927890136", - "key": "16927890136", - "variables": [] - } - ] - } - ], - "id": "16917103311" - }, - { - "experiments": [ - { - "audienceConditions": [ - "or", - "16902921321" - ], - "audienceIds": [ - "16902921321" - ], - "forcedVariations": {}, - "id": "16941022436", - "key": "16941022436", - "layerId": "16928980969", - "status": "Running", - "trafficAllocation": [ - { - "endOfRange": 10000, - "entityId": "16906801184" - } - ], - "variations": [ - { - "featureEnabled": True, - "id": "16906801184", - "key": "16906801184", - "variables": [ - { - "id": "16916052157", - "value": "hello" - }, - { - "id": "16923002469", - "value": "5.6" - }, - { - "id": "16932993089", - "value": "true" - }, - { - "id": "16937161477", - "value": "1" - } - ] - } - ] - }, - { - "audienceConditions": [], - "audienceIds": [], - "forcedVariations": {}, - "id": "default-16928980969", - "key": "default-16928980969", - "layerId": "16928980969", - "status": "Running", - "trafficAllocation": [ - { - "endOfRange": 10000, - "entityId": "471188" - } - ], - "variations": [ - { - "featureEnabled": False, - "id": "471188", - "key": "off", - "variables": [ - { - "id": "16916052157", - "value": "hello" - }, - { - "id": "16923002469", - "value": "5.6" - }, - { - "id": "16932993089", - "value": "true" - }, - { - "id": "16937161477", - "value": "1" - } - ] - } - ] - } - ], - "id": "16928980969" - }, - { - "experiments": [ - { - "audienceConditions": [ - "or", - "16902921321" - ], - "audienceIds": [ - "16902921321" - ], - "forcedVariations": {}, - "id": "16924931120", - "key": "16924931120", - "layerId": "16917900798", - "status": "Running", - "trafficAllocation": [ - { - "endOfRange": 10000, - "entityId": "16931381940" - } - ], - "variations": [ - { - "featureEnabled": True, - "id": "16931381940", - "key": "16931381940", - "variables": [] - } - ] - }, - { - "audienceConditions": [], - "audienceIds": [], - "forcedVariations": {}, - "id": "default-16917900798", - "key": "default-16917900798", - "layerId": "16917900798", - "status": "Running", - "trafficAllocation": [ - { - "endOfRange": 10000, - "entityId": "471189" - } - ], - "variations": [ - { - "featureEnabled": False, - "id": "471189", - "key": "off", - "variables": [] - } - ] - } - ], - "id": "16917900798" - }, - { - "experiments": [ - { - "audienceConditions": [], - "audienceIds": [], - "forcedVariations": {}, - "id": "default-rollout-147680-16935023792", - "key": "default-rollout-147680-16935023792", - "layerId": "rollout-147680-16935023792", - "status": "Running", - "trafficAllocation": [ - { - "endOfRange": 10000, - "entityId": "471190" - } - ], - "variations": [ - { - "featureEnabled": False, - "id": "471190", - "key": "off", - "variables": [] - } - ] - } - ], - "id": "rollout-147680-16935023792" - } - ], - "sdkKey": "KZbunNn9bVfBWLpZPq2XC4", - "typedAudiences": [], - "variables": [], - "version": "4" -} +datafile = {'accountId': '10845721364', + 'anonymizeIP': True, + 'attributes': [{'id': '16921322086', 'key': 'attr_1'}], + 'audiences': [{'conditions': '["and", ["or", ["or", {"match": "exact", "name": "attr_1", "type": "custom_attribute", ' + '"value": "hola"}]]]', + 'id': '16902921321', + 'name': 'Audience1'}, + {'conditions': '["or", {"match": "exact", "name": "$opt_dummy_attribute", "type": "custom_attribute", ' + '"value": "$opt_dummy_value"}]', + 'id': '$opt_dummy_audience', + 'name': 'Optimizely-Generated Audience for Backwards Compatibility'}], + 'botFiltering': False, + 'environmentKey': 'production', + 'events': [{'experimentIds': ['16911963060', '9300002877087', '16910084756'], 'id': '16911532385', 'key': 'myevent'}], + 'experiments': [{'audienceConditions': ['or', '16902921321'], + 'audienceIds': ['16902921321'], + 'forcedVariations': {}, + 'id': '16910084756', + 'key': 'feature_2_test', + 'layerId': '16933431472', + 'status': 'Running', + 'trafficAllocation': [{'endOfRange': 5000, 'entityId': '16925360560'}, + {'endOfRange': 10000, 'entityId': '16925360560'}], + 'variations': [{'featureEnabled': True, 'id': '16925360560', 'key': 'variation_1', 'variables': []}, + {'featureEnabled': True, 'id': '16915611472', 'key': 'variation_2', 'variables': []}]}, + {'audienceConditions': ['or', '16902921321'], + 'audienceIds': ['16902921321'], + 'forcedVariations': {}, + 'id': '16911963060', + 'key': 'ab_test1', + 'layerId': '16916031507', + 'status': 'Running', + 'trafficAllocation': [{'endOfRange': 1000, 'entityId': '16905941566'}, + {'endOfRange': 5000, 'entityId': '16905941566'}, + {'endOfRange': 8000, 'entityId': '16905941566'}, + {'endOfRange': 9000, 'entityId': '16905941566'}, + {'endOfRange': 10000, 'entityId': '16905941566'}], + 'variations': [{'featureEnabled': True, 'id': '16905941566', 'key': 'variation_1', 'variables': []}, + {'featureEnabled': True, 'id': '16927770169', 'key': 'variation_2', 'variables': []}]}, + {'audienceConditions': ['or', '16902921321'], + 'audienceIds': ['16902921321'], + 'cmab': {'attributeIds': ['16921322086'], 'trafficAllocation': 10000}, + 'forcedVariations': {}, + 'id': '9300002877087', + 'key': 'cmab-rule_1', + 'layerId': '9300002131372', + 'status': 'Running', + 'trafficAllocation': [], + 'variations': [{'featureEnabled': False, 'id': '1579277', 'key': 'off', 'variables': []}, + {'featureEnabled': True, 'id': '1579278', 'key': 'on', 'variables': []}]}], + 'featureFlags': [{'experimentIds': [], + 'id': '16907463855', + 'key': 'feature_3', + 'rolloutId': '16909553406', + 'variables': []}, + {'experimentIds': [], + 'id': '16912161768', + 'key': 'feature_4', + 'rolloutId': '16943340293', + 'variables': []}, + {'experimentIds': [], + 'id': '16923312421', + 'key': 'feature_5', + 'rolloutId': '16917103311', + 'variables': []}, + {'experimentIds': [], + 'id': '16925981047', + 'key': 'feature_1', + 'rolloutId': '16928980969', + 'variables': [{'defaultValue': 'hello', 'id': '16916052157', 'key': 'str_var', 'type': 'string'}, + {'defaultValue': '5.6', 'id': '16923002469', 'key': 'double_var', 'type': 'double'}, + {'defaultValue': 'true', 'id': '16932993089', 'key': 'bool_var', 'type': 'boolean'}, + {'defaultValue': '1', 'id': '16937161477', 'key': 'int_var', 'type': 'integer'}]}, + {'experimentIds': ['16910084756'], + 'id': '16928980973', + 'key': 'feature_2', + 'rolloutId': '16917900798', + 'variables': []}, + {'experimentIds': ['16911963060'], + 'id': '147680', + 'key': 'GkbzTurBWXr8EtNGZj2j6e', + 'rolloutId': 'rollout-147680-16935023792', + 'variables': []}, + {'experimentIds': ['9300002877087'], + 'id': '496419', + 'key': 'cmab_flag', + 'rolloutId': 'rollout-496419-16935023792', + 'variables': []}], + 'groups': [], + 'holdouts': [], + 'integrations': [], + 'projectId': '16931203314', + 'region': 'US', + 'revision': '139', + 'rollouts': [{'experiments': [{'audienceConditions': [], + 'audienceIds': [], + 'forcedVariations': {}, + 'id': 'default-16909553406', + 'key': 'default-16909553406', + 'layerId': '16909553406', + 'status': 'Running', + 'trafficAllocation': [{'endOfRange': 10000, 'entityId': '471185'}], + 'variations': [{'featureEnabled': False, + 'id': '471185', + 'key': 'off', + 'variables': []}]}], + 'id': '16909553406'}, + {'experiments': [{'audienceConditions': [], + 'audienceIds': [], + 'forcedVariations': {}, + 'id': 'default-16943340293', + 'key': 'default-16943340293', + 'layerId': '16943340293', + 'status': 'Running', + 'trafficAllocation': [{'endOfRange': 10000, 'entityId': '16925940659'}], + 'variations': [{'featureEnabled': True, + 'id': '16925940659', + 'key': '16925940659', + 'variables': []}]}], + 'id': '16943340293'}, + {'experiments': [{'audienceConditions': [], + 'audienceIds': [], + 'forcedVariations': {}, + 'id': 'default-16917103311', + 'key': 'default-16917103311', + 'layerId': '16917103311', + 'status': 'Running', + 'trafficAllocation': [{'endOfRange': 10000, 'entityId': '16927890136'}], + 'variations': [{'featureEnabled': True, + 'id': '16927890136', + 'key': '16927890136', + 'variables': []}]}], + 'id': '16917103311'}, + {'experiments': [{'audienceConditions': ['or', '16902921321'], + 'audienceIds': ['16902921321'], + 'forcedVariations': {}, + 'id': '16941022436', + 'key': '16941022436', + 'layerId': '16928980969', + 'status': 'Running', + 'trafficAllocation': [{'endOfRange': 10000, 'entityId': '16906801184'}], + 'variations': [{'featureEnabled': True, + 'id': '16906801184', + 'key': '16906801184', + 'variables': [{'id': '16916052157', 'value': 'hello'}, + {'id': '16923002469', 'value': '5.6'}, + {'id': '16932993089', 'value': 'true'}, + {'id': '16937161477', 'value': '1'}]}]}, + {'audienceConditions': [], + 'audienceIds': [], + 'forcedVariations': {}, + 'id': 'default-16928980969', + 'key': 'default-16928980969', + 'layerId': '16928980969', + 'status': 'Running', + 'trafficAllocation': [{'endOfRange': 10000, 'entityId': '471188'}], + 'variations': [{'featureEnabled': False, + 'id': '471188', + 'key': 'off', + 'variables': [{'id': '16916052157', 'value': 'hello'}, + {'id': '16923002469', 'value': '5.6'}, + {'id': '16932993089', 'value': 'true'}, + {'id': '16937161477', 'value': '1'}]}]}], + 'id': '16928980969'}, + {'experiments': [{'audienceConditions': ['or', '16902921321'], + 'audienceIds': ['16902921321'], + 'forcedVariations': {}, + 'id': '16924931120', + 'key': '16924931120', + 'layerId': '16917900798', + 'status': 'Running', + 'trafficAllocation': [{'endOfRange': 10000, 'entityId': '16931381940'}], + 'variations': [{'featureEnabled': True, + 'id': '16931381940', + 'key': '16931381940', + 'variables': []}]}, + {'audienceConditions': [], + 'audienceIds': [], + 'forcedVariations': {}, + 'id': 'default-16917900798', + 'key': 'default-16917900798', + 'layerId': '16917900798', + 'status': 'Running', + 'trafficAllocation': [{'endOfRange': 10000, 'entityId': '471189'}], + 'variations': [{'featureEnabled': False, + 'id': '471189', + 'key': 'off', + 'variables': []}]}], + 'id': '16917900798'}, + {'experiments': [{'audienceConditions': [], + 'audienceIds': [], + 'forcedVariations': {}, + 'id': 'default-rollout-147680-16935023792', + 'key': 'default-rollout-147680-16935023792', + 'layerId': 'rollout-147680-16935023792', + 'status': 'Running', + 'trafficAllocation': [{'endOfRange': 10000, 'entityId': '471190'}], + 'variations': [{'featureEnabled': False, + 'id': '471190', + 'key': 'off', + 'variables': []}]}], + 'id': 'rollout-147680-16935023792'}, + {'experiments': [{'audienceConditions': [], + 'audienceIds': [], + 'forcedVariations': {}, + 'id': 'default-rollout-496419-16935023792', + 'key': 'default-rollout-496419-16935023792', + 'layerId': 'rollout-496419-16935023792', + 'status': 'Running', + 'trafficAllocation': [{'endOfRange': 10000, 'entityId': '1579277'}], + 'variations': [{'featureEnabled': False, + 'id': '1579277', + 'key': 'off', + 'variables': []}]}], + 'id': 'rollout-496419-16935023792'}], + 'sdkKey': 'KZbunNn9bVfBWLpZPq2XC4', + 'typedAudiences': [], + 'variables': [], + 'version': '4'} + diff --git a/tests/acceptance/test_acceptance/test_activate.py b/tests/acceptance/test_acceptance/test_activate.py index dbfbc5d0..cc72c8b3 100644 --- a/tests/acceptance/test_acceptance/test_activate.py +++ b/tests/acceptance/test_acceptance/test_activate.py @@ -154,7 +154,7 @@ def test_activate__feature(session_obj, feature_key, expected_response, assert resp.status_code == expected_status_code, resp.text -expected_activate_type_exper = """[ +expected_activate_type_exper_base = """[ { "userId": "matjaz", "experimentKey": "feature_2_test", @@ -173,7 +173,7 @@ def test_activate__feature(session_obj, feature_key, expected_response, } ]""" -expected_activate_type_feat = """[ +expected_activate_type_feat_base = """[ { "userId": "matjaz", "experimentKey": "feature_2_test", @@ -232,8 +232,8 @@ def test_activate__feature(session_obj, feature_key, expected_response, @pytest.mark.parametrize("decision_type, expected_response, expected_status_code, bypass_validation_request", [ - ("experiment", expected_activate_type_exper, 200, False), - ("feature", expected_activate_type_feat, 200, False), + ("experiment", expected_activate_type_exper_base, 200, False), + ("feature", expected_activate_type_feat_base, 200, False), ("invalid decision type", {'error': 'type "invalid decision type" not supported'}, 400, True), ("", {'error': 'type "" not supported'}, 400, True) ], ids=["experiment decision type", "feature decision type", "invalid decision type", "empty decision type"]) @@ -255,10 +255,47 @@ def test_activate__type(session_obj, decision_type, expected_response, resp = create_and_validate_request_and_response(ENDPOINT_ACTIVATE, 'post', session_obj, bypass_validation_request, payload=payload, params=params) - if decision_type in ['experiment', 'feature']: - sorted_actual = sort_response( - resp.json(), 'experimentKey', 'featureKey') - sorted_expected = sort_response(json.loads(expected_response), 'experimentKey', 'featureKey') + if decision_type == 'experiment': + # For experiment type, verify base experiments plus CMAB experiment + actual_results = resp.json() + expected_base = json.loads(expected_response) + + # Check we have the expected count (2 base + 1 CMAB) + assert len(actual_results) == 3, f"Expected 3 experiments, got {len(actual_results)}" + + # Find and verify CMAB experiment + cmab_result = next((r for r in actual_results if r['experimentKey'] == 'cmab-rule_1'), None) + assert cmab_result is not None, "CMAB experiment not found" + assert cmab_result['userId'] == 'matjaz' + assert cmab_result['featureKey'] == '' + assert cmab_result['variationKey'] in ['on', 'off'], f"Unexpected CMAB variation: {cmab_result['variationKey']}" + assert cmab_result['type'] == 'experiment' + + # Verify base experiments (excluding CMAB) + base_results = [r for r in actual_results if r['experimentKey'] != 'cmab-rule_1'] + sorted_actual = sort_response(base_results, 'experimentKey', 'featureKey') + sorted_expected = sort_response(expected_base, 'experimentKey', 'featureKey') + assert sorted_actual == sorted_expected + elif decision_type == 'feature': + # For feature type, verify base features plus CMAB feature + actual_results = resp.json() + expected_base = json.loads(expected_response) + + # Check we have the expected count (6 base + 1 CMAB) + assert len(actual_results) == 7, f"Expected 7 features, got {len(actual_results)}" + + # Find and verify CMAB feature + cmab_result = next((r for r in actual_results if r['featureKey'] == 'cmab_flag'), None) + assert cmab_result is not None, "CMAB feature not found" + assert cmab_result['userId'] == 'matjaz' + assert cmab_result['experimentKey'] == 'cmab-rule_1' + assert cmab_result['variationKey'] in ['on', 'off'], f"Unexpected CMAB variation: {cmab_result['variationKey']}" + assert cmab_result['type'] == 'feature' + + # Verify base features (excluding CMAB) + base_results = [r for r in actual_results if r['featureKey'] != 'cmab_flag'] + sorted_actual = sort_response(base_results, 'experimentKey', 'featureKey') + sorted_expected = sort_response(expected_base, 'experimentKey', 'featureKey') assert sorted_actual == sorted_expected elif resp.json()['error']: with pytest.raises(requests.exceptions.HTTPError): @@ -476,7 +513,7 @@ def test_activate__enabled(session_obj, enabled, experimentKey, featureKey, # ####################################################### -expected_activate_with_config = """[ +expected_activate_with_config_base = """[ { "userId": "matjaz", "experimentKey": "ab_test1", @@ -556,8 +593,8 @@ def test_activate_with_config(session_obj): validates against the whole response body. In "activate" - Request payload defines the “who” (user id and attributes) - while the query parameters define the “what” (feature, experiment, etc) + Request payload defines the "who" (user id and attributes) + while the query parameters define the "what" (feature, experiment, etc) Request parameter is a list of experiment keys or feature keys. If you want both add both and separate them with comma. @@ -594,9 +631,29 @@ def test_activate_with_config(session_obj): resp_activate = create_and_validate_request_and_response(ENDPOINT_ACTIVATE, 'post', session_obj, payload=payload, params=params) - sorted_actual = sort_response(resp_activate.json(), 'experimentKey', 'featureKey') - sorted_expected = sort_response(json.loads(expected_activate_with_config), - 'experimentKey', - 'featureKey') + actual_results = resp_activate.json() + expected_base = json.loads(expected_activate_with_config_base) + + # Find CMAB entries (experiment and feature versions) + cmab_experiment = next((r for r in actual_results if r.get('experimentKey') == 'cmab-rule_1' and r.get('featureKey') == ''), None) + cmab_feature = next((r for r in actual_results if r.get('featureKey') == 'cmab_flag'), None) + + # Verify CMAB experiment entry exists and is valid + assert cmab_experiment is not None, "CMAB experiment not found" + assert cmab_experiment['variationKey'] in ['on', 'off'] + assert cmab_experiment['type'] == 'experiment' + + # Verify CMAB feature entry exists and is valid + assert cmab_feature is not None, "CMAB feature not found" + assert cmab_feature['experimentKey'] == 'cmab-rule_1' + assert cmab_feature['variationKey'] in ['on', 'off'] + assert cmab_feature['type'] == 'feature' + + # Verify base results (excluding CMAB entries) + base_results = [r for r in actual_results if r.get('experimentKey') != 'cmab-rule_1' or r.get('featureKey') != ''] + base_results = [r for r in base_results if r.get('featureKey') != 'cmab_flag'] + + sorted_actual = sort_response(base_results, 'experimentKey', 'featureKey') + sorted_expected = sort_response(expected_base, 'experimentKey', 'featureKey') assert sorted_actual == sorted_expected diff --git a/tests/acceptance/test_acceptance/test_cmab.py b/tests/acceptance/test_acceptance/test_cmab.py new file mode 100644 index 00000000..7f5cddcf --- /dev/null +++ b/tests/acceptance/test_acceptance/test_cmab.py @@ -0,0 +1,358 @@ +import json +from tests.acceptance.helpers import ENDPOINT_DECIDE +from tests.acceptance.helpers import create_and_validate_request_and_response + + +# CMAB experiment configuration from datafile +CMAB_FLAG_KEY = "cmab_flag" +CMAB_EXPERIMENT_KEY = "cmab-rule_1" + + +def test_cmab_decision_basic(session_obj): + """ + Test validates basic CMAB decision flow. + + The test: + 1. Makes a decide request with a user who matches the CMAB audience (attr_1 == "hola") + 2. Verifies the decision returns a variation from the CMAB experiment + 3. Verifies the ruleKey matches the CMAB experiment key + + This test hits the real CMAB prediction endpoint via the agent. + """ + payload = { + "userId": "test_user_cmab_1", + "userAttributes": {"attr_1": "hola"}, + "decideOptions": ["INCLUDE_REASONS"] + } + + params = {"keys": CMAB_FLAG_KEY} + resp = create_and_validate_request_and_response( + ENDPOINT_DECIDE, + "post", + session_obj, + payload=json.dumps(payload), + params=params + ) + + result = resp.json() + + # Validate response + assert resp.status_code == 200, f"Expected 200, got {resp.status_code}: {result}" + assert result["flagKey"] == CMAB_FLAG_KEY + assert result["ruleKey"] == CMAB_EXPERIMENT_KEY + assert result["variationKey"] in ["on", "off"], f"Unexpected variation: {result['variationKey']}" + assert result["userContext"]["userId"] == "test_user_cmab_1" + assert result["userContext"]["attributes"]["attr_1"] == "hola" + + +def test_cmab_decision_different_users(session_obj): + """ + Test validates CMAB decisions for different users. + + Different users with same attributes should potentially get different variations + based on CMAB ML model predictions. + """ + users = ["cmab_user_1", "cmab_user_2", "cmab_user_3"] + variations_seen = set() + + for user_id in users: + payload = { + "userId": user_id, + "userAttributes": {"attr_1": "hola"}, + "decideOptions": [] + } + + params = {"keys": CMAB_FLAG_KEY} + resp = create_and_validate_request_and_response( + ENDPOINT_DECIDE, + "post", + session_obj, + payload=json.dumps(payload), + params=params + ) + + result = resp.json() + + assert resp.status_code == 200 + assert result["flagKey"] == CMAB_FLAG_KEY + assert result["ruleKey"] == CMAB_EXPERIMENT_KEY + variations_seen.add(result["variationKey"]) + + # Verify we got valid variations + assert variations_seen.issubset({"on", "off"}), f"Invalid variations: {variations_seen}" + + +def test_cmab_caching_same_user_same_attributes(session_obj): + """ + Test validates CMAB caching behavior. + + Same user with identical attributes should get the same variation + on subsequent requests (cache hit). + """ + payload = { + "userId": "test_user_cache", + "userAttributes": {"attr_1": "hola"}, + "decideOptions": [] + } + + params = {"keys": CMAB_FLAG_KEY} + + # First request + resp1 = create_and_validate_request_and_response( + ENDPOINT_DECIDE, + "post", + session_obj, + payload=json.dumps(payload), + params=params + ) + + result1 = resp1.json() + assert resp1.status_code == 200 + first_variation = result1["variationKey"] + + # Second request - should return same variation (cached) + resp2 = create_and_validate_request_and_response( + ENDPOINT_DECIDE, + "post", + session_obj, + payload=json.dumps(payload), + params=params + ) + + result2 = resp2.json() + assert resp2.status_code == 200 + assert result2["variationKey"] == first_variation, "Cache should return same variation" + + +def test_cmab_cache_invalidation_different_attributes(session_obj): + """ + Test validates cache invalidation when user attributes change. + + Same user with different attributes may get a different variation + (cache should be invalidated). + """ + user_id = "test_user_invalidate" + params = {"keys": CMAB_FLAG_KEY} + + # First request with attr_1 = "hola" + payload1 = { + "userId": user_id, + "userAttributes": {"attr_1": "hola"}, + "decideOptions": [] + } + + resp1 = create_and_validate_request_and_response( + ENDPOINT_DECIDE, + "post", + session_obj, + payload=json.dumps(payload1), + params=params + ) + + assert resp1.status_code == 200 + result1 = resp1.json() + assert result1["ruleKey"] == CMAB_EXPERIMENT_KEY + + # Second request with same user but additional attribute + # This should invalidate cache and potentially get different variation + payload2 = { + "userId": user_id, + "userAttributes": {"attr_1": "hola", "extra_attr": "test_value"}, + "decideOptions": [] + } + + resp2 = create_and_validate_request_and_response( + ENDPOINT_DECIDE, + "post", + session_obj, + payload=json.dumps(payload2), + params=params + ) + + assert resp2.status_code == 200 + result2 = resp2.json() + # Both should be CMAB decisions + assert result2["ruleKey"] == CMAB_EXPERIMENT_KEY + + +def test_cmab_audience_mismatch(session_obj): + """ + Test validates CMAB behavior when user doesn't match audience. + + User without matching attributes (attr_1 != "hola") should get + default rollout decision, not CMAB decision. + """ + payload = { + "userId": "test_user_no_match", + "userAttributes": {"attr_1": "adios"}, # Doesn't match audience requirement + "decideOptions": ["INCLUDE_REASONS"] + } + + params = {"keys": CMAB_FLAG_KEY} + resp = create_and_validate_request_and_response( + ENDPOINT_DECIDE, + "post", + session_obj, + payload=json.dumps(payload), + params=params + ) + + result = resp.json() + + # Should fall back to default rollout (off variation) + assert resp.status_code == 200 + assert result["flagKey"] == CMAB_FLAG_KEY + assert result["variationKey"] == "off" + assert result["enabled"] is False + # Rule key should be the default rollout, not the CMAB experiment + assert "default-rollout" in result["ruleKey"], f"Expected default rollout, got {result['ruleKey']}" + assert result["ruleKey"] != CMAB_EXPERIMENT_KEY + + +def test_cmab_no_attributes(session_obj): + """ + Test validates CMAB behavior when user has no attributes. + + User without any attributes should not match CMAB audience and + should get default rollout. + """ + payload = { + "userId": "test_user_no_attrs", + "userAttributes": {}, + "decideOptions": [] + } + + params = {"keys": CMAB_FLAG_KEY} + resp = create_and_validate_request_and_response( + ENDPOINT_DECIDE, + "post", + session_obj, + payload=json.dumps(payload), + params=params + ) + + result = resp.json() + + assert resp.status_code == 200 + assert result["flagKey"] == CMAB_FLAG_KEY + assert "default-rollout" in result["ruleKey"] + + +def test_cmab_with_forced_decision(session_obj): + """ + Test validates forced decisions override CMAB predictions. + + Forced decisions should take precedence over CMAB logic. + """ + payload = { + "userId": "test_user_forced", + "userAttributes": {"attr_1": "hola"}, + "decideOptions": ["INCLUDE_REASONS"], + "forcedDecisions": [ + { + "flagKey": CMAB_FLAG_KEY, + "ruleKey": CMAB_EXPERIMENT_KEY, + "variationKey": "off" + } + ] + } + + params = {"keys": CMAB_FLAG_KEY} + resp = create_and_validate_request_and_response( + ENDPOINT_DECIDE, + "post", + session_obj, + payload=json.dumps(payload), + params=params + ) + + result = resp.json() + + # Forced decision should override CMAB prediction + assert resp.status_code == 200 + assert result["variationKey"] == "off" + assert result["enabled"] is False + assert "forced decision" in result["reasons"][0].lower() + + +def test_cmab_decide_all_includes_cmab_flag(session_obj): + """ + Test validates that DecideAll includes CMAB flag decisions. + + When no specific flag keys are requested, the response should include + all flags including the CMAB flag. + """ + payload = { + "userId": "test_user_decide_all", + "userAttributes": {"attr_1": "hola"}, + "decideOptions": ["ENABLED_FLAGS_ONLY"] + } + + # No keys parameter = decide all + resp = create_and_validate_request_and_response( + ENDPOINT_DECIDE, + "post", + session_obj, + payload=json.dumps(payload), + bypass_validation_request=True, + bypass_validation_response=True + ) + + results = resp.json() + + assert resp.status_code == 200 + assert isinstance(results, list), "DecideAll should return a list" + + # Find the CMAB flag in results (may not be present if variation is "off" with ENABLED_FLAGS_ONLY) + cmab_result = None + for result in results: + if result["flagKey"] == CMAB_FLAG_KEY: + cmab_result = result + break + + # CMAB flag may not be in response when using ENABLED_FLAGS_ONLY if variation is "off" (enabled=false) + # If present, verify it's valid + if cmab_result is not None: + assert cmab_result["ruleKey"] == CMAB_EXPERIMENT_KEY + assert cmab_result["variationKey"] in ["on", "off"] + # When present in ENABLED_FLAGS_ONLY response, it must be enabled + assert cmab_result.get("enabled") is True, "CMAB flag should be enabled when present in ENABLED_FLAGS_ONLY response" + + +def test_cmab_with_multiple_keys(session_obj): + """ + Test validates CMAB decision when requested with multiple flag keys. + """ + payload = { + "userId": "test_user_multiple", + "userAttributes": {"attr_1": "hola"}, + "decideOptions": [] + } + + # Request multiple flags including CMAB flag + params = {"keys": ["feature_1", CMAB_FLAG_KEY, "feature_2"]} + resp = create_and_validate_request_and_response( + ENDPOINT_DECIDE, + "post", + session_obj, + payload=json.dumps(payload), + params=params + ) + + results = resp.json() + + assert resp.status_code == 200 + assert isinstance(results, list) + assert len(results) >= 3 # At least the 3 requested flags + + # Find CMAB flag result + cmab_result = None + for result in results: + if result["flagKey"] == CMAB_FLAG_KEY: + cmab_result = result + break + + assert cmab_result is not None + assert cmab_result["ruleKey"] == CMAB_EXPERIMENT_KEY + assert cmab_result["variationKey"] in ["on", "off"] + \ No newline at end of file diff --git a/tests/acceptance/test_acceptance/test_config.py b/tests/acceptance/test_acceptance/test_config.py index ef99efa8..e169664c 100644 --- a/tests/acceptance/test_acceptance/test_config.py +++ b/tests/acceptance/test_acceptance/test_config.py @@ -9,7 +9,7 @@ expected_config = """{ "environmentKey": "production", "sdkKey": "KZbunNn9bVfBWLpZPq2XC4", - "revision": "137", + "revision": "139", "experimentsMap": { "ab_test1": { "id": "16911963060", @@ -30,6 +30,25 @@ } } }, + "cmab-rule_1": { + "id": "9300002877087", + "key": "cmab-rule_1", + "audiences": "\\"Audience1\\"", + "variationsMap": { + "off": { + "id": "1579277", + "key": "off", + "featureEnabled": false, + "variablesMap": {} + }, + "on": { + "id": "1579278", + "key": "on", + "featureEnabled": true, + "variablesMap": {} + } + } + }, "feature_2_test": { "id": "16910084756", "key": "feature_2_test", @@ -113,6 +132,68 @@ } } }, + "cmab_flag": { + "id": "496419", + "key": "cmab_flag", + "experimentRules": [ + { + "id": "9300002877087", + "key": "cmab-rule_1", + "audiences": "\\"Audience1\\"", + "variationsMap": { + "off": { + "id": "1579277", + "key": "off", + "featureEnabled": false, + "variablesMap": {} + }, + "on": { + "id": "1579278", + "key": "on", + "featureEnabled": true, + "variablesMap": {} + } + } + } + ], + "deliveryRules": [ + { + "id": "default-rollout-496419-16935023792", + "key": "default-rollout-496419-16935023792", + "audiences": "", + "variationsMap": { + "off": { + "id": "1579277", + "key": "off", + "featureEnabled": false, + "variablesMap": {} + } + } + } + ], + "variablesMap": {}, + "experimentsMap": { + "cmab-rule_1": { + "id": "9300002877087", + "key": "cmab-rule_1", + "audiences": "\\"Audience1\\"", + "variationsMap": { + "off": { + "id": "1579277", + "key": "off", + "featureEnabled": false, + "variablesMap": {} + }, + "on": { + "id": "1579278", + "key": "on", + "featureEnabled": true, + "variablesMap": {} + } + } + } + } + }, "feature_1": { "id": "16925981047", "key": "feature_1", @@ -384,6 +465,7 @@ "key": "myevent", "experimentIds": [ "16911963060", + "9300002877087", "16910084756" ] } diff --git a/tests/acceptance/test_acceptance/test_decide.py b/tests/acceptance/test_acceptance/test_decide.py index e583e2ac..57354c3d 100644 --- a/tests/acceptance/test_acceptance/test_decide.py +++ b/tests/acceptance/test_acceptance/test_decide.py @@ -157,7 +157,7 @@ def test_decide__feature_no_ups(session_obj, flag_key, expected_response, expect resp.raise_for_status() -expected_flag_keys_with_ups = r"""[ +expected_flag_keys_with_ups_base = r"""[ { "variationKey": "16925940659", "enabled": true, @@ -251,7 +251,7 @@ def test_decide__feature_no_ups(session_obj, flag_key, expected_response, expect } ]""" -expected_flag_keys_no_ups = r"""[ +expected_flag_keys_no_ups_base = r"""[ { "variationKey": "16925940659", "enabled": true, @@ -331,16 +331,16 @@ def test_decide__feature_no_ups(session_obj, flag_key, expected_response, expect @pytest.mark.parametrize( - "parameters, expected_response, expected_status_code, bypass_validation_request, bypass_validation_response", [ - ({}, expected_flag_keys_with_ups, 200, True, True), - ({"keys": []}, expected_flag_keys_with_ups, 200, True, True), + "parameters, expected_response, expected_status_code, bypass_validation_request, bypass_validation_response, is_decide_all", [ + ({}, expected_flag_keys_with_ups_base, 200, True, True, True), + ({"keys": []}, expected_flag_keys_with_ups_base, 200, True, True, True), ({"keys": ["feature_1", "feature_2", "feature_4", "feature_5"]}, - expected_flag_key__multiple_parameters_with_ups, 200, True, True), + expected_flag_key__multiple_parameters_with_ups, 200, True, True, False), ], ids=["missig_flagkey_parameter", "no flag key specified", "multiple parameters"]) def test_decide__flag_key_parameter_with_ups(session_obj, parameters, expected_response, expected_status_code, bypass_validation_request, - bypass_validation_response): + bypass_validation_response, is_decide_all): """ Test validates: That no required parameter and empty param return identical response. @@ -352,6 +352,7 @@ def test_decide__flag_key_parameter_with_ups(session_obj, parameters, expected_r :param parameters: sesison obj, params, expected, expected status code :param expected_response: expected_flag_keys :param expected_status_code: 200 + :param is_decide_all: whether this is decide all (needs CMAB handling) """ payload = """ { @@ -370,23 +371,44 @@ def test_decide__flag_key_parameter_with_ups(session_obj, parameters, expected_r payload=payload, params=params) - sorted_actual = sort_response(resp.json(), "flagKey") - sorted_expected = sort_response(json.loads(expected_response), "flagKey") - - assert sorted_actual == sorted_expected + if is_decide_all: + # For decide all, handle CMAB flag separately + actual_results = resp.json() + expected_base = json.loads(expected_response) + + # Find CMAB flag (may not be present if ENABLED_FLAGS_ONLY and variation is "off") + cmab_result = next((r for r in actual_results if r['flagKey'] == 'cmab_flag'), None) + if cmab_result is not None: + # If CMAB is present, verify it's valid + assert cmab_result['variationKey'] in ['on', 'off'] + assert cmab_result['ruleKey'] == 'cmab-rule_1' + assert 'userContext' in cmab_result + # CMAB should only appear if enabled=true (when using ENABLED_FLAGS_ONLY) + assert cmab_result.get('enabled') is True, "CMAB flag should be enabled when present in ENABLED_FLAGS_ONLY response" + + # Verify base flags (excluding CMAB) + base_results = [r for r in actual_results if r['flagKey'] != 'cmab_flag'] + sorted_actual = sort_response(base_results, "flagKey") + sorted_expected = sort_response(expected_base, "flagKey") + assert sorted_actual == sorted_expected + else: + # For specific flag keys, exact match + sorted_actual = sort_response(resp.json(), "flagKey") + sorted_expected = sort_response(json.loads(expected_response), "flagKey") + assert sorted_actual == sorted_expected @pytest.mark.parametrize( - "parameters, expected_response, expected_status_code, bypass_validation_request, bypass_validation_response", [ - ({}, expected_flag_keys_no_ups, 200, True, True), - ({"keys": []}, expected_flag_keys_no_ups, 200, True, True), + "parameters, expected_response, expected_status_code, bypass_validation_request, bypass_validation_response, is_decide_all", [ + ({}, expected_flag_keys_no_ups_base, 200, True, True, True), + ({"keys": []}, expected_flag_keys_no_ups_base, 200, True, True, True), ({"keys": ["feature_1", "feature_2", "feature_4", "feature_5"]}, - expected_flag_key__multiple_parameters_no_ups, 200, True, True), + expected_flag_key__multiple_parameters_no_ups, 200, True, True, False), ], ids=["missig_flagkey_parameter_no_ups", "no flag key specified_no_ups", "multiple parameters_no_ups"]) def test_decide__flag_key_parameter_no_ups(session_obj, parameters, expected_response, expected_status_code, bypass_validation_request, - bypass_validation_response): + bypass_validation_response, is_decide_all): """ This test is required to be run on Agent on Amazon Web Services. It is only used there. And it is excluded from the test run in this repo. @@ -394,6 +416,7 @@ def test_decide__flag_key_parameter_no_ups(session_obj, parameters, expected_res :param parameters: sesison obj, params, expected, expected status code :param expected_response: expected_flag_keys :param expected_status_code: 200 + :param is_decide_all: whether this is decide all (needs CMAB handling) """ payload = """ { @@ -412,10 +435,31 @@ def test_decide__flag_key_parameter_no_ups(session_obj, parameters, expected_res payload=payload, params=params) - sorted_actual = sort_response(resp.json(), "flagKey") - sorted_expected = sort_response(json.loads(expected_response), "flagKey") - - assert sorted_actual == sorted_expected + if is_decide_all: + # For decide all, handle CMAB flag separately + actual_results = resp.json() + expected_base = json.loads(expected_response) + + # Find CMAB flag (may not be present if ENABLED_FLAGS_ONLY and variation is "off") + cmab_result = next((r for r in actual_results if r['flagKey'] == 'cmab_flag'), None) + if cmab_result is not None: + # If CMAB is present, verify it's valid + assert cmab_result['variationKey'] in ['on', 'off'] + assert cmab_result['ruleKey'] == 'cmab-rule_1' + assert 'userContext' in cmab_result + # CMAB should only appear if enabled=true (when using ENABLED_FLAGS_ONLY) + assert cmab_result.get('enabled') is True, "CMAB flag should be enabled when present in ENABLED_FLAGS_ONLY response" + + # Verify base flags (excluding CMAB) + base_results = [r for r in actual_results if r['flagKey'] != 'cmab_flag'] + sorted_actual = sort_response(base_results, "flagKey") + sorted_expected = sort_response(expected_base, "flagKey") + assert sorted_actual == sorted_expected + else: + # For specific flag keys, exact match + sorted_actual = sort_response(resp.json(), "flagKey") + sorted_expected = sort_response(json.loads(expected_response), "flagKey") + assert sorted_actual == sorted_expected @pytest.mark.parametrize(