diff --git a/README.md b/README.md index dbe86ee..cc2a78a 100644 --- a/README.md +++ b/README.md @@ -239,8 +239,8 @@ from skyflow.utils.enums import RedactionType detokenize_request = DetokenizeRequest( data=[ - {'token': 'token1', 'redaction': RedactionType.PLAIN_TEXT}, - {'token': 'token2', 'redaction': RedactionType.PLAIN_TEXT} + {'token': 'token1', 'redaction_type': RedactionType.PLAIN_TEXT}, + {'token': 'token2', 'redaction_type': RedactionType.PLAIN_TEXT} ], continue_on_error=True ) diff --git a/samples/vault_api/detokenize_records.py b/samples/vault_api/detokenize_records.py index e93d5a1..d0d10e0 100644 --- a/samples/vault_api/detokenize_records.py +++ b/samples/vault_api/detokenize_records.py @@ -55,11 +55,11 @@ def perform_detokenization(): detokenize_data = [ { 'token': '', # Token to be detokenized - 'redaction': RedactionType.REDACTED + 'redaction_type': RedactionType.REDACTED }, { 'token': '', # Token to be detokenized - 'redaction': RedactionType.MASKED + 'redaction_type': RedactionType.MASKED } ] diff --git a/skyflow/utils/_skyflow_messages.py b/skyflow/utils/_skyflow_messages.py index 8e65eba..232bd8b 100644 --- a/skyflow/utils/_skyflow_messages.py +++ b/skyflow/utils/_skyflow_messages.py @@ -122,7 +122,7 @@ class Error(Enum): INVOKE_CONNECTION_FAILED = f"{error_prefix} Invoke Connection operation failed." INVALID_IDS_TYPE = f"{error_prefix} Validation error. 'ids' has a value of type {{}}. Specify 'ids' as list." - INVALID_REDACTION_TYPE = f"{error_prefix} Validation error. 'redaction' has a value of type {{}}. Specify 'redaction' as type Skyflow.RedactionType." + INVALID_REDACTION_TYPE = f"{error_prefix} Validation error. 'redaction_type' has a value of type {{}}. Specify 'redaction_type' as type Skyflow.RedactionType." INVALID_COLUMN_NAME = f"{error_prefix} Validation error. column_name has a value of type {{}}. Specify 'column' as a string." INVALID_COLUMN_VALUE = f"{error_prefix} Validation error. column_values key has a value of type {{}}. Specify column_values key as list." INVALID_COLUMN_VALUES = f"{error_prefix} Validation error. column_values key is an empty list. Specify at least one column value when column_name is passed." @@ -131,7 +131,7 @@ class Error(Enum): INVALID_OFF_SET_VALUE = f"{error_prefix} Validation error. offset key has a value of type {{}}. Specify offset key as integer." INVALID_LIMIT_VALUE = f"{error_prefix} Validation error. limit key has a value of type {{}}. Specify limit key as integer." INVALID_DOWNLOAD_URL_VALUE = f"{error_prefix} Validation error. download_url key has a value of type {{}}. Specify download_url key as boolean." - REDACTION_WITH_TOKENS_NOT_SUPPORTED = f"{error_prefix} Validation error. 'redaction' can't be used when tokens are specified. Remove 'redaction' from payload if tokens are specified." + REDACTION_WITH_TOKENS_NOT_SUPPORTED = f"{error_prefix} Validation error. 'redaction_type' can't be used when tokens are specified. Remove 'redaction_type' from payload if tokens are specified." TOKENS_GET_COLUMN_NOT_SUPPORTED = f"{error_prefix} Validation error. Column name and/or column values can't be used when tokens are specified. Remove unique column values or tokens from the payload." BOTH_IDS_AND_COLUMN_DETAILS_SPECIFIED = f"{error_prefix} Validation error. Both Skyflow IDs and column details can't be specified. Either specify Skyflow IDs or unique column details." INVALID_ORDER_BY_VALUE = f"{error_prefix} Validation error. order_by key has a value of type {{}}. Specify order_by key as Skyflow.OrderBy" @@ -139,7 +139,7 @@ class Error(Enum): UPDATE_FIELD_KEY_ERROR = f"{error_prefix} Validation error. Fields are empty in an update payload. Specify at least one field." INVALID_FIELDS_TYPE = f"{error_prefix} Validation error. The 'data' key has a value of type {{}}. Specify 'data' as a dictionary." IDS_KEY_ERROR = f"{error_prefix} Validation error. 'ids' key is missing from the payload. Specify an 'ids' key." - INVALID_TOKENS_LIST_VALUE = f"{error_prefix} Validation error. The 'data' field is invalid. Specify 'data' as a list of dictionaries containing 'token' and 'redaction'." + INVALID_TOKENS_LIST_VALUE = f"{error_prefix} Validation error. The 'data' field is invalid. Specify 'data' as a list of dictionaries containing 'token' and 'redaction_type'." INVALID_DATA_FOR_DETOKENIZE = f"{error_prefix}" EMPTY_TOKENS_LIST_VALUE = f"{error_prefix} Validation error. Tokens are empty in detokenize payload. Specify at lease one token" INVALID_TOKEN_TYPE = f"{ERROR}: [{error_prefix}] Invalid {{}} request. Tokens should be of type string." @@ -417,6 +417,9 @@ class HttpStatus(Enum): BAD_REQUEST = "Bad Request" class Warning(Enum): + DETOKENIZE_REDACTION_KEY_DEPRECATED = ( + f"{WARN}: [{error_prefix}] 'redaction' key in detokenize data is deprecated and will be removed in a future version. Use 'redaction_type' instead." + ) UPDATE_LOG_LEVEL_DEPRECATED = ( f"{WARN}: [{error_prefix}] Skyflow.update_log_level() is deprecated. " "Use Skyflow.set_log_level() instead." diff --git a/skyflow/utils/constants.py b/skyflow/utils/constants.py index 17ba96e..05d2838 100644 --- a/skyflow/utils/constants.py +++ b/skyflow/utils/constants.py @@ -174,6 +174,7 @@ class RequestParameter: VALUE = 'value' COLUMN_GROUP = 'column_group' REDACTION = 'redaction' + REDACTION_TYPE = 'redaction_type' class FileUploadField: diff --git a/skyflow/utils/validations/_validations.py b/skyflow/utils/validations/_validations.py index 6cc2c81..42abe18 100644 --- a/skyflow/utils/validations/_validations.py +++ b/skyflow/utils/validations/_validations.py @@ -11,7 +11,7 @@ FileUploadField, DeidentifyFileRequestField, RequestOperation, ConfigType, SqlCommand, ConfigField, OptionField, CredentialField, Detect ) -from skyflow.utils.logger import log_info, log_error_log +from skyflow.utils.logger import log_info, log_warn, log_error_log from skyflow.vault.detect import DeidentifyTextRequest, ReidentifyTextRequest, TokenFormat, Transformations, \ GetDetectRunRequest, Bleep, DeidentifyFileRequest from skyflow.vault.detect._file_input import FileInput @@ -713,7 +713,17 @@ def validate_detokenize_request(logger, request): invalid_input_error_code) token = item.get(ResponseField.TOKEN) - redaction = item.get(RequestParameter.REDACTION, None) + + has_redaction = RequestParameter.REDACTION in item + has_redaction_type = RequestParameter.REDACTION_TYPE in item + + if has_redaction: + log_warn(SkyflowMessages.Warning.DETOKENIZE_REDACTION_KEY_DEPRECATED.value, logger) + + if has_redaction_type: + redaction = item.get(RequestParameter.REDACTION_TYPE) + else: + redaction = item.get(RequestParameter.REDACTION, None) if not isinstance(token, str) or not token: raise SkyflowError(SkyflowMessages.Error.INVALID_TOKEN_TYPE.value.format(RequestOperation.DETOKENIZE), diff --git a/skyflow/vault/controller/_vault.py b/skyflow/vault/controller/_vault.py index 7d51ee8..fd085e3 100644 --- a/skyflow/vault/controller/_vault.py +++ b/skyflow/vault/controller/_vault.py @@ -223,7 +223,7 @@ def detokenize(self, request: DetokenizeRequest): tokens_list = [ V1DetokenizeRecordRequest( token=item.get(ResponseField.TOKEN), - redaction=item.get(RequestParameter.REDACTION, RedactionType.DEFAULT) + redaction=item.get(RequestParameter.REDACTION_TYPE) or item.get(RequestParameter.REDACTION, RedactionType.DEFAULT) ) for item in request.data ] diff --git a/tests/utils/validations/test__validations.py b/tests/utils/validations/test__validations.py index ec4d5be..c5ad6b7 100644 --- a/tests/utils/validations/test__validations.py +++ b/tests/utils/validations/test__validations.py @@ -1057,11 +1057,36 @@ def test_validate_detokenize_request_invalid_continue_on_error_type(self): self.assertEqual(context.exception.message, SkyflowMessages.Error.INVALID_CONTINUE_ON_ERROR_TYPE.value) def test_validate_detokenize_request_invalid_redaction_type(self): - request = DetokenizeRequest(data=[{"token": "token123", "redaction": "invalid"}], continue_on_error=False) + request = DetokenizeRequest(data=[{"token": "token123", "redaction_type": "invalid"}], continue_on_error=False) with self.assertRaises(SkyflowError) as context: validate_detokenize_request(self.logger, request) self.assertEqual(context.exception.message, SkyflowMessages.Error.INVALID_REDACTION_TYPE.value.format(str(type("invalid")))) + def test_validate_detokenize_request_deprecated_redaction_key_emits_warn(self): + from unittest.mock import patch + request = DetokenizeRequest(data=[{"token": "token123", "redaction": RedactionType.PLAIN_TEXT}], continue_on_error=False) + with patch('skyflow.utils.validations._validations.log_warn') as mock_warn: + validate_detokenize_request(self.logger, request) + mock_warn.assert_called_once() + self.assertIn("redaction_type", mock_warn.call_args[0][0]) + + def test_validate_detokenize_request_both_keys_prioritizes_redaction_type_and_warns(self): + from unittest.mock import patch + request = DetokenizeRequest( + data=[{"token": "token123", "redaction": RedactionType.PLAIN_TEXT, "redaction_type": RedactionType.MASKED}], + continue_on_error=False + ) + with patch('skyflow.utils.validations._validations.log_warn') as mock_warn: + validate_detokenize_request(self.logger, request) + mock_warn.assert_called_once() + + def test_validate_detokenize_request_redaction_type_only_no_warn(self): + from unittest.mock import patch + request = DetokenizeRequest(data=[{"token": "token123", "redaction_type": RedactionType.PLAIN_TEXT}], continue_on_error=False) + with patch('skyflow.utils.validations._validations.log_warn') as mock_warn: + validate_detokenize_request(self.logger, request) + mock_warn.assert_not_called() + def test_validate_deidentify_file_request_wait_time_negative(self): file_input = FileInput(file_path=self.temp_file_path)