diff --git a/v3/src/main/java/com/skyflow/utils/Constants.java b/v3/src/main/java/com/skyflow/utils/Constants.java index cfc29792..33da8b27 100644 --- a/v3/src/main/java/com/skyflow/utils/Constants.java +++ b/v3/src/main/java/com/skyflow/utils/Constants.java @@ -7,11 +7,11 @@ public final class Constants extends BaseConstants { public static final String SDK_PREFIX = SDK_NAME + SDK_VERSION; public static final Integer INSERT_BATCH_SIZE = 50; public static final Integer MAX_INSERT_BATCH_SIZE = 1000; - public static final Integer INSERT_CONCURRENCY_LIMIT = 10; + public static final Integer INSERT_CONCURRENCY_LIMIT = 1; public static final Integer MAX_INSERT_CONCURRENCY_LIMIT = 10; public static final Integer DETOKENIZE_BATCH_SIZE = 50; - public static final Integer DETOKENIZE_CONCURRENCY_LIMIT = 10; + public static final Integer DETOKENIZE_CONCURRENCY_LIMIT = 1; public static final Integer MAX_DETOKENIZE_BATCH_SIZE = 1000; public static final Integer MAX_DETOKENIZE_CONCURRENCY_LIMIT = 10; diff --git a/v3/src/main/java/com/skyflow/utils/validations/Validations.java b/v3/src/main/java/com/skyflow/utils/validations/Validations.java index 058a002e..af2ad382 100644 --- a/v3/src/main/java/com/skyflow/utils/validations/Validations.java +++ b/v3/src/main/java/com/skyflow/utils/validations/Validations.java @@ -93,37 +93,39 @@ public static void validateDetokenizeRequest(DetokenizeRequest request) throws S if (tokens == null || tokens.isEmpty()) { LogUtil.printErrorLog(Utils.parameterizedString( ErrorLogs.EMPTY_DETOKENIZE_DATA.getLog(), InterfaceName.DETOKENIZE.getName() - )); - throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyDetokenizeData.getMessage()); + )); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyDetokenizeData.getMessage()); } - for (String token : tokens) { + for (int index = 0; index < tokens.size(); index++) { + String token = tokens.get(index); if (token == null || token.trim().isEmpty()) { LogUtil.printErrorLog(Utils.parameterizedString( - ErrorLogs.EMPTY_OR_NULL_TOKEN_IN_DETOKENIZE_DATA.getLog(), InterfaceName.DETOKENIZE.getName() - )); + ErrorLogs.EMPTY_OR_NULL_TOKEN_IN_DETOKENIZE_DATA.getLog(), + InterfaceName.DETOKENIZE.getName(), + String.valueOf(index))); throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyTokenInDetokenizeData.getMessage()); } } + List groupRedactions = request.getTokenGroupRedactions(); if (groupRedactions != null && !groupRedactions.isEmpty()) { - for (TokenGroupRedactions group : groupRedactions) { - if (group == null) { - LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.NULL_TOKEN_REDACTION_GROUP_OBJECT.getLog(), InterfaceName.DETOKENIZE.getName())); - throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullTokenGroupRedactions.getMessage()); - } - String groupName = group.getTokenGroupName(); - String redaction = group.getRedaction(); - if (groupName == null || groupName.trim().isEmpty()) { - LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.NULL_TOKEN_GROUP_NAME_IN_TOKEN_GROUP.getLog(), InterfaceName.DETOKENIZE.getName())); - throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullTokenGroupNameInTokenGroup.getMessage()); - } - if (redaction == null || redaction.trim().isEmpty()) { - LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.EMPTY_OR_NULL_REDACTION_IN_TOKEN_GROUP.getLog(), InterfaceName.DETOKENIZE.getName())); - throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullRedactionInTokenGroup.getMessage()); - } + for (TokenGroupRedactions group : groupRedactions) { + if (group == null) { + LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.NULL_TOKEN_REDACTION_GROUP_OBJECT.getLog(), InterfaceName.DETOKENIZE.getName())); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullTokenGroupRedactions.getMessage()); + } + String groupName = group.getTokenGroupName(); + String redaction = group.getRedaction(); + if (groupName == null || groupName.trim().isEmpty()) { + LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.NULL_TOKEN_GROUP_NAME_IN_TOKEN_GROUP.getLog(), InterfaceName.DETOKENIZE.getName())); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullTokenGroupNameInTokenGroup.getMessage()); + } + if (redaction == null || redaction.trim().isEmpty()) { + LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.EMPTY_OR_NULL_REDACTION_IN_TOKEN_GROUP.getLog(), InterfaceName.DETOKENIZE.getName())); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullRedactionInTokenGroup.getMessage()); } } - + } } public static void validateVaultConfiguration(VaultConfig vaultConfig) throws SkyflowException { diff --git a/v3/src/main/java/com/skyflow/vault/controller/VaultController.java b/v3/src/main/java/com/skyflow/vault/controller/VaultController.java index 6b5af623..ca54230a 100644 --- a/v3/src/main/java/com/skyflow/vault/controller/VaultController.java +++ b/v3/src/main/java/com/skyflow/vault/controller/VaultController.java @@ -19,6 +19,7 @@ import com.skyflow.utils.validations.Validations; import com.skyflow.vault.data.*; import io.github.cdimascio.dotenv.Dotenv; +import io.github.cdimascio.dotenv.DotenvException; import java.util.ArrayList; import java.util.Collections; @@ -309,9 +310,22 @@ private InsertResponse insertBatch(List batch, String tableNam private void configureInsertConcurrencyAndBatchSize(int totalRequests) { try { - Dotenv dotenv = Dotenv.load(); - String userProvidedBatchSize = dotenv.get("INSERT_BATCH_SIZE"); - String userProvidedConcurrencyLimit = dotenv.get("INSERT_CONCURRENCY_LIMIT"); + String userProvidedBatchSize = System.getenv("INSERT_BATCH_SIZE"); + String userProvidedConcurrencyLimit = System.getenv("INSERT_CONCURRENCY_LIMIT"); + + Dotenv dotenv = null; + try { + dotenv = Dotenv.load(); + } catch (DotenvException ignored) { + // ignore the case if .env file is not found + } + + if (userProvidedBatchSize == null && dotenv != null) { + userProvidedBatchSize = dotenv.get("INSERT_BATCH_SIZE"); + } + if (userProvidedConcurrencyLimit == null && dotenv != null) { + userProvidedConcurrencyLimit = dotenv.get("INSERT_CONCURRENCY_LIMIT"); + } if (userProvidedBatchSize != null) { try { @@ -365,9 +379,22 @@ private void configureInsertConcurrencyAndBatchSize(int totalRequests) { private void configureDetokenizeConcurrencyAndBatchSize(int totalRequests) { try { - Dotenv dotenv = Dotenv.load(); - String userProvidedBatchSize = dotenv.get("DETOKENIZE_BATCH_SIZE"); - String userProvidedConcurrencyLimit = dotenv.get("DETOKENIZE_CONCURRENCY_LIMIT"); + String userProvidedBatchSize = System.getenv("DETOKENIZE_BATCH_SIZE"); + String userProvidedConcurrencyLimit = System.getenv("DETOKENIZE_BATCH_SIZE"); + + Dotenv dotenv = null; + try { + dotenv = Dotenv.load(); + } catch (DotenvException ignored) { + // ignore the case if .env file is not found + } + + if (userProvidedBatchSize == null && dotenv != null) { + userProvidedBatchSize = dotenv.get("DETOKENIZE_BATCH_SIZE"); + } + if (userProvidedConcurrencyLimit == null && dotenv != null) { + userProvidedConcurrencyLimit = dotenv.get("DETOKENIZE_BATCH_SIZE"); + } if (userProvidedBatchSize != null) { try { diff --git a/v3/test/java/com/skyflow/vault/controller/VaultControllerTests.java b/v3/test/java/com/skyflow/vault/controller/VaultControllerTests.java index 811a9891..605d29eb 100644 --- a/v3/test/java/com/skyflow/vault/controller/VaultControllerTests.java +++ b/v3/test/java/com/skyflow/vault/controller/VaultControllerTests.java @@ -395,7 +395,7 @@ public void testFractionalLastBatch() throws Exception { // Last batch should have 50 records, concurrency should be 101 assertEquals(100, getPrivateInt(controller, "insertBatchSize")); - assertEquals(10, getPrivateInt(controller, "insertConcurrencyLimit")); + assertEquals(Constants.INSERT_CONCURRENCY_LIMIT.intValue(), getPrivateInt(controller, "insertConcurrencyLimit")); } private int getPrivateInt(Object obj, String field) throws Exception {