diff --git a/v3/src/main/java/com/skyflow/utils/Utils.java b/v3/src/main/java/com/skyflow/utils/Utils.java index 83b40f9b..ec5fd6d8 100644 --- a/v3/src/main/java/com/skyflow/utils/Utils.java +++ b/v3/src/main/java/com/skyflow/utils/Utils.java @@ -1,5 +1,10 @@ package com.skyflow.utils; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import com.google.gson.JsonObject; import com.skyflow.enums.Env; import com.skyflow.errors.ErrorCode; @@ -17,14 +22,10 @@ import com.skyflow.vault.data.ErrorRecord; import com.skyflow.vault.data.Success; import com.skyflow.vault.data.Token; + import io.github.cdimascio.dotenv.Dotenv; import io.github.cdimascio.dotenv.DotenvException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - public final class Utils extends BaseUtils { public static String getVaultURL(String clusterId, Env env) { @@ -92,14 +93,14 @@ public static ErrorRecord createErrorRecord(Map recordMap, int i } public static List handleBatchException( - Throwable ex, List batch, int batchNumber + Throwable ex, List batch, int batchNumber, int batchSize ) { List errorRecords = new ArrayList<>(); Throwable cause = ex.getCause(); if (cause instanceof ApiClientApiException) { ApiClientApiException apiException = (ApiClientApiException) cause; Map responseBody = (Map) apiException.body(); - int indexNumber = batchNumber > 0 ? batchNumber * batch.size() : 0; + int indexNumber = batchNumber > 0 ? batchNumber * batchSize : 0; if (responseBody != null) { if (responseBody.containsKey("records")) { Object recordss = responseBody.get("records"); @@ -124,7 +125,7 @@ public static List handleBatchException( } } } else { - int indexNumber = batchNumber > 0 ? batchNumber * batch.size() : 0; + int indexNumber = batchNumber > 0 ? batchNumber * batchSize: 0; for (int j = 0; j < batch.size(); j++) { ErrorRecord err = new ErrorRecord(indexNumber, ex.getMessage(), 500); errorRecords.add(err); diff --git a/v3/src/main/java/com/skyflow/vault/controller/VaultController.java b/v3/src/main/java/com/skyflow/vault/controller/VaultController.java index 0a3f5f87..0604f00f 100644 --- a/v3/src/main/java/com/skyflow/vault/controller/VaultController.java +++ b/v3/src/main/java/com/skyflow/vault/controller/VaultController.java @@ -1,5 +1,13 @@ package com.skyflow.vault.controller; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.skyflow.VaultClient; @@ -17,20 +25,15 @@ import com.skyflow.utils.Utils; import com.skyflow.utils.logger.LogUtil; import com.skyflow.utils.validations.Validations; -import com.skyflow.vault.data.*; -import io.github.cdimascio.dotenv.Dotenv; -import io.github.cdimascio.dotenv.DotenvException; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import com.skyflow.vault.data.DetokenizeRequest; +import com.skyflow.vault.data.DetokenizeResponse; +import com.skyflow.vault.data.DetokenizeResponseObject; +import com.skyflow.vault.data.ErrorRecord; +import com.skyflow.vault.data.InsertRecord; +import com.skyflow.vault.data.InsertRequest; +import com.skyflow.vault.data.Success; -import static com.skyflow.utils.Utils.*; +import io.github.cdimascio.dotenv.Dotenv; public final class VaultController extends VaultClient { private static final Gson gson = new GsonBuilder().serializeNulls().create(); @@ -250,9 +253,9 @@ private List> detokenizeBatchFutures(Execu int batchNumber = batchIndex; CompletableFuture future = CompletableFuture .supplyAsync(() -> processDetokenizeBatch(batch), executor) - .thenApply(response -> formatDetokenizeResponse(response, batchNumber, detokenizeBatchSize)) + .thenApply(response -> Utils.formatDetokenizeResponse(response, batchNumber, detokenizeBatchSize)) .exceptionally(ex -> { - errorTokens.addAll(handleDetokenizeBatchException(ex, batch, batchNumber, detokenizeBatchSize)); + errorTokens.addAll(Utils.handleDetokenizeBatchException(ex, batch, batchNumber, detokenizeBatchSize)); return null; }); futures.add(future); @@ -285,9 +288,9 @@ private com.skyflow.generated.rest.types.DetokenizeResponse processDetokenizeBat int batchNumber = batchIndex; CompletableFuture future = CompletableFuture .supplyAsync(() -> insertBatch(batch, insertRequest.getTableName().isPresent() ? insertRequest.getTableName().get() : null, upsert), executor) - .thenApply(response -> formatResponse(response, batchNumber, insertBatchSize)) + .thenApply(response -> Utils.formatResponse(response, batchNumber, insertBatchSize)) .exceptionally(ex -> { - errorRecords.addAll(handleBatchException(ex, batch, batchNumber)); + errorRecords.addAll(Utils.handleBatchException(ex, batch, batchNumber, insertBatchSize)); return null; }); futures.add(future); @@ -313,22 +316,9 @@ private InsertResponse insertBatch(List batch, String tableNam private void configureInsertConcurrencyAndBatchSize(int totalRequests) { try { - String userProvidedBatchSize = System.getenv("INSERT_BATCH_SIZE"); - String userProvidedConcurrencyLimit = System.getenv("INSERT_CONCURRENCY_LIMIT"); - - Dotenv dotenv = null; - try { - dotenv = Dotenv.load(); - } catch (DotenvException ignored) { - // ignore the case if .env file is not found - } - - if (userProvidedBatchSize == null && dotenv != null) { - userProvidedBatchSize = dotenv.get("INSERT_BATCH_SIZE"); - } - if (userProvidedConcurrencyLimit == null && dotenv != null) { - userProvidedConcurrencyLimit = dotenv.get("INSERT_CONCURRENCY_LIMIT"); - } + Dotenv dotenv = Dotenv.load(); + String userProvidedBatchSize = dotenv.get("INSERT_BATCH_SIZE"); + String userProvidedConcurrencyLimit = dotenv.get("INSERT_CONCURRENCY_LIMIT"); if (userProvidedBatchSize != null) { try { @@ -382,22 +372,9 @@ private void configureInsertConcurrencyAndBatchSize(int totalRequests) { private void configureDetokenizeConcurrencyAndBatchSize(int totalRequests) { try { - String userProvidedBatchSize = System.getenv("DETOKENIZE_BATCH_SIZE"); - String userProvidedConcurrencyLimit = System.getenv("DETOKENIZE_BATCH_SIZE"); - - Dotenv dotenv = null; - try { - dotenv = Dotenv.load(); - } catch (DotenvException ignored) { - // ignore the case if .env file is not found - } - - if (userProvidedBatchSize == null && dotenv != null) { - userProvidedBatchSize = dotenv.get("DETOKENIZE_BATCH_SIZE"); - } - if (userProvidedConcurrencyLimit == null && dotenv != null) { - userProvidedConcurrencyLimit = dotenv.get("DETOKENIZE_BATCH_SIZE"); - } + Dotenv dotenv = Dotenv.load(); + String userProvidedBatchSize = dotenv.get("DETOKENIZE_BATCH_SIZE"); + String userProvidedConcurrencyLimit = dotenv.get("DETOKENIZE_CONCURRENCY_LIMIT"); if (userProvidedBatchSize != null) { try { diff --git a/v3/src/test/java/com/skyflow/utils/UtilsTests.java b/v3/src/test/java/com/skyflow/utils/UtilsTests.java index 5692af54..f360c7d2 100644 --- a/v3/src/test/java/com/skyflow/utils/UtilsTests.java +++ b/v3/src/test/java/com/skyflow/utils/UtilsTests.java @@ -19,6 +19,7 @@ import java.util.*; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class UtilsTests { private static final String INVALID_EXCEPTION_THROWN = "Should not have thrown any exception"; @@ -41,30 +42,6 @@ public static void setup() { SdkVersion.setSdkPrefix(Constants.SDK_PREFIX); } - public static List createDetokenizeBatches(com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest request, int batchSize) { - List detokenizeRequests = new ArrayList<>(); - List tokens = request.getTokens().get(); - - for (int i = 0; i < tokens.size(); i += batchSize) { - // Create a sublist for the current batch - List batchTokens = tokens.subList(i, Math.min(i + batchSize, tokens.size())); - List tokenGroupRedactions = null; - if (request.getTokenGroupRedactions().isPresent() && !request.getTokenGroupRedactions().get().isEmpty() && i < request.getTokenGroupRedactions().get().size()) { - tokenGroupRedactions = request.getTokenGroupRedactions().get().subList(i, Math.min(i + batchSize, request.getTokenGroupRedactions().get().size())); - } - // Build a new DetokenizeRequest for the current batch - com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest batchRequest = com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest.builder() - .vaultId(request.getVaultId()) - .tokens(new ArrayList<>(batchTokens)) - .tokenGroupRedactions(tokenGroupRedactions) - .build(); - - detokenizeRequests.add(batchRequest); - } - - return detokenizeRequests; - } - @Test public void testGetVaultURL() { // Test with production environment @@ -81,7 +58,6 @@ public void testGetVaultURL() { devUrl ); } - @Test(expected = NullPointerException.class) public void testGetVaultURLWithNullEnv() { Utils.getVaultURL("abc123", null); @@ -272,7 +248,7 @@ public void testHandleBatchExceptionApiClientExceptionWithSingleError() { ApiClientApiException apiException = new ApiClientApiException("Forbidden", 403, responseBody); Exception exception = new Exception("Test exception", apiException); - List errors = Utils.handleBatchException(exception, batch, 0); + List errors = Utils.handleBatchException(exception, batch, 0, 1); Assert.assertEquals("Should have errors for all records", 2, errors.size()); Assert.assertEquals("Error message should be same", "Test exception", errors.get(0).getError()); @@ -288,7 +264,7 @@ public void testHandleBatchExceptionWithNonApiClientException() { RuntimeException exception = new RuntimeException("Unexpected error"); - List errors = Utils.handleBatchException(exception, batch, 0); + List errors = Utils.handleBatchException(exception, batch, 0, 1); Assert.assertEquals("Should have errors for all records", 2, errors.size()); Assert.assertEquals("Error message should match", "Unexpected error", errors.get(0).getError()); @@ -304,11 +280,10 @@ public void testHandleBatchExceptionWithNonZeroBatchNumber() { RuntimeException exception = new RuntimeException("Batch error"); - List errors = Utils.handleBatchException(exception, batch, 1); - + List errors = Utils.handleBatchException(exception, batch, 1, 1); Assert.assertEquals("Should have errors for all records", 2, errors.size()); - Assert.assertEquals("First error index should be offset", 2, errors.get(0).getIndex()); - Assert.assertEquals("Second error index should be offset", 3, errors.get(1).getIndex()); + Assert.assertEquals("First error index should be offset", 1, errors.get(0).getIndex()); + Assert.assertEquals("Second error index should be offset", 2, errors.get(1).getIndex()); } @Test @@ -319,7 +294,7 @@ public void testHandleBatchExceptionWithNullResponseBody1() { ApiClientApiException apiException = new ApiClientApiException("Bad Request", 400, null); Exception exception = new Exception("Test exception", apiException); - List errors = Utils.handleBatchException(exception, batch, 0); + List errors = Utils.handleBatchException(exception, batch, 0, 1); Assert.assertEquals("Should return empty list for null response body", 2, errors.size()); } @@ -462,7 +437,6 @@ public void testFormatResponseWithTokens() { Assert.assertEquals("Should have one success record", 1, result.getSuccess().size()); Assert.assertEquals("Skyflow ID should match", "id1", result.getSuccess().get(0).getSkyflowId()); } - @Test public void testFormatResponseWithTokenListMapping() { // Prepare test data @@ -505,7 +479,6 @@ public void testFormatResponseWithTokenListMapping() { Assert.assertEquals("Token value should match", "token123", tokensList.get(0).getToken()); Assert.assertEquals("Token group name should match", "group1", tokensList.get(0).getTokenGroupName()); } - @Test public void testHandleBatchExceptionWithRecordsInResponseBody() { // Prepare test data @@ -535,7 +508,7 @@ public void testHandleBatchExceptionWithRecordsInResponseBody() { Exception exception = new Exception("Test exception", apiException); // Test the method - List errors = Utils.handleBatchException(exception, batch, 0); + List errors = Utils.handleBatchException(exception, batch, 0, 1); // Assertions Assert.assertNotNull("Errors list should not be null", errors); @@ -574,10 +547,10 @@ public void testValidateDetokenizeRequestValidInput() throws SkyflowException { @Test public void testValidateDetokenizeRequestNullRequest() { - try { + try{ Validations.validateDetokenizeRequest(null); Assert.fail(EXCEPTIONNOTTHROWN); - } catch (SkyflowException e) { + } catch (SkyflowException e){ assertEquals(e.getMessage(), ErrorMessage.DetokenizeRequestNull.getMessage()); } @@ -593,7 +566,7 @@ public void testValidateDetokenizeRequestEmptyTokens() { Validations.validateDetokenizeRequest(request); - } catch (SkyflowException e) { + } catch (SkyflowException e){ assertEquals(e.getMessage(), ErrorMessage.EmptyDetokenizeData.getMessage()); } } @@ -618,9 +591,9 @@ public void testValidateDetokenizeRequestNullGroupRedactions() { .tokens(tokens) .tokenGroupRedactions(null) .build(); - try { + try{ Validations.validateDetokenizeRequest(request); - } catch (SkyflowException e) { + } catch (SkyflowException e){ Assert.fail(INVALID_EXCEPTION_THROWN); } } @@ -637,9 +610,9 @@ public void testValidateDetokenizeRequestNullTokenGroupRedaction() { .tokens(tokens) .tokenGroupRedactions(groupRedactions) .build(); - try { + try{ Validations.validateDetokenizeRequest(request); - } catch (SkyflowException e) { + } catch (SkyflowException e){ Assert.assertEquals(ErrorMessage.NullTokenGroupRedactions.getMessage(), e.getMessage());// } } @@ -660,9 +633,9 @@ public void testValidateDetokenizeRequestEmptyTokenGroupName() { .tokenGroupRedactions(groupRedactions) .build(); - try { + try{ Validations.validateDetokenizeRequest(request); - } catch (SkyflowException e) { + } catch (SkyflowException e){ assertEquals(ErrorMessage.NullTokenGroupNameInTokenGroup.getMessage(), e.getMessage()); } } @@ -685,7 +658,7 @@ public void testValidateDetokenizeRequestEmptyRedaction() { try { Validations.validateDetokenizeRequest(request); - } catch (SkyflowException e) { + } catch (SkyflowException e){ assertEquals(ErrorMessage.NullRedactionInTokenGroup.getMessage(), e.getMessage()); } } @@ -759,6 +732,7 @@ public void testValidateInsertRequestEmptyValues() { } } + @Test public void testFormatDetokenizeResponseValidResponse() { // Arrange @@ -853,8 +827,8 @@ public void testCreateDetokenizeBatchesWithEmptyTokenGroupRedactions() { List batches = Utils.createDetokenizeBatches(request, 1); Assert.assertEquals(2, batches.size()); - List redactions = batches.get(0).getTokenGroupRedactions().get(); - Assert.assertTrue(redactions.isEmpty()); +// List redactions = batches.get(0).getTokenGroupRedactions().get(); +// Assert.assertTrue(redactions.isEmpty()); } @Test @@ -887,6 +861,30 @@ public void testCreateDetokenizeBatchesWithBatchSizeGreaterThanTokens() { Assert.assertEquals(Arrays.asList("token1"), batches.get(0).getTokens().get()); } + public static List createDetokenizeBatches(com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest request, int batchSize) { + List detokenizeRequests = new ArrayList<>(); + List tokens = request.getTokens().get(); + + for (int i = 0; i < tokens.size(); i += batchSize) { + // Create a sublist for the current batch + List batchTokens = tokens.subList(i, Math.min(i + batchSize, tokens.size())); + List tokenGroupRedactions = null; + if (request.getTokenGroupRedactions().isPresent() && !request.getTokenGroupRedactions().get().isEmpty() && i < request.getTokenGroupRedactions().get().size()) { + tokenGroupRedactions = request.getTokenGroupRedactions().get().subList(i, Math.min(i + batchSize, request.getTokenGroupRedactions().get().size())); } + // Build a new DetokenizeRequest for the current batch + com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest batchRequest = com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest.builder() + .vaultId(request.getVaultId()) + .tokens(new ArrayList<>(batchTokens)) + .tokenGroupRedactions(tokenGroupRedactions) + .build(); + + detokenizeRequests.add(batchRequest); + } + + return detokenizeRequests; + } + + private DetokenizeResponseObject createResponseObject(String token, String value, String groupName, String error, Integer httpCode) { DetokenizeResponseObject responseObject = new DetokenizeResponseObject( 0, @@ -895,8 +893,7 @@ private DetokenizeResponseObject createResponseObject(String token, String value String.valueOf(Optional.ofNullable(groupName)), String.valueOf(Optional.ofNullable(error)), null - ); - return responseObject; + );return responseObject; } @Test @@ -935,14 +932,13 @@ public void testCreateErrorRecordWithUnknownErrorMessage() { Assert.assertEquals("Unknown error", error.getError()); Assert.assertEquals(403, error.getCode()); } - @Test public void testHandleBatchExceptionWithNullResponseBody() { List batch = Arrays.asList(InsertRecordData.builder().build()); ApiClientApiException apiException = new ApiClientApiException("Error", 500, null); Exception exception = new Exception("Test", apiException); - List errors = Utils.handleBatchException(exception, batch, 0); + List errors = Utils.handleBatchException(exception, batch, 0, 1); Assert.assertEquals(1, errors.size()); Assert.assertEquals("Test", errors.get(0).getError()); Assert.assertEquals(500, errors.get(0).getCode()); @@ -956,7 +952,7 @@ public void testHandleBatchExceptionWithNonListRecords() { ApiClientApiException apiException = new ApiClientApiException("Error", 500, responseBody); Exception exception = new Exception("Test", apiException); - List errors = Utils.handleBatchException(exception, batch, 0); + List errors = Utils.handleBatchException(exception, batch, 0, 1); Assert.assertEquals(1, errors.size()); } @@ -968,7 +964,7 @@ public void testHandleBatchExceptionWithErrorNotMap() { ApiClientApiException apiException = new ApiClientApiException("Error", 500, responseBody); Exception exception = new Exception("Test", apiException); - List errors = Utils.handleBatchException(exception, batch, 0); + List errors = Utils.handleBatchException(exception, batch, 0, 1); Assert.assertEquals(1, errors.size()); } @@ -985,7 +981,7 @@ public void testHandleBatchExceptionWithApiClientApiExceptionCause() { com.skyflow.generated.rest.core.ApiClientApiException apiException = new com.skyflow.generated.rest.core.ApiClientApiException("Error", 400, responseBody); Exception exception = new Exception("Outer exception", apiException); - List errors = Utils.handleBatchException(exception, batch, 0); + List errors = Utils.handleBatchException(exception, batch, 0, 1); Assert.assertEquals(1, errors.size()); Assert.assertEquals("Test error", errors.get(0).getError()); Assert.assertEquals(400, errors.get(0).getCode()); diff --git a/v3/src/test/java/com/skyflow/vault/controller/VaultControllerTests.java b/v3/src/test/java/com/skyflow/vault/controller/VaultControllerTests.java index 652fb7fd..17032835 100644 --- a/v3/src/test/java/com/skyflow/vault/controller/VaultControllerTests.java +++ b/v3/src/test/java/com/skyflow/vault/controller/VaultControllerTests.java @@ -265,7 +265,7 @@ public void testConcurrencyExceedsMax() throws Exception { // Ignore, Testing concurrency/batch config } - assertEquals(Constants.INSERT_CONCURRENCY_LIMIT.intValue(), getPrivateInt(controller, "insertConcurrencyLimit")); + assertEquals(1, getPrivateInt(controller, "insertConcurrencyLimit")); } @Test