diff --git a/common/src/main/java/com/skyflow/errors/ErrorMessage.java b/common/src/main/java/com/skyflow/errors/ErrorMessage.java index f4b2ce7b..4998936b 100644 --- a/common/src/main/java/com/skyflow/errors/ErrorMessage.java +++ b/common/src/main/java/com/skyflow/errors/ErrorMessage.java @@ -16,6 +16,8 @@ public enum ErrorMessage { EmptyVaultId("%s0 Initialization failed. Invalid vault ID. Vault ID must not be empty."), InvalidClusterId("%s0 Initialization failed. Invalid cluster ID. Specify cluster ID."), EmptyClusterId("%s0 Initialization failed. Invalid cluster ID. Specify a valid cluster ID."), + EmptyVaultUrl("%s0 Initialization failed. Vault URL is empty. Specify a valid vault URL."), + InvalidVaultUrlFormat("%s0 Initialization failed. Vault URL must start with 'https://'."), // Connection config InvalidConnectionId("%s0 Initialization failed. Invalid connection ID. Specify a valid connection ID."), @@ -73,11 +75,13 @@ public enum ErrorMessage { InsufficientTokensPassedForTokenModeEnableStrict("%s0 Validation error. 'tokenMode' is set to 'ENABLE_STRICT', but some fields are missing tokens. Specify tokens for all fields."), BatchInsertPartialSuccess("%s0 Insert operation completed with partial success."), BatchInsertFailure("%s0 Insert operation failed."), + RecordSizeExceedError("%s0 Maximum number of records exceeded. The limit is 10000."), // Detokenize InvalidDetokenizeData("%s0 Validation error. Invalid detokenize data. Specify valid detokenize data."), EmptyDetokenizeData("%s0 Validation error. Invalid data tokens. Specify at least one data token."), EmptyTokenInDetokenizeData("%s0 Validation error. Invalid data tokens. Specify a valid data token."), + TokensSizeExceedError("%s0 Maximum number of tokens exceeded. The limit is 10000."), // Get IdsKeyError("%s0 Validation error. 'ids' key is missing from the payload. Specify an 'ids' key."), diff --git a/common/src/main/java/com/skyflow/logs/ErrorLogs.java b/common/src/main/java/com/skyflow/logs/ErrorLogs.java index 8f1156fd..5cde7d96 100644 --- a/common/src/main/java/com/skyflow/logs/ErrorLogs.java +++ b/common/src/main/java/com/skyflow/logs/ErrorLogs.java @@ -25,6 +25,8 @@ public enum ErrorLogs { EMPTY_ROLES("Invalid credentials. Roles can not be empty."), EMPTY_OR_NULL_ROLE_IN_ROLES("Invalid credentials. Role can not be null or empty in roles at index %s1."), EMPTY_OR_NULL_CONTEXT("Invalid credentials. Context can not be empty."), + EMPTY_VAULT_URL("Invalid vault config. Vault URL can not be empty."), + INVALID_VAULT_URL_FORMAT("Invalid vault config. Vault URL format is incorrect"), // Bearer token generation INVALID_BEARER_TOKEN("Bearer token is invalid or expired."), @@ -49,6 +51,8 @@ public enum ErrorLogs { EMPTY_TABLE_NAME("Invalid %s1 request. Table name can not be empty."), VALUES_IS_REQUIRED("Invalid %s1 request. Values are required."), EMPTY_VALUES("Invalid %s1 request. Values can not be empty."), + RECORD_SIZE_EXCEED("Maximum number of records exceeded. The limit is 10000."), + TOKENS_SIZE_EXCEED("Maximum number of tokens exceeded. The limit is 10000."), EMPTY_OR_NULL_VALUE_IN_VALUES("Invalid %s1 request. Value can not be null or empty in values for key \"%s2\"."), EMPTY_OR_NULL_KEY_IN_VALUES("Invalid %s1 request. Key can not be null or empty in values"), EMPTY_UPSERT("Invalid %s1 request. Upsert can not be empty."), diff --git a/common/src/main/java/com/skyflow/utils/BaseUtils.java b/common/src/main/java/com/skyflow/utils/BaseUtils.java index b7cf6bb0..a13c9ab5 100644 --- a/common/src/main/java/com/skyflow/utils/BaseUtils.java +++ b/common/src/main/java/com/skyflow/utils/BaseUtils.java @@ -11,6 +11,8 @@ import com.skyflow.serviceaccount.util.BearerToken; import com.skyflow.serviceaccount.util.Token; import com.skyflow.utils.logger.LogUtil; +import io.github.cdimascio.dotenv.Dotenv; +import io.github.cdimascio.dotenv.DotenvException; import org.apache.commons.codec.binary.Base64; import java.io.File; @@ -43,6 +45,7 @@ public static String getVaultURL(String clusterId, Env env, String vaultDomain) return sb.toString(); } + public static String generateBearerToken(Credentials credentials) throws SkyflowException { String bearerToken; if (credentials.getPath() != null) { diff --git a/common/src/main/java/com/skyflow/utils/validations/BaseValidations.java b/common/src/main/java/com/skyflow/utils/validations/BaseValidations.java index 8460ec1d..410b99aa 100644 --- a/common/src/main/java/com/skyflow/utils/validations/BaseValidations.java +++ b/common/src/main/java/com/skyflow/utils/validations/BaseValidations.java @@ -34,7 +34,8 @@ public static void validateVaultConfig(VaultConfig vaultConfig) throws SkyflowEx } else if (clusterId.trim().isEmpty()) { LogUtil.printErrorLog(ErrorLogs.EMPTY_CLUSTER_ID.getLog()); throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyClusterId.getMessage()); - } else if (credentials != null) { + } + else if (credentials != null) { validateCredentials(credentials); } } diff --git a/v3/pom.xml b/v3/pom.xml index 4655a067..068a7bcb 100644 --- a/v3/pom.xml +++ b/v3/pom.xml @@ -11,7 +11,7 @@ skyflow-java - 3.0.0-beta.3 + 2.0.0-beta.4-dev.f012079 jar ${project.groupId}:${project.artifactId} Skyflow V3 SDK for the Java programming language diff --git a/v3/src/main/java/com/skyflow/Skyflow.java b/v3/src/main/java/com/skyflow/Skyflow.java index 7973c747..ed357bdd 100644 --- a/v3/src/main/java/com/skyflow/Skyflow.java +++ b/v3/src/main/java/com/skyflow/Skyflow.java @@ -56,7 +56,7 @@ public SkyflowClientBuilder() { public SkyflowClientBuilder addVaultConfig(VaultConfig vaultConfig) throws SkyflowException { LogUtil.printInfoLog(InfoLogs.VALIDATING_VAULT_CONFIG.getLog()); - Validations.validateVaultConfig(vaultConfig); + Validations.validateVaultConfiguration(vaultConfig); VaultConfig vaultConfigCopy; try { vaultConfigCopy = (VaultConfig) vaultConfig.clone(); diff --git a/v3/src/main/java/com/skyflow/VaultClient.java b/v3/src/main/java/com/skyflow/VaultClient.java index b43ccd60..a2297b49 100644 --- a/v3/src/main/java/com/skyflow/VaultClient.java +++ b/v3/src/main/java/com/skyflow/VaultClient.java @@ -39,7 +39,7 @@ public class VaultClient { private String token; private String apiKey; - protected VaultClient(VaultConfig vaultConfig, Credentials credentials) { + protected VaultClient(VaultConfig vaultConfig, Credentials credentials) throws SkyflowException { super(); this.vaultConfig = vaultConfig; this.commonCredentials = credentials; @@ -79,8 +79,11 @@ protected void setBearerToken() throws SkyflowException { this.apiClient = this.apiClientBuilder.build(); } - private void updateVaultURL() { - String vaultURL = Utils.getVaultURL(this.vaultConfig.getClusterId(), this.vaultConfig.getEnv()); + private void updateVaultURL() throws SkyflowException { + String vaultURL = Utils.getEnvVaultURL(); + if (vaultURL == null || vaultURL.isEmpty()) { + vaultURL = Utils.getVaultURL(this.vaultConfig.getClusterId(), this.vaultConfig.getEnv()); + } this.apiClientBuilder.url(vaultURL); } diff --git a/v3/src/main/java/com/skyflow/utils/Utils.java b/v3/src/main/java/com/skyflow/utils/Utils.java index f2142bd6..468f3ff6 100644 --- a/v3/src/main/java/com/skyflow/utils/Utils.java +++ b/v3/src/main/java/com/skyflow/utils/Utils.java @@ -2,16 +2,23 @@ import com.google.gson.JsonObject; import com.skyflow.enums.Env; +import com.skyflow.errors.ErrorCode; +import com.skyflow.errors.ErrorMessage; +import com.skyflow.errors.SkyflowException; import com.skyflow.generated.rest.core.ApiClientApiException; import com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest; import com.skyflow.generated.rest.types.InsertRecordData; import com.skyflow.generated.rest.types.InsertResponse; import com.skyflow.generated.rest.types.RecordResponseObject; import com.skyflow.generated.rest.types.TokenGroupRedactions; +import com.skyflow.logs.ErrorLogs; +import com.skyflow.utils.logger.LogUtil; import com.skyflow.vault.data.DetokenizeResponse; import com.skyflow.vault.data.ErrorRecord; import com.skyflow.vault.data.Success; import com.skyflow.vault.data.Token; +import io.github.cdimascio.dotenv.Dotenv; +import io.github.cdimascio.dotenv.DotenvException; import java.util.ArrayList; import java.util.HashMap; @@ -239,4 +246,24 @@ public static com.skyflow.vault.data.InsertResponse formatResponse(InsertRespons return formattedResponse; } + public static String getEnvVaultURL() throws SkyflowException { + try { + String vaultURL = System.getenv("VAULT_URL"); + if (vaultURL == null) { + Dotenv dotenv = Dotenv.load(); + vaultURL = dotenv.get("VAULT_URL"); + } + if (vaultURL != null && vaultURL.trim().isEmpty()) { + LogUtil.printErrorLog(ErrorLogs.EMPTY_VAULT_URL.getLog()); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyVaultUrl.getMessage()); + } else if (vaultURL != null && !vaultURL.startsWith(BaseConstants.SECURE_PROTOCOL)) { + LogUtil.printErrorLog(ErrorLogs.INVALID_VAULT_URL_FORMAT.getLog()); + throw new SkyflowException( ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.InvalidVaultUrlFormat.getMessage()); + } + return vaultURL; + } catch (DotenvException e) { + return null; + } + } + } diff --git a/v3/src/main/java/com/skyflow/utils/validations/Validations.java b/v3/src/main/java/com/skyflow/utils/validations/Validations.java index b7ea4bbd..4e278db3 100644 --- a/v3/src/main/java/com/skyflow/utils/validations/Validations.java +++ b/v3/src/main/java/com/skyflow/utils/validations/Validations.java @@ -1,10 +1,13 @@ package com.skyflow.utils.validations; +import com.skyflow.config.Credentials; +import com.skyflow.config.VaultConfig; import com.skyflow.enums.InterfaceName; import com.skyflow.errors.ErrorCode; import com.skyflow.errors.ErrorMessage; import com.skyflow.errors.SkyflowException; import com.skyflow.logs.ErrorLogs; +import com.skyflow.utils.BaseUtils; import com.skyflow.utils.Utils; import com.skyflow.utils.logger.LogUtil; import com.skyflow.vault.data.DetokenizeRequest; @@ -46,6 +49,9 @@ public static void validateInsertRequest(InsertRequest insertRequest) throws Sky ErrorLogs.EMPTY_VALUES.getLog(), InterfaceName.INSERT.getName() )); throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyValues.getMessage()); + } else if(values.size() > 10000) { + LogUtil.printErrorLog(ErrorLogs.RECORD_SIZE_EXCEED.getLog()); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.RecordSizeExceedError.getMessage()); } else if (upsert != null && upsert.isEmpty()){ LogUtil.printErrorLog(Utils.parameterizedString( ErrorLogs.EMPTY_UPSERT.getLog(), InterfaceName.INSERT.getName() @@ -82,6 +88,10 @@ public static void validateDetokenizeRequest(DetokenizeRequest request) throws S throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.DetokenizeRequestNull.getMessage()); } List tokens = request.getTokens(); + if(tokens.size() > 10000) { + LogUtil.printErrorLog(ErrorLogs.TOKENS_SIZE_EXCEED.getLog()); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.TokensSizeExceedError.getMessage()); + } if (tokens == null || tokens.isEmpty()) { LogUtil.printErrorLog(Utils.parameterizedString( ErrorLogs.EMPTY_DETOKENIZE_DATA.getLog(), InterfaceName.DETOKENIZE.getName() @@ -118,4 +128,26 @@ public static void validateDetokenizeRequest(DetokenizeRequest request) throws S } + public static void validateVaultConfiguration(VaultConfig vaultConfig) throws SkyflowException { + String vaultId = vaultConfig.getVaultId(); + String clusterId = vaultConfig.getClusterId(); + Credentials credentials = vaultConfig.getCredentials(); + if (vaultId == null) { + LogUtil.printErrorLog(ErrorLogs.VAULT_ID_IS_REQUIRED.getLog()); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.InvalidVaultId.getMessage()); + } else if (vaultId.trim().isEmpty()) { + LogUtil.printErrorLog(ErrorLogs.EMPTY_VAULT_ID.getLog()); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyVaultId.getMessage()); + } else if (Utils.getEnvVaultURL() == null) { + if (clusterId == null) { + LogUtil.printErrorLog(ErrorLogs.CLUSTER_ID_IS_REQUIRED.getLog()); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.InvalidClusterId.getMessage()); + } else if (clusterId.trim().isEmpty()) { + LogUtil.printErrorLog(ErrorLogs.EMPTY_CLUSTER_ID.getLog()); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyClusterId.getMessage()); + } + } else if (credentials != null) { + validateCredentials(credentials); + } + } } diff --git a/v3/src/main/java/com/skyflow/vault/controller/VaultController.java b/v3/src/main/java/com/skyflow/vault/controller/VaultController.java index 0ac80f4a..952cd3ee 100644 --- a/v3/src/main/java/com/skyflow/vault/controller/VaultController.java +++ b/v3/src/main/java/com/skyflow/vault/controller/VaultController.java @@ -34,7 +34,7 @@ public final class VaultController extends VaultClient { private int detokenizeBatchSize; private int detokenizeConcurrencyLimit; - public VaultController(VaultConfig vaultConfig, Credentials credentials) { + public VaultController(VaultConfig vaultConfig, Credentials credentials) throws SkyflowException { super(vaultConfig, credentials); this.insertBatchSize = Constants.INSERT_BATCH_SIZE; this.insertConcurrencyLimit = Constants.INSERT_CONCURRENCY_LIMIT;