From bf03bb13e08a118f27b0348303d9b98fae8b82b9 Mon Sep 17 00:00:00 2001 From: skyflow-shravan Date: Tue, 9 Sep 2025 16:41:37 +0530 Subject: [PATCH 1/3] SK-2286 take vault url from env and override --- .../java/com/skyflow/errors/ErrorMessage.java | 2 ++ .../main/java/com/skyflow/utils/BaseUtils.java | 17 +++++++++++++++++ .../utils/validations/BaseValidations.java | 14 ++++++++------ v3/src/main/java/com/skyflow/VaultClient.java | 5 ++++- .../skyflow/utils/validations/Validations.java | 2 ++ 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/common/src/main/java/com/skyflow/errors/ErrorMessage.java b/common/src/main/java/com/skyflow/errors/ErrorMessage.java index f4b2ce7b..f971544a 100644 --- a/common/src/main/java/com/skyflow/errors/ErrorMessage.java +++ b/common/src/main/java/com/skyflow/errors/ErrorMessage.java @@ -16,6 +16,7 @@ public enum ErrorMessage { EmptyVaultId("%s0 Initialization failed. Invalid vault ID. Vault ID must not be empty."), InvalidClusterId("%s0 Initialization failed. Invalid cluster ID. Specify cluster ID."), EmptyClusterId("%s0 Initialization failed. Invalid cluster ID. Specify a valid cluster ID."), + EmptyVaultUrl("%s0 Initialization failed. Vault URL is empty. Specify a valid vault URL."), // Connection config InvalidConnectionId("%s0 Initialization failed. Invalid connection ID. Specify a valid connection ID."), @@ -73,6 +74,7 @@ public enum ErrorMessage { InsufficientTokensPassedForTokenModeEnableStrict("%s0 Validation error. 'tokenMode' is set to 'ENABLE_STRICT', but some fields are missing tokens. Specify tokens for all fields."), BatchInsertPartialSuccess("%s0 Insert operation completed with partial success."), BatchInsertFailure("%s0 Insert operation failed."), + RecordSizeExceedError("%s0 Maximum number of records exceeded. The limit is 10000."), // Detokenize InvalidDetokenizeData("%s0 Validation error. Invalid detokenize data. Specify valid detokenize data."), diff --git a/common/src/main/java/com/skyflow/utils/BaseUtils.java b/common/src/main/java/com/skyflow/utils/BaseUtils.java index b7cf6bb0..9ff02df9 100644 --- a/common/src/main/java/com/skyflow/utils/BaseUtils.java +++ b/common/src/main/java/com/skyflow/utils/BaseUtils.java @@ -11,6 +11,8 @@ import com.skyflow.serviceaccount.util.BearerToken; import com.skyflow.serviceaccount.util.Token; import com.skyflow.utils.logger.LogUtil; +import io.github.cdimascio.dotenv.Dotenv; +import io.github.cdimascio.dotenv.DotenvException; import org.apache.commons.codec.binary.Base64; import java.io.File; @@ -43,6 +45,21 @@ public static String getVaultURL(String clusterId, Env env, String vaultDomain) return sb.toString(); } + public static String getEnvVaultURL() { + try { + Dotenv dotenv = Dotenv.load(); + String vaultURL = dotenv.get("VAULT_URL"); + if (vaultURL != null && vaultURL.trim().isEmpty()) { + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyVaultUrl.getMessage()); + } + return vaultURL; + } catch (DotenvException e) { + return null; + } catch (SkyflowException e) { + throw new RuntimeException(e); + } + } + public static String generateBearerToken(Credentials credentials) throws SkyflowException { String bearerToken; if (credentials.getPath() != null) { diff --git a/common/src/main/java/com/skyflow/utils/validations/BaseValidations.java b/common/src/main/java/com/skyflow/utils/validations/BaseValidations.java index 8460ec1d..deb5decd 100644 --- a/common/src/main/java/com/skyflow/utils/validations/BaseValidations.java +++ b/common/src/main/java/com/skyflow/utils/validations/BaseValidations.java @@ -28,12 +28,14 @@ public static void validateVaultConfig(VaultConfig vaultConfig) throws SkyflowEx } else if (vaultId.trim().isEmpty()) { LogUtil.printErrorLog(ErrorLogs.EMPTY_VAULT_ID.getLog()); throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyVaultId.getMessage()); - } else if (clusterId == null) { - LogUtil.printErrorLog(ErrorLogs.CLUSTER_ID_IS_REQUIRED.getLog()); - throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.InvalidClusterId.getMessage()); - } else if (clusterId.trim().isEmpty()) { - LogUtil.printErrorLog(ErrorLogs.EMPTY_CLUSTER_ID.getLog()); - throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyClusterId.getMessage()); + } else if (BaseUtils.getEnvVaultURL() == null) { + if (clusterId == null) { + LogUtil.printErrorLog(ErrorLogs.CLUSTER_ID_IS_REQUIRED.getLog()); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.InvalidClusterId.getMessage()); + } else if (clusterId.trim().isEmpty()) { + LogUtil.printErrorLog(ErrorLogs.EMPTY_CLUSTER_ID.getLog()); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyClusterId.getMessage()); + } } else if (credentials != null) { validateCredentials(credentials); } diff --git a/v3/src/main/java/com/skyflow/VaultClient.java b/v3/src/main/java/com/skyflow/VaultClient.java index b43ccd60..4b7c4613 100644 --- a/v3/src/main/java/com/skyflow/VaultClient.java +++ b/v3/src/main/java/com/skyflow/VaultClient.java @@ -80,7 +80,10 @@ protected void setBearerToken() throws SkyflowException { } private void updateVaultURL() { - String vaultURL = Utils.getVaultURL(this.vaultConfig.getClusterId(), this.vaultConfig.getEnv()); + String vaultURL = Utils.getEnvVaultURL(); + if (vaultURL == null || vaultURL.isEmpty()) { + vaultURL = Utils.getVaultURL(this.vaultConfig.getClusterId(), this.vaultConfig.getEnv()); + } this.apiClientBuilder.url(vaultURL); } diff --git a/v3/src/main/java/com/skyflow/utils/validations/Validations.java b/v3/src/main/java/com/skyflow/utils/validations/Validations.java index b7ea4bbd..7afb371e 100644 --- a/v3/src/main/java/com/skyflow/utils/validations/Validations.java +++ b/v3/src/main/java/com/skyflow/utils/validations/Validations.java @@ -46,6 +46,8 @@ public static void validateInsertRequest(InsertRequest insertRequest) throws Sky ErrorLogs.EMPTY_VALUES.getLog(), InterfaceName.INSERT.getName() )); throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyValues.getMessage()); + } else if(values.size() > 10000) { + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.RecordSizeExceedError.getMessage()); } else if (upsert != null && upsert.isEmpty()){ LogUtil.printErrorLog(Utils.parameterizedString( ErrorLogs.EMPTY_UPSERT.getLog(), InterfaceName.INSERT.getName() From a244385964ee431674926138052ddbed82a5ae05 Mon Sep 17 00:00:00 2001 From: skyflow-shravan Date: Tue, 9 Sep 2025 17:06:48 +0530 Subject: [PATCH 2/3] SK-2286 add validation for vault url format --- common/src/main/java/com/skyflow/errors/ErrorMessage.java | 1 + common/src/main/java/com/skyflow/utils/BaseUtils.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/common/src/main/java/com/skyflow/errors/ErrorMessage.java b/common/src/main/java/com/skyflow/errors/ErrorMessage.java index f971544a..87d07e08 100644 --- a/common/src/main/java/com/skyflow/errors/ErrorMessage.java +++ b/common/src/main/java/com/skyflow/errors/ErrorMessage.java @@ -17,6 +17,7 @@ public enum ErrorMessage { InvalidClusterId("%s0 Initialization failed. Invalid cluster ID. Specify cluster ID."), EmptyClusterId("%s0 Initialization failed. Invalid cluster ID. Specify a valid cluster ID."), EmptyVaultUrl("%s0 Initialization failed. Vault URL is empty. Specify a valid vault URL."), + InvalidVaultUrlFormat("%s0 Initialization failed. Vault URL must start with 'https://'."), // Connection config InvalidConnectionId("%s0 Initialization failed. Invalid connection ID. Specify a valid connection ID."), diff --git a/common/src/main/java/com/skyflow/utils/BaseUtils.java b/common/src/main/java/com/skyflow/utils/BaseUtils.java index 9ff02df9..453d98c7 100644 --- a/common/src/main/java/com/skyflow/utils/BaseUtils.java +++ b/common/src/main/java/com/skyflow/utils/BaseUtils.java @@ -51,6 +51,8 @@ public static String getEnvVaultURL() { String vaultURL = dotenv.get("VAULT_URL"); if (vaultURL != null && vaultURL.trim().isEmpty()) { throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyVaultUrl.getMessage()); + } else if (vaultURL != null && !vaultURL.startsWith(BaseConstants.SECURE_PROTOCOL)) { + throw new SkyflowException( ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.InvalidVaultUrlFormat.getMessage()); } return vaultURL; } catch (DotenvException e) { From 3e029f002624a2f3002064cab143437daca046ad Mon Sep 17 00:00:00 2001 From: skyflow-shravan Date: Tue, 9 Sep 2025 17:14:07 +0530 Subject: [PATCH 3/3] SK-2286 add validation for vault url format --- common/src/main/java/com/skyflow/errors/ErrorMessage.java | 1 + .../main/java/com/skyflow/utils/validations/Validations.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/common/src/main/java/com/skyflow/errors/ErrorMessage.java b/common/src/main/java/com/skyflow/errors/ErrorMessage.java index 87d07e08..4998936b 100644 --- a/common/src/main/java/com/skyflow/errors/ErrorMessage.java +++ b/common/src/main/java/com/skyflow/errors/ErrorMessage.java @@ -81,6 +81,7 @@ public enum ErrorMessage { InvalidDetokenizeData("%s0 Validation error. Invalid detokenize data. Specify valid detokenize data."), EmptyDetokenizeData("%s0 Validation error. Invalid data tokens. Specify at least one data token."), EmptyTokenInDetokenizeData("%s0 Validation error. Invalid data tokens. Specify a valid data token."), + TokensSizeExceedError("%s0 Maximum number of tokens exceeded. The limit is 10000."), // Get IdsKeyError("%s0 Validation error. 'ids' key is missing from the payload. Specify an 'ids' key."), diff --git a/v3/src/main/java/com/skyflow/utils/validations/Validations.java b/v3/src/main/java/com/skyflow/utils/validations/Validations.java index 7afb371e..e88df47e 100644 --- a/v3/src/main/java/com/skyflow/utils/validations/Validations.java +++ b/v3/src/main/java/com/skyflow/utils/validations/Validations.java @@ -84,6 +84,9 @@ public static void validateDetokenizeRequest(DetokenizeRequest request) throws S throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.DetokenizeRequestNull.getMessage()); } List tokens = request.getTokens(); + if(tokens.size() > 10000) { + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.TokensSizeExceedError.getMessage()); + } if (tokens == null || tokens.isEmpty()) { LogUtil.printErrorLog(Utils.parameterizedString( ErrorLogs.EMPTY_DETOKENIZE_DATA.getLog(), InterfaceName.DETOKENIZE.getName()