From 03f3990647d05812514e4250932b69863c9232f5 Mon Sep 17 00:00:00 2001 From: luxl Date: Mon, 8 Jun 2026 17:12:35 +0800 Subject: [PATCH 1/3] [Feature-17937] Support sensitive workflow variables fix #17937 Co-authored-by: Cursor --- .../SensitiveWorkflowVariableAPITest.java | 301 ++++++++++++++++++ .../api/test/pages/workflow/ExecutorPage.java | 12 + .../pages/workflow/WorkflowInstancePage.java | 25 ++ .../impl/TaskDefinitionServiceImpl.java | 19 +- .../impl/WorkflowDefinitionServiceImpl.java | 57 +++- .../impl/WorkflowInstanceServiceImpl.java | 42 ++- .../api/utils/SensitivePropertyUtils.java | 179 +++++++++++ .../WorkflowDefinitionControllerTest.java | 12 +- .../WorkflowInstanceControllerTest.java | 10 +- .../WorkflowDefinitionServiceTest.java | 178 +++++++++++ .../service/WorkflowInstanceServiceTest.java | 77 +++++ .../handler/RunWorkflowCommandHandler.java | 7 +- .../runner/TaskExecutionContextFactory.java | 7 +- .../utils/MasterSensitivePropertyUtils.java | 94 ++++++ .../expand/CuringParamsServiceImpl.java | 7 +- .../expand/CuringParamsServiceImplTest.java | 40 +++ .../task/api/log/SensitiveDataConverter.java | 37 ++- .../plugin/task/api/model/Property.java | 10 + .../api/utils/PropertySensitiveUtils.java | 110 +++++++ .../api/log/SensitiveDataConverterTest.java | 36 +++ .../api/utils/PropertySensitiveUtilsTest.java | 114 +++++++ .../form/fields/custom-parameters.ts | 2 +- .../src/locales/en_US/project.ts | 2 + .../src/locales/zh_CN/project.ts | 2 + .../node/fields/use-custom-params.ts | 13 +- .../projects/task/components/node/types.ts | 1 + .../components/dag/dag-save-modal.tsx | 18 +- .../projects/workflow/components/dag/types.ts | 1 + .../workflow/definition/create/index.tsx | 3 +- .../workflow/definition/detail/index.tsx | 3 +- .../instance/components/variables-view.tsx | 30 +- .../workflow/instance/detail/index.tsx | 3 +- .../worker/executor/PhysicalTaskExecutor.java | 36 +++ .../rpc/PhysicalTaskExecutorOperatorImpl.java | 7 +- 34 files changed, 1438 insertions(+), 57 deletions(-) create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/SensitiveWorkflowVariableAPITest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/SensitivePropertyUtils.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/MasterSensitivePropertyUtils.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/PropertySensitiveUtils.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/PropertySensitiveUtilsTest.java diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/SensitiveWorkflowVariableAPITest.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/SensitiveWorkflowVariableAPITest.java new file mode 100644 index 000000000000..3a136da67e9c --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/SensitiveWorkflowVariableAPITest.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.test.cases; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.dolphinscheduler.api.test.core.DolphinScheduler; +import org.apache.dolphinscheduler.api.test.entity.HttpResponse; +import org.apache.dolphinscheduler.api.test.entity.LoginResponseData; +import org.apache.dolphinscheduler.api.test.pages.LoginPage; +import org.apache.dolphinscheduler.api.test.pages.project.ProjectPage; +import org.apache.dolphinscheduler.api.test.pages.workflow.ExecutorPage; +import org.apache.dolphinscheduler.api.test.pages.workflow.WorkflowDefinitionPage; +import org.apache.dolphinscheduler.api.test.pages.workflow.WorkflowInstancePage; +import org.apache.dolphinscheduler.api.test.utils.JSONUtils; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.dao.entity.User; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.org.awaitility.Awaitility; + +@DolphinScheduler(composeFiles = "docker/basic/docker-compose.yaml") +@Slf4j +public class SensitiveWorkflowVariableAPITest { + + private static final String USERNAME = "admin"; + private static final String PASSWORD = "dolphinscheduler123"; + private static final String SECRET = "Ds_SeCrEt_2026_06_08_Case_A1"; + private static final String SENSITIVE_DATA_MASK = "******"; + + private static String sessionId; + private static User loginUser; + private static ProjectPage projectPage; + private static WorkflowDefinitionPage workflowDefinitionPage; + private static WorkflowInstancePage workflowInstancePage; + private static ExecutorPage executorPage; + + @BeforeAll + public static void setup() { + LoginPage loginPage = new LoginPage(); + HttpResponse loginHttpResponse = loginPage.login(USERNAME, PASSWORD); + sessionId = + JSONUtils.convertValue(loginHttpResponse.getBody().getData(), LoginResponseData.class).getSessionId(); + loginUser = new User(); + loginUser.setUserName(USERNAME); + loginUser.setId(1); + loginUser.setUserType(UserType.ADMIN_USER); + projectPage = new ProjectPage(sessionId); + workflowDefinitionPage = new WorkflowDefinitionPage(sessionId); + workflowInstancePage = new WorkflowInstancePage(sessionId); + executorPage = new ExecutorPage(sessionId); + } + + @Test + @SuppressWarnings("unchecked") + public void testSensitiveGlobalParamUsesRealValueAtRuntimeAndMasksLog() { + String projectName = "sensitive-project-" + System.currentTimeMillis(); + String workflowName = "sensitive-workflow-" + System.currentTimeMillis(); + + HttpResponse createProjectResponse = projectPage.createProject(loginUser, projectName); + assertTrue(createProjectResponse.getBody().getSuccess()); + long projectCode = ((Number) ((LinkedHashMap) createProjectResponse.getBody().getData()) + .get("code")).longValue(); + + long taskCode = System.currentTimeMillis(); + HttpResponse createWorkflowResponse = workflowDefinitionPage.createWorkflowDefinition( + loginUser, + projectCode, + sensitiveWorkflowJson(taskCode), + workflowName); + assertTrue(createWorkflowResponse.getBody().getSuccess()); + long workflowDefinitionCode = + ((Number) ((LinkedHashMap) createWorkflowResponse.getBody().getData()).get("code")) + .longValue(); + + HttpResponse queryWorkflowResponse = + workflowDefinitionPage.queryWorkflowDefinitionByCode(loginUser, projectCode, workflowDefinitionCode); + assertTrue(queryWorkflowResponse.getBody().getSuccess()); + String workflowResponseData = queryWorkflowResponse.getBody().getData().toString(); + assertTrue(workflowResponseData.contains(SENSITIVE_DATA_MASK)); + assertFalse(workflowResponseData.contains(SECRET)); + + HttpResponse releaseWorkflowResponse = workflowDefinitionPage.releaseWorkflowDefinition( + loginUser, + projectCode, + workflowDefinitionCode, + ReleaseState.ONLINE); + assertTrue(releaseWorkflowResponse.getBody().getSuccess()); + + HttpResponse startWorkflowResponse = executorPage.startWorkflowInstance( + loginUser, + projectCode, + workflowDefinitionCode, + scheduleTime(), + FailureStrategy.END, + WarningType.NONE, + sensitiveStartParams()); + assertTrue(startWorkflowResponse.getBody().getSuccess()); + List workflowInstanceIds = (List) startWorkflowResponse.getBody().getData(); + assertEquals(1, workflowInstanceIds.size()); + int workflowInstanceId = workflowInstanceIds.get(0); + + Awaitility.await() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted(() -> { + HttpResponse instanceResponse = + workflowInstancePage.queryWorkflowInstanceById(loginUser, projectCode, workflowInstanceId); + assertTrue(instanceResponse.getBody().getSuccess()); + Map workflowInstance = (Map) instanceResponse.getBody().getData(); + assertEquals("SUCCESS", workflowInstance.get("state")); + }); + + HttpResponse taskListResponse = + workflowInstancePage.queryTaskInstanceList(loginUser, projectCode, workflowInstanceId); + assertTrue(taskListResponse.getBody().getSuccess()); + Map taskPage = (Map) taskListResponse.getBody().getData(); + List> taskInstances = (List>) taskPage.get("totalList"); + assertEquals(1, taskInstances.size()); + int taskInstanceId = (int) taskInstances.get(0).get("id"); + + HttpResponse logResponse = workflowInstancePage.queryTaskLog(loginUser, taskInstanceId, 0, 1000); + assertTrue(logResponse.getBody().getSuccess()); + String logContent = ((Map) logResponse.getBody().getData()).get("message").toString(); + assertTrue(logContent.contains(" -> " + SECRET.length()), logContent); + assertTrue(logContent.contains(SENSITIVE_DATA_MASK), logContent); + assertFalse(logContent.contains(SECRET), logContent); + + long plainTaskCode = System.currentTimeMillis(); + HttpResponse createPlainWorkflowResponse = workflowDefinitionPage.createWorkflowDefinition( + loginUser, + projectCode, + plainWorkflowJson(plainTaskCode), + "plain-workflow-" + System.currentTimeMillis()); + assertTrue(createPlainWorkflowResponse.getBody().getSuccess()); + long plainWorkflowDefinitionCode = + ((Number) ((LinkedHashMap) createPlainWorkflowResponse.getBody().getData()).get("code")) + .longValue(); + HttpResponse releasePlainWorkflowResponse = workflowDefinitionPage.releaseWorkflowDefinition( + loginUser, + projectCode, + plainWorkflowDefinitionCode, + ReleaseState.ONLINE); + assertTrue(releasePlainWorkflowResponse.getBody().getSuccess()); + + HttpResponse startPlainWorkflowResponse = executorPage.startWorkflowInstance( + loginUser, + projectCode, + plainWorkflowDefinitionCode, + scheduleTime(), + FailureStrategy.END, + WarningType.NONE); + assertTrue(startPlainWorkflowResponse.getBody().getSuccess()); + List plainWorkflowInstanceIds = (List) startPlainWorkflowResponse.getBody().getData(); + assertEquals(1, plainWorkflowInstanceIds.size()); + int plainWorkflowInstanceId = plainWorkflowInstanceIds.get(0); + + Awaitility.await() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted(() -> { + HttpResponse instanceResponse = + workflowInstancePage.queryWorkflowInstanceById(loginUser, projectCode, + plainWorkflowInstanceId); + assertTrue(instanceResponse.getBody().getSuccess()); + Map workflowInstance = (Map) instanceResponse.getBody().getData(); + assertEquals("SUCCESS", workflowInstance.get("state")); + }); + + HttpResponse plainTaskListResponse = + workflowInstancePage.queryTaskInstanceList(loginUser, projectCode, plainWorkflowInstanceId); + assertTrue(plainTaskListResponse.getBody().getSuccess()); + Map plainTaskPage = (Map) plainTaskListResponse.getBody().getData(); + List> plainTaskInstances = (List>) plainTaskPage.get("totalList"); + assertEquals(1, plainTaskInstances.size()); + int plainTaskInstanceId = (int) plainTaskInstances.get(0).get("id"); + + HttpResponse plainLogResponse = workflowInstancePage.queryTaskLog(loginUser, plainTaskInstanceId, 0, 1000); + assertTrue(plainLogResponse.getBody().getSuccess()); + String plainLogContent = ((Map) plainLogResponse.getBody().getData()).get("message").toString(); + assertTrue(plainLogContent.contains(SECRET), plainLogContent); + } + + private String sensitiveWorkflowJson(long taskCode) { + return "{" + + "\"taskDefinitionJson\":[{" + + "\"code\":" + taskCode + "," + + "\"delayTime\":\"0\"," + + "\"description\":\"\"," + + "\"environmentCode\":-1," + + "\"failRetryInterval\":\"1\"," + + "\"failRetryTimes\":\"0\"," + + "\"flag\":\"YES\"," + + "\"name\":\"sensitive_shell\"," + + "\"taskParams\":{\"localParams\":[],\"rawScript\":\"echo -n ${var} | wc -c\"," + + "\"resourceList\":[]}," + + "\"taskPriority\":\"MEDIUM\"," + + "\"taskType\":\"SHELL\"," + + "\"timeout\":0," + + "\"timeoutFlag\":\"CLOSE\"," + + "\"timeoutNotifyStrategy\":\"\"," + + "\"workerGroup\":\"default\"," + + "\"cpuQuota\":-1," + + "\"memoryMax\":-1," + + "\"taskExecuteType\":\"BATCH\"" + + "}]," + + "\"taskRelationJson\":[{" + + "\"name\":\"\"," + + "\"preTaskCode\":0," + + "\"preTaskVersion\":0," + + "\"postTaskCode\":" + taskCode + "," + + "\"postTaskVersion\":0," + + "\"conditionType\":\"NONE\"," + + "\"conditionParams\":{}" + + "}]," + + "\"executionType\":\"PARALLEL\"," + + "\"description\":\"\"," + + "\"globalParams\":[{\"prop\":\"var\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"" + SECRET + + "\",\"sensitive\":true}]," + + "\"timeout\":0" + + "}"; + } + + private String plainWorkflowJson(long taskCode) { + return "{" + + "\"taskDefinitionJson\":[{" + + "\"code\":" + taskCode + "," + + "\"delayTime\":\"0\"," + + "\"description\":\"\"," + + "\"environmentCode\":-1," + + "\"failRetryInterval\":\"1\"," + + "\"failRetryTimes\":\"0\"," + + "\"flag\":\"YES\"," + + "\"name\":\"plain_shell\"," + + "\"taskParams\":{\"localParams\":[],\"rawScript\":\"echo " + SECRET + "\"," + + "\"resourceList\":[]}," + + "\"taskPriority\":\"MEDIUM\"," + + "\"taskType\":\"SHELL\"," + + "\"timeout\":0," + + "\"timeoutFlag\":\"CLOSE\"," + + "\"timeoutNotifyStrategy\":\"\"," + + "\"workerGroup\":\"default\"," + + "\"cpuQuota\":-1," + + "\"memoryMax\":-1," + + "\"taskExecuteType\":\"BATCH\"" + + "}]," + + "\"taskRelationJson\":[{" + + "\"name\":\"\"," + + "\"preTaskCode\":0," + + "\"preTaskVersion\":0," + + "\"postTaskCode\":" + taskCode + "," + + "\"postTaskVersion\":0," + + "\"conditionType\":\"NONE\"," + + "\"conditionParams\":{}" + + "}]," + + "\"executionType\":\"PARALLEL\"," + + "\"description\":\"\"," + + "\"globalParams\":[]," + + "\"timeout\":0" + + "}"; + } + + private String sensitiveStartParams() { + return "[{\"prop\":\"var\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"" + SENSITIVE_DATA_MASK + + "\",\"sensitive\":true}]"; + } + + private String scheduleTime() { + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Date date = new Date(); + return String.format("%s,%s", formatter.format(date), formatter.format(date)); + } +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/workflow/ExecutorPage.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/workflow/ExecutorPage.java index d6aa56b757ad..7f48bf70b4b7 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/workflow/ExecutorPage.java +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/workflow/ExecutorPage.java @@ -44,12 +44,24 @@ public HttpResponse startWorkflowInstance(User loginUser, String scheduleTime, FailureStrategy failureStrategy, WarningType warningType) { + return startWorkflowInstance(loginUser, projectCode, workflowDefinitionCode, scheduleTime, failureStrategy, + warningType, null); + } + + public HttpResponse startWorkflowInstance(User loginUser, + long projectCode, + long workflowDefinitionCode, + String scheduleTime, + FailureStrategy failureStrategy, + WarningType warningType, + String startParams) { Map params = new HashMap<>(); params.put("loginUser", loginUser); params.put("workflowDefinitionCode", workflowDefinitionCode); params.put("scheduleTime", scheduleTime); params.put("failureStrategy", failureStrategy); params.put("warningType", warningType); + params.put("startParams", startParams); Map headers = new HashMap<>(); headers.put(Constants.SESSION_ID_KEY, sessionId); diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/workflow/WorkflowInstancePage.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/workflow/WorkflowInstancePage.java index 887c73ea372f..486816235de6 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/workflow/WorkflowInstancePage.java +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/workflow/WorkflowInstancePage.java @@ -69,6 +69,31 @@ public HttpResponse queryTaskListByWorkflowInstanceId(User loginUser, long proje return requestClient.get(url, headers, params); } + public HttpResponse queryTaskInstanceList(User loginUser, long projectCode, long workflowInstanceId) { + Map params = new HashMap<>(); + params.put("loginUser", loginUser); + params.put("workflowInstanceId", workflowInstanceId); + params.put("pageNo", 1); + params.put("pageSize", 10); + Map headers = new HashMap<>(); + headers.put(Constants.SESSION_ID_KEY, sessionId); + RequestClient requestClient = new RequestClient(); + String url = String.format("/projects/%s/task-instances", projectCode); + return requestClient.get(url, headers, params); + } + + public HttpResponse queryTaskLog(User loginUser, int taskInstanceId, int skipLineNum, int limit) { + Map params = new HashMap<>(); + params.put("loginUser", loginUser); + params.put("taskInstanceId", taskInstanceId); + params.put("skipLineNum", skipLineNum); + params.put("limit", limit); + Map headers = new HashMap<>(); + headers.put(Constants.SESSION_ID_KEY, sessionId); + RequestClient requestClient = new RequestClient(); + return requestClient.get("/log/detail", headers, params); + } + public HttpResponse queryWorkflowInstanceById(User loginUser, long projectCode, long workflowInstanceId) { Map params = new HashMap<>(); params.put("loginUser", loginUser); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java index 4828f9acd553..8d82fb002446 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java @@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.api.service.WorkflowTaskRelationService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.api.utils.SensitivePropertyUtils; import org.apache.dolphinscheduler.api.vo.TaskDefinitionVO; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; @@ -138,7 +139,7 @@ public TaskDefinition queryTaskDefinitionByName(User loginUser, long projectCode log.error("Task definition does not exist, taskName:{}.", taskName); throw new ServiceException(Status.TASK_DEFINE_NOT_EXIST, taskName); } - return taskDefinition; + return SensitivePropertyUtils.decryptAndMaskTaskDefinition(taskDefinition); } public void updateDag(User loginUser, long workflowDefinitionCode, @@ -191,7 +192,7 @@ public TaskDefinition getTaskDefinition(User loginUser, } Project project = projectDao.queryByCode(taskDefinition.getProjectCode()); projectService.checkProjectAndAuthThrowException(loginUser, project, TASK_DEFINITION); - return taskDefinition; + return SensitivePropertyUtils.decryptAndMaskTaskDefinition(taskDefinition); } /** @@ -223,6 +224,12 @@ private TaskDefinitionLog updateTask(User loginUser, long projectCode, long task } TaskDefinitionLog taskDefinitionToUpdate = JSONUtils.parseObject(taskDefinitionJsonObj, TaskDefinitionLog.class); + if (taskDefinitionToUpdate == null) { + log.warn("Parameter taskDefinitionJson is invalid."); + throw new ServiceException(Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj); + } + taskDefinitionToUpdate.setTaskParams(SensitivePropertyUtils.mergeAndEncryptLocalParamsInTaskParams( + taskDefinitionToUpdate.getTaskParams(), taskDefinition.getTaskParams())); if (TimeoutFlag.CLOSE == taskDefinition.getTimeoutFlag()) { taskDefinition.setTimeoutNotifyStrategy(null); } @@ -230,10 +237,6 @@ private TaskDefinitionLog updateTask(User loginUser, long projectCode, long task log.warn("Task definition does not need update because no change, taskDefinitionCode:{}.", taskCode); return null; } - if (taskDefinitionToUpdate == null) { - log.warn("Parameter taskDefinitionJson is invalid."); - throw new ServiceException(Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj); - } if (!checkTaskParameters(taskDefinitionToUpdate.getTaskType(), taskDefinitionToUpdate.getTaskParams())) { throw new ServiceException(Status.WORKFLOW_NODE_S_PARAMETER_INVALID, taskDefinitionToUpdate.getName()); } @@ -613,7 +616,9 @@ public TaskDefinitionVO queryTaskDefinitionDetail(User loginUser, long projectCo taskRelationList = taskRelationList.stream() .filter(v -> v.getPreTaskCode() != 0).collect(Collectors.toList()); } - TaskDefinitionVO taskDefinitionVo = TaskDefinitionVO.fromTaskDefinition(taskDefinition); + TaskDefinitionVO taskDefinitionVo = + TaskDefinitionVO + .fromTaskDefinition(SensitivePropertyUtils.decryptAndMaskTaskDefinition(taskDefinition)); taskDefinitionVo.setWorkflowTaskRelationList(taskRelationList); return taskDefinitionVo; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java index fecf8ec40641..4b1e21a3b8fd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowDefinitionServiceImpl.java @@ -50,6 +50,7 @@ import org.apache.dolphinscheduler.api.utils.CheckUtils; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.api.utils.SensitivePropertyUtils; import org.apache.dolphinscheduler.api.validator.GlobalParamsValidator; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.ReleaseState; @@ -257,6 +258,8 @@ public WorkflowDefinition createWorkflowDefinition(User loginUser, List taskDefinitionLogs = generateTaskDefinitionList(taskDefinitionJson); List taskRelationList = generateTaskRelationList(taskRelationJson, taskDefinitionLogs); + globalParams = SensitivePropertyUtils.encryptGlobalParams(globalParams); + encryptSensitiveLocalParams(taskDefinitionLogs); long workflowDefinitionCode = CodeGenerateUtils.genCode(); WorkflowDefinition workflowDefinition = @@ -445,7 +448,10 @@ public List queryWorkflowDefinitionList(User loginUser, long projectCod projectService.checkProjectAndAuthThrowException(loginUser, project, WORKFLOW_DEFINITION); List resourceList = workflowDefinitionDao.queryAllDefinitionList(projectCode); - return resourceList.stream().map(processService::genDagData).collect(Collectors.toList()); + return resourceList.stream() + .map(processService::genDagData) + .map(SensitivePropertyUtils::decryptAndMaskDagData) + .collect(Collectors.toList()); } /** @@ -521,6 +527,7 @@ public PageInfo queryWorkflowDefinitionListPaging(@NonNull U Schedule schedule = scheduleMap.get(pd.getCode()); pd.setScheduleReleaseState(schedule == null ? null : schedule.getReleaseState()); pd.setSchedule(schedule); + SensitivePropertyUtils.decryptAndMaskWorkflowDefinition(pd); } PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); @@ -549,7 +556,7 @@ public DagData queryWorkflowDefinitionByCode(User loginUser, long projectCode, l log.error("workflow definition does not exist, workflowDefinitionCode:{}.", code); throw new ServiceException(Status.WORKFLOW_DEFINITION_NOT_EXIST, String.valueOf(code)); } - return processService.genDagData(workflowDefinition); + return SensitivePropertyUtils.decryptAndMaskDagData(processService.genDagData(workflowDefinition)); } @Override @@ -583,7 +590,7 @@ public DagData queryWorkflowDefinitionByName(User loginUser, long projectCode, S log.error("workflow definition does not exist, projectCode:{}.", projectCode); throw new ServiceException(Status.WORKFLOW_DEFINITION_NOT_EXIST, name); } - return processService.genDagData(workflowDefinition); + return SensitivePropertyUtils.decryptAndMaskDagData(processService.genDagData(workflowDefinition)); } /** @@ -649,6 +656,9 @@ public WorkflowDefinition updateWorkflowDefinition(User loginUser, throw new ServiceException(Status.WORKFLOW_DEFINITION_NAME_EXIST, name); } } + globalParams = + SensitivePropertyUtils.mergeAndEncryptGlobalParams(globalParams, workflowDefinition.getGlobalParams()); + mergeAndEncryptSensitiveLocalParams(taskDefinitionLogs); WorkflowDefinition workflowDefinitionDeepCopy = JSONUtils.parseObject(JSONUtils.toJsonString(workflowDefinition), WorkflowDefinition.class); workflowDefinition.set(projectCode, name, description, globalParams, locations, timeout); @@ -657,6 +667,35 @@ public WorkflowDefinition updateWorkflowDefinition(User loginUser, taskDefinitionLogs); } + private void encryptSensitiveLocalParams(List taskDefinitionLogs) { + if (CollectionUtils.isEmpty(taskDefinitionLogs)) { + return; + } + taskDefinitionLogs.forEach(taskDefinitionLog -> taskDefinitionLog.setTaskParams( + SensitivePropertyUtils.encryptLocalParamsInTaskParams(taskDefinitionLog.getTaskParams()))); + } + + private void mergeAndEncryptSensitiveLocalParams(List taskDefinitionLogs) { + if (CollectionUtils.isEmpty(taskDefinitionLogs)) { + return; + } + Set taskDefinitionCodes = taskDefinitionLogs.stream() + .map(TaskDefinitionLog::getCode) + .filter(taskCode -> taskCode > 0) + .collect(Collectors.toSet()); + if (CollectionUtils.isEmpty(taskDefinitionCodes)) { + encryptSensitiveLocalParams(taskDefinitionLogs); + return; + } + Map existingTaskParamsMap = taskDefinitionDao.queryByCodes(taskDefinitionCodes) + .stream() + .collect(Collectors.toMap(TaskDefinition::getCode, TaskDefinition::getTaskParams)); + taskDefinitionLogs.forEach(taskDefinitionLog -> taskDefinitionLog.setTaskParams( + SensitivePropertyUtils.mergeAndEncryptLocalParamsInTaskParams( + taskDefinitionLog.getTaskParams(), + existingTaskParamsMap.get(taskDefinitionLog.getCode())))); + } + /** * Task want to delete whether used in other task, should throw exception when have be used. *

@@ -982,7 +1021,7 @@ public List getTaskNodeListByDefinitionCode(User loginUser, long log.error("workflow definition does not exist, workflowDefinitionCode:{}.", code); throw new ServiceException(Status.WORKFLOW_DEFINITION_NOT_EXIST, String.valueOf(code)); } - DagData dagData = processService.genDagData(workflowDefinition); + DagData dagData = SensitivePropertyUtils.decryptAndMaskDagData(processService.genDagData(workflowDefinition)); return dagData.getTaskDefinitionList(); } @@ -1021,7 +1060,8 @@ public Map> getNodeListMapByDefinitionCodes(User logi } Map> taskNodeMap = new HashMap<>(); for (WorkflowDefinition workflowDefinition : workflowDefinitionListInProject) { - DagData dagData = processService.genDagData(workflowDefinition); + DagData dagData = + SensitivePropertyUtils.decryptAndMaskDagData(processService.genDagData(workflowDefinition)); taskNodeMap.put(workflowDefinition.getCode(), dagData.getTaskDefinitionList()); } return taskNodeMap; @@ -1798,7 +1838,8 @@ public WorkflowDefinitionVariablesDTO viewVariables(User loginUser, long project } // global params - List globalParams = workflowDefinition.getGlobalParamList(); + List globalParams = + SensitivePropertyUtils.decryptAndMaskSensitiveValues(workflowDefinition.getGlobalParamList()); Map> localUserDefParams = getLocalParams(workflowDefinition); @@ -1828,7 +1869,9 @@ private Map> getLocalParams(WorkflowDefinition workf Map localParamsMap = new HashMap<>(); String localParams = JSONUtils.getNodeString(taskDefinition.getTaskParams(), LOCAL_PARAMS); if (!StringUtils.isEmpty(localParams)) { - List localParamsList = JSONUtils.toList(localParams, Property.class); + List localParamsList = + SensitivePropertyUtils.decryptAndMaskSensitiveValues( + JSONUtils.toList(localParams, Property.class)); localParamsMap.put(TASK_TYPE, taskDefinition.getTaskType()); localParamsMap.put(LOCAL_PARAMS_LIST, localParamsList); if (CollectionUtils.isNotEmpty(localParamsList)) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowInstanceServiceImpl.java index 5d024bbb5520..296e5310e3e7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkflowInstanceServiceImpl.java @@ -41,6 +41,7 @@ import org.apache.dolphinscheduler.api.service.WorkflowInstanceService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.api.utils.SensitivePropertyUtils; import org.apache.dolphinscheduler.common.constants.CommandKeyConstants; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.ContextType; @@ -79,6 +80,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.utils.GlobalParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.PropertySensitiveUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.model.TaskNode; @@ -94,6 +96,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -216,7 +219,10 @@ public WorkflowInstance queryWorkflowInstanceById(User loginUser, long projectCo throw new ServiceException(Status.WORKFLOW_DEFINITION_NOT_EXIST, workflowInstanceId); } workflowInstance.setLocations(workflowDefinition.getLocations()); - workflowInstance.setDagData(processService.genDagData(workflowDefinition)); + workflowInstance + .setGlobalParams(SensitivePropertyUtils.decryptAndMaskGlobalParams(workflowInstance.getGlobalParams())); + workflowInstance.setDagData( + SensitivePropertyUtils.decryptAndMaskDagData(processService.genDagData(workflowDefinition))); return workflowInstance; } @@ -458,6 +464,8 @@ public WorkflowDefinition updateWorkflowInstance(User loginUser, long projectCod timezoneId = commandParam.getTimeZone(); } + globalParams = + SensitivePropertyUtils.mergeAndEncryptGlobalParams(globalParams, workflowInstance.getGlobalParams()); setWorkflowInstance(workflowInstance, scheduleTime, globalParams, timeout, timezoneId); List taskDefinitionLogs = JSONUtils.toList(taskDefinitionJson, TaskDefinitionLog.class); if (taskDefinitionLogs.isEmpty()) { @@ -470,6 +478,7 @@ public WorkflowDefinition updateWorkflowInstance(User loginUser, long projectCod throw new ServiceException(Status.WORKFLOW_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName()); } } + mergeAndEncryptSensitiveLocalParams(taskDefinitionLogs); int saveTaskResult = processService.saveTaskDefine(loginUser, projectCode, taskDefinitionLogs, syncDefine); if (saveTaskResult == Constants.DEFINITION_FAILURE) { log.error("Update task definition error, projectCode:{}, workflowInstanceId:{}", projectCode, @@ -526,6 +535,28 @@ public WorkflowDefinition updateWorkflowInstance(User loginUser, long projectCod return workflowDefinition; } + private void mergeAndEncryptSensitiveLocalParams(List taskDefinitionLogs) { + if (CollectionUtils.isEmpty(taskDefinitionLogs)) { + return; + } + Set taskDefinitionCodes = taskDefinitionLogs.stream() + .map(TaskDefinitionLog::getCode) + .filter(taskCode -> taskCode > 0) + .collect(Collectors.toSet()); + if (CollectionUtils.isEmpty(taskDefinitionCodes)) { + taskDefinitionLogs.forEach(taskDefinitionLog -> taskDefinitionLog.setTaskParams( + SensitivePropertyUtils.encryptLocalParamsInTaskParams(taskDefinitionLog.getTaskParams()))); + return; + } + Map existingTaskParamsMap = taskDefinitionDao.queryByCodes(taskDefinitionCodes) + .stream() + .collect(Collectors.toMap(TaskDefinition::getCode, TaskDefinition::getTaskParams)); + taskDefinitionLogs.forEach(taskDefinitionLog -> taskDefinitionLog.setTaskParams( + SensitivePropertyUtils.mergeAndEncryptLocalParamsInTaskParams( + taskDefinitionLog.getTaskParams(), + existingTaskParamsMap.get(taskDefinitionLog.getCode())))); + } + /** * update workflow instance attributes */ @@ -645,7 +676,9 @@ private List processGlobalParams(WorkflowInstance workflowInstance, Ma if (StringUtils.isNotEmpty(globalParamsJson)) { // Replace placeholders String replacedJsonStr = ParameterUtils.convertParameterPlaceholders(globalParamsJson, parameterMap); - finalGlobalParams = GlobalParameterUtils.deserializeGlobalParameter(replacedJsonStr); + finalGlobalParams = + SensitivePropertyUtils.decryptSensitiveValues( + GlobalParameterUtils.deserializeGlobalParameter(replacedJsonStr)); // Merge into context map if (finalGlobalParams != null) { @@ -656,7 +689,7 @@ private List processGlobalParams(WorkflowInstance workflowInstance, Ma } } } - return finalGlobalParams; + return PropertySensitiveUtils.maskSensitiveValues(finalGlobalParams); } /** @@ -683,7 +716,8 @@ private Map> processLocalParams(WorkflowInstance wor if (!StringUtils.isEmpty(localParams)) { // Replace placeholders and deserialize localParams = ParameterUtils.convertParameterPlaceholders(localParams, parameterMap); - List localParamsList = JSONUtils.toList(localParams, Property.class); + List localParamsList = SensitivePropertyUtils.decryptAndMaskSensitiveValues( + JSONUtils.toList(localParams, Property.class)); Map localParamsMap = new HashMap<>(); localParamsMap.put(TASK_TYPE, taskDefinitionLog.getTaskType()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/SensitivePropertyUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/SensitivePropertyUtils.java new file mode 100644 index 000000000000..8abb3af0b249 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/SensitivePropertyUtils.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.utils; + +import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.DagData; +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; +import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.utils.GlobalParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.PropertySensitiveUtils; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +import lombok.experimental.UtilityClass; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +@UtilityClass +public class SensitivePropertyUtils { + + public String encryptGlobalParams(String globalParams) { + List properties = GlobalParameterUtils.deserializeGlobalParameter(globalParams); + if (CollectionUtils.isEmpty(properties)) { + return globalParams; + } + return GlobalParameterUtils.serializeGlobalParameter(encryptSensitiveValues(properties)); + } + + public String mergeAndEncryptGlobalParams(String submittedGlobalParams, String existingGlobalParams) { + List submittedProperties = GlobalParameterUtils.deserializeGlobalParameter(submittedGlobalParams); + if (CollectionUtils.isEmpty(submittedProperties)) { + return submittedGlobalParams; + } + List existingProperties = + decryptSensitiveValues(GlobalParameterUtils.deserializeGlobalParameter(existingGlobalParams)); + return GlobalParameterUtils.serializeGlobalParameter(encryptSensitiveValues( + PropertySensitiveUtils.mergeSensitiveValuePlaceholders(submittedProperties, existingProperties))); + } + + public String decryptAndMaskGlobalParams(String globalParams) { + List properties = GlobalParameterUtils.deserializeGlobalParameter(globalParams); + if (CollectionUtils.isEmpty(properties)) { + return globalParams; + } + return GlobalParameterUtils.serializeGlobalParameter(decryptAndMaskSensitiveValues(properties)); + } + + public List decryptAndMaskSensitiveValues(List properties) { + return PropertySensitiveUtils.maskSensitiveValues(decryptSensitiveValues(properties)); + } + + public DagData decryptAndMaskDagData(DagData dagData) { + if (dagData == null) { + return null; + } + decryptAndMaskWorkflowDefinition(dagData.getWorkflowDefinition()); + if (CollectionUtils.isNotEmpty(dagData.getTaskDefinitionList())) { + dagData.getTaskDefinitionList().forEach(SensitivePropertyUtils::decryptAndMaskTaskDefinition); + } + return dagData; + } + + public WorkflowDefinition decryptAndMaskWorkflowDefinition(WorkflowDefinition workflowDefinition) { + if (workflowDefinition == null) { + return null; + } + List globalParams = + decryptAndMaskSensitiveValues(GlobalParameterUtils.deserializeGlobalParameter( + workflowDefinition.getGlobalParams())); + workflowDefinition.setGlobalParams(GlobalParameterUtils.serializeGlobalParameter(globalParams)); + workflowDefinition.setGlobalParamList(globalParams); + return workflowDefinition; + } + + public TaskDefinition decryptAndMaskTaskDefinition(TaskDefinition taskDefinition) { + if (taskDefinition == null) { + return null; + } + taskDefinition.setTaskParams(decryptAndMaskLocalParamsInTaskParams(taskDefinition.getTaskParams())); + return taskDefinition; + } + + public String encryptLocalParamsInTaskParams(String taskParams) { + return transformLocalParamsInTaskParams(taskParams, SensitivePropertyUtils::encryptSensitiveValues); + } + + public String mergeAndEncryptLocalParamsInTaskParams(String submittedTaskParams, String existingTaskParams) { + return transformLocalParamsInTaskParams(submittedTaskParams, submittedProperties -> { + List existingProperties = decryptSensitiveValues(getLocalParams(existingTaskParams)); + return encryptSensitiveValues( + PropertySensitiveUtils.mergeSensitiveValuePlaceholders(submittedProperties, existingProperties)); + }); + } + + public String decryptAndMaskLocalParamsInTaskParams(String taskParams) { + return transformLocalParamsInTaskParams(taskParams, SensitivePropertyUtils::decryptAndMaskSensitiveValues); + } + + public String decryptLocalParamsInTaskParams(String taskParams) { + return transformLocalParamsInTaskParams(taskParams, SensitivePropertyUtils::decryptSensitiveValues); + } + + public List encryptSensitiveValues(List properties) { + if (CollectionUtils.isEmpty(properties)) { + return Collections.emptyList(); + } + List encryptedProperties = PropertySensitiveUtils.copy(properties); + encryptedProperties.stream() + .filter(PropertySensitiveUtils::isSensitive) + .filter(property -> StringUtils.isNotEmpty(property.getValue())) + .filter(property -> !PropertySensitiveUtils.isSensitiveValuePlaceholder(property.getValue())) + .forEach(property -> property.setValue(PasswordUtils.encodePassword(property.getValue()))); + return encryptedProperties; + } + + public List decryptSensitiveValues(List properties) { + if (CollectionUtils.isEmpty(properties)) { + return Collections.emptyList(); + } + List decryptedProperties = PropertySensitiveUtils.copy(properties); + decryptedProperties.stream() + .filter(PropertySensitiveUtils::isSensitive) + .filter(property -> StringUtils.isNotEmpty(property.getValue())) + .filter(property -> !PropertySensitiveUtils.isSensitiveValuePlaceholder(property.getValue())) + .forEach(property -> property.setValue(PasswordUtils.decodePassword(property.getValue()))); + return decryptedProperties; + } + + public List getLocalParams(String taskParams) { + if (StringUtils.isEmpty(taskParams)) { + return Collections.emptyList(); + } + String localParams = JSONUtils.getNodeString(taskParams, LOCAL_PARAMS); + if (StringUtils.isEmpty(localParams)) { + return Collections.emptyList(); + } + return JSONUtils.toList(localParams, Property.class); + } + + private String transformLocalParamsInTaskParams(String taskParams, + Function, List> transformFunction) { + if (StringUtils.isEmpty(taskParams)) { + return taskParams; + } + ObjectNode taskParamsNode = JSONUtils.parseObject(taskParams); + JsonNode localParamsNode = taskParamsNode.findValue(LOCAL_PARAMS); + if (localParamsNode == null || localParamsNode.isNull()) { + return taskParams; + } + List localParams = JSONUtils.toList(localParamsNode.toString(), Property.class); + taskParamsNode.set(LOCAL_PARAMS, JSONUtils.toJsonNode(transformFunction.apply(localParams))); + return JSONUtils.toJsonString(taskParamsNode); + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkflowDefinitionControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkflowDefinitionControllerTest.java index a65213038a97..06336a3960e9 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkflowDefinitionControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkflowDefinitionControllerTest.java @@ -33,6 +33,9 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition; import org.apache.dolphinscheduler.dao.entity.WorkflowDefinitionLog; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; import java.text.MessageFormat; import java.util.ArrayList; @@ -331,13 +334,20 @@ public void testDeleteWorkflowDefinitionVersion() { @Test public void testViewVariables() { long projectCode = 1L; + Property sensitiveParam = new Property("secret", Direct.IN, DataType.VARCHAR, "******"); + sensitiveParam.setSensitive(true); + WorkflowDefinitionVariablesDTO variablesDTO = + new WorkflowDefinitionVariablesDTO(Collections.singletonList(sensitiveParam), Collections.emptyMap()); Mockito.when(processDefinitionService.viewVariables(user, projectCode, 1L)) - .thenReturn(new WorkflowDefinitionVariablesDTO()); + .thenReturn(variablesDTO); Result result = workflowDefinitionController.viewVariables(user, projectCode, 1L); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + WorkflowDefinitionVariablesDTO resultData = (WorkflowDefinitionVariablesDTO) result.getData(); + Assertions.assertEquals("******", resultData.getGlobalParams().get(0).getValue()); + Assertions.assertTrue(resultData.getGlobalParams().get(0).isSensitive()); } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkflowInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkflowInstanceControllerTest.java index ada5e70e92a6..6325bd522260 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkflowInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkflowInstanceControllerTest.java @@ -33,6 +33,9 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition; import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; import java.util.ArrayList; import java.util.Collections; @@ -193,8 +196,10 @@ public void testQueryParentInstanceBySubId() throws Exception { @Test public void testViewVariables() throws Exception { + Property sensitiveParam = new Property("secret", Direct.IN, DataType.VARCHAR, "******"); + sensitiveParam.setSensitive(true); WorkflowInstanceVariablesDTO mockResult = - new WorkflowInstanceVariablesDTO(Collections.emptyList(), Collections.emptyMap()); + new WorkflowInstanceVariablesDTO(Collections.singletonList(sensitiveParam), Collections.emptyMap()); Mockito.when(workflowInstanceService.viewVariables(Mockito.any(), Mockito.eq(1113L), Mockito.eq(123))) .thenReturn(mockResult); MvcResult mvcResult = mockMvc @@ -206,6 +211,9 @@ public void testViewVariables() throws Exception { Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Assertions.assertNotNull(result); Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + String responseContent = mvcResult.getResponse().getContentAsString(); + Assertions.assertTrue(responseContent.contains("\"value\":\"******\"")); + Assertions.assertTrue(responseContent.contains("\"sensitive\":true")); } @Test diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java index 3ea6da8fb3a9..133e1c33e2b7 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java @@ -35,6 +35,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.apache.dolphinscheduler.api.dto.workflow.WorkflowDefinitionVariablesDTO; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; @@ -51,9 +52,11 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.entity.DagData; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Schedule; +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; import org.apache.dolphinscheduler.dao.entity.TaskMainInfo; import org.apache.dolphinscheduler.dao.entity.User; @@ -74,8 +77,11 @@ import org.apache.dolphinscheduler.dao.repository.WorkflowDefinitionLogDao; import org.apache.dolphinscheduler.dao.repository.WorkflowTaskRelationDao; import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; import org.apache.dolphinscheduler.plugin.task.api.model.ConditionDependentItem; import org.apache.dolphinscheduler.plugin.task.api.model.ConditionDependentTaskModel; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo; import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; @@ -111,8 +117,10 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.mock.web.MockMultipartFile; @@ -208,6 +216,7 @@ public class WorkflowDefinitionServiceTest extends BaseServiceTestTool { protected final static String name = "testProcessDefinitionName"; protected final static String description = "this is a description"; protected final static int timeout = 60; + private static final String SENSITIVE_DATA_MASK = "******"; @BeforeEach public void before() { @@ -845,6 +854,48 @@ public void testCreateWorkflowDefinitionShouldSyncVersionToResponse() { Assertions.assertEquals(1, workflowDefinition.getVersion()); } + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void testCreateWorkflowDefinitionShouldEncryptSensitiveParamsBeforeSaving() { + Project project = getProject(projectCode); + when(projectDao.queryByCode(projectCode)).thenReturn(project); + Mockito.doNothing().when(projectService).checkHasProjectWritePermissionThrowException(eq(user), eq(project)); + when(workflowDefinitionDao.verifyByDefineName(projectCode, name)).thenReturn(null); + when(processService.transformTask(anyList(), anyList())).thenReturn(getTaskNodeList()); + when(processService.saveTaskDefine(eq(user), eq(projectCode), anyList(), eq(Boolean.TRUE))).thenReturn(1); + when(processService.saveWorkflowDefine(any(User.class), any(WorkflowDefinition.class), eq(Boolean.TRUE), + eq(Boolean.TRUE))).thenReturn(1); + when(processService.saveTaskRelation(eq(user), eq(projectCode), anyLong(), eq(1), anyList(), anyList(), + eq(Boolean.TRUE))).thenReturn(Constants.EXIT_CODE_SUCCESS); + + try (MockedStatic ignored = mockEncryptionEnabled()) { + workflowDefinitionService.createWorkflowDefinition( + user, projectCode, name, description, sensitiveGlobalParams("global_secret"), "[]", timeout, + taskRelationJson, sensitiveTaskDefinitionJson(123456789L, "local_secret"), null, + WorkflowExecutionTypeEnum.PARALLEL); + } + + ArgumentCaptor workflowDefinitionCaptor = ArgumentCaptor.forClass(WorkflowDefinition.class); + verify(processService).saveWorkflowDefine(eq(user), workflowDefinitionCaptor.capture(), eq(Boolean.TRUE), + eq(Boolean.TRUE)); + Property savedGlobalParam = + JSONUtils.toList(workflowDefinitionCaptor.getValue().getGlobalParams(), Property.class).get(0); + Assertions.assertTrue(savedGlobalParam.isSensitive()); + Assertions.assertNotEquals("global_secret", savedGlobalParam.getValue()); + assertDecodedValue("global_secret", savedGlobalParam.getValue()); + + ArgumentCaptor taskDefinitionLogsCaptor = ArgumentCaptor.forClass(List.class); + verify(processService).saveTaskDefine(eq(user), eq(projectCode), taskDefinitionLogsCaptor.capture(), + eq(Boolean.TRUE)); + TaskDefinitionLog taskDefinitionLog = (TaskDefinitionLog) taskDefinitionLogsCaptor.getValue().get(0); + Property savedLocalParam = + JSONUtils.toList(JSONUtils.getNodeString(taskDefinitionLog.getTaskParams(), Constants.LOCAL_PARAMS), + Property.class).get(0); + Assertions.assertTrue(savedLocalParam.isSensitive()); + Assertions.assertNotEquals("local_secret", savedLocalParam.getValue()); + assertDecodedValue("local_secret", savedLocalParam.getValue()); + } + @Test public void testUpdateWorkflowDefinitionShouldSyncVersionToResponse() { Project project = getProject(projectCode); @@ -871,6 +922,59 @@ public void testUpdateWorkflowDefinitionShouldSyncVersionToResponse() { Assertions.assertEquals(2, resultDefinition.getVersion()); } + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void testUpdateWorkflowDefinitionShouldKeepOldSensitiveValueWhenPlaceholderSubmitted() { + Project project = getProject(projectCode); + WorkflowDefinition workflowDefinition = getWorkflowDefinition(); + workflowDefinition.setName(name); + TaskDefinition existingTaskDefinition = new TaskDefinition(); + existingTaskDefinition.setCode(123456789L); + + when(projectDao.queryByCode(projectCode)).thenReturn(project); + Mockito.doNothing().when(projectService).checkHasProjectWritePermissionThrowException(eq(user), eq(project)); + when(processService.transformTask(anyList(), anyList())).thenReturn(getTaskNodeList()); + when(workflowDefinitionDao.queryByCode(processDefinitionCode)).thenReturn(Optional.of(workflowDefinition)); + when(processService.saveTaskDefine(eq(user), eq(projectCode), anyList(), eq(Boolean.TRUE))).thenReturn(1); + when(processService.saveWorkflowDefine(any(User.class), any(WorkflowDefinition.class), eq(Boolean.TRUE), + eq(Boolean.TRUE))).thenReturn(2); + when(workflowTaskRelationDao.queryByWorkflowDefinitionCode(processDefinitionCode)) + .thenReturn(Collections.emptyList()); + when(processService.saveTaskRelation(eq(user), eq(projectCode), eq(processDefinitionCode), eq(2), anyList(), + anyList(), eq(Boolean.TRUE))).thenReturn(Constants.EXIT_CODE_SUCCESS); + + try (MockedStatic ignored = mockEncryptionEnabled()) { + workflowDefinition + .setGlobalParams(sensitiveGlobalParams(PasswordUtils.encodePassword("old_global_secret"))); + existingTaskDefinition.setTaskParams(sensitiveTaskParams(PasswordUtils.encodePassword("old_local_secret"))); + when(taskDefinitionDao.queryByCodes(Collections.singleton(123456789L))) + .thenReturn(Collections.singletonList(existingTaskDefinition)); + + workflowDefinitionService.updateWorkflowDefinition( + user, projectCode, name, processDefinitionCode, description, + sensitiveGlobalParams(SENSITIVE_DATA_MASK), "[]", timeout, taskRelationJson, + sensitiveTaskDefinitionJson(123456789L, SENSITIVE_DATA_MASK), WorkflowExecutionTypeEnum.PARALLEL); + } + + ArgumentCaptor workflowDefinitionCaptor = ArgumentCaptor.forClass(WorkflowDefinition.class); + verify(processService).saveWorkflowDefine(eq(user), workflowDefinitionCaptor.capture(), eq(Boolean.TRUE), + eq(Boolean.TRUE)); + Property savedGlobalParam = + JSONUtils.toList(workflowDefinitionCaptor.getValue().getGlobalParams(), Property.class).get(0); + Assertions.assertNotEquals(SENSITIVE_DATA_MASK, savedGlobalParam.getValue()); + assertDecodedValue("old_global_secret", savedGlobalParam.getValue()); + + ArgumentCaptor taskDefinitionLogsCaptor = ArgumentCaptor.forClass(List.class); + verify(processService).saveTaskDefine(eq(user), eq(projectCode), taskDefinitionLogsCaptor.capture(), + eq(Boolean.TRUE)); + TaskDefinitionLog taskDefinitionLog = (TaskDefinitionLog) taskDefinitionLogsCaptor.getValue().get(0); + Property savedLocalParam = + JSONUtils.toList(JSONUtils.getNodeString(taskDefinitionLog.getTaskParams(), Constants.LOCAL_PARAMS), + Property.class).get(0); + Assertions.assertNotEquals(SENSITIVE_DATA_MASK, savedLocalParam.getValue()); + assertDecodedValue("old_local_secret", savedLocalParam.getValue()); + } + @Test public void testGetNewProcessName() { String processName1 = "test_copy_" + DateUtils.getCurrentTimeStamp(); @@ -901,6 +1005,41 @@ public void testViewVariables() { Assertions.assertEquals(Status.PROJECT_NOT_FOUND.getCode(), ex.getCode()); } + @Test + @SuppressWarnings("unchecked") + public void testViewVariablesShouldMaskSensitiveGlobalAndLocalParams() { + Project project = getProject(projectCode); + when(projectDao.queryByCode(projectCode)).thenReturn(project); + Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(user, project, WORKFLOW_DEFINITION); + + WorkflowDefinition workflowDefinition = getWorkflowDefinition(); + WorkflowTaskRelation workflowTaskRelation = new WorkflowTaskRelation(); + workflowTaskRelation.setPostTaskCode(123456789L); + TaskDefinition taskDefinition = new TaskDefinition(); + taskDefinition.setCode(123456789L); + taskDefinition.setName("shell"); + taskDefinition.setTaskType("SHELL"); + + try (MockedStatic ignored = mockEncryptionEnabled()) { + workflowDefinition.setGlobalParams(sensitiveGlobalParams(PasswordUtils.encodePassword("global_secret"))); + taskDefinition.setTaskParams(sensitiveTaskParams(PasswordUtils.encodePassword("local_secret"))); + } + when(workflowDefinitionDao.queryByCode(processDefinitionCode)).thenReturn(Optional.of(workflowDefinition)); + when(workflowTaskRelationDao.queryByWorkflowDefinitionCode(processDefinitionCode)) + .thenReturn(Collections.singletonList(workflowTaskRelation)); + when(taskDefinitionDao.queryByCodes(Collections.singleton(123456789L))) + .thenReturn(Collections.singletonList(taskDefinition)); + + try (MockedStatic ignored = mockEncryptionEnabled()) { + WorkflowDefinitionVariablesDTO result = + workflowDefinitionService.viewVariables(user, projectCode, processDefinitionCode); + + Assertions.assertEquals(SENSITIVE_DATA_MASK, result.getGlobalParams().get(0).getValue()); + List localParams = (List) result.getLocalParams().get("shell").get("localParamsList"); + Assertions.assertEquals(SENSITIVE_DATA_MASK, localParams.get(0).getValue()); + } + } + /** * get mock processDefinition * @@ -918,6 +1057,45 @@ private WorkflowDefinition getWorkflowDefinition() { return workflowDefinition; } + private MockedStatic mockEncryptionEnabled() { + MockedStatic propertyUtilsMockedStatic = Mockito.mockStatic(PropertyUtils.class); + propertyUtilsMockedStatic.when(() -> PropertyUtils.getBoolean( + DataSourceConstants.DATASOURCE_ENCRYPTION_ENABLE, false)).thenReturn(Boolean.TRUE); + propertyUtilsMockedStatic.when(() -> PropertyUtils.getString( + DataSourceConstants.DATASOURCE_ENCRYPTION_SALT, + DataSourceConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT)) + .thenReturn(DataSourceConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); + return propertyUtilsMockedStatic; + } + + private void assertDecodedValue(String expectedValue, String encryptedValue) { + try (MockedStatic ignored = mockEncryptionEnabled()) { + Assertions.assertEquals(expectedValue, PasswordUtils.decodePassword(encryptedValue)); + } + } + + private String sensitiveGlobalParams(String value) { + return "[{\"prop\":\"secret_global\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"" + value + + "\",\"sensitive\":true}]"; + } + + private String sensitiveTaskDefinitionJson(long taskCode, String localParamValue) { + return "[{\"code\":" + taskCode + + ",\"name\":\"test1\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\"," + + "\"taskParams\":" + sensitiveTaskParams(localParamValue) + + ",\"flag\":\"YES\",\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0," + + "\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":null,\"timeout\":0," + + "\"environmentCode\":-1}]"; + } + + private String sensitiveTaskParams(String localParamValue) { + return "{\"resourceList\":[],\"localParams\":[{\"prop\":\"secret_local\",\"direct\":\"IN\"," + + "\"type\":\"VARCHAR\",\"value\":\"" + localParamValue + "\",\"sensitive\":true}]," + + "\"rawScript\":\"echo ${secret_local}\",\"dependence\":{}," + + "\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{}," + + "\"switchResult\":{}}"; + } + /** * get mock Project * diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowInstanceServiceTest.java index 88e9051ffb16..8abbb96c66d6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowInstanceServiceTest.java @@ -44,6 +44,7 @@ import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.DependentResultTaskInstanceContext; import org.apache.dolphinscheduler.dao.entity.Project; @@ -57,6 +58,7 @@ import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition; import org.apache.dolphinscheduler.dao.entity.WorkflowDefinitionLog; import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; +import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.WorkflowDefinitionLogMapper; import org.apache.dolphinscheduler.dao.repository.ProjectDao; import org.apache.dolphinscheduler.dao.repository.TaskDefinitionDao; @@ -67,6 +69,8 @@ import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao; import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceMapDao; import org.apache.dolphinscheduler.extract.master.command.RunWorkflowCommandParam; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; @@ -123,6 +127,9 @@ public class WorkflowInstanceServiceTest { @Mock WorkflowDefinitionLogMapper workflowDefinitionLogMapper; + @Mock + TaskDefinitionLogMapper taskDefinitionLogMapper; + @Mock WorkflowDefinitionDao workflowDefinitionDao; @@ -156,6 +163,8 @@ public class WorkflowInstanceServiceTest { @Mock AlertDao alertDao; + private static final String SENSITIVE_DATA_MASK = "******"; + private String shellJson = "[{\"name\":\"\",\"preTaskCode\":0,\"preTaskVersion\":0,\"postTaskCode\":123456789," + "\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":\"{}\"},{\"name\":\"\",\"preTaskCode\":123456789," + "\"preTaskVersion\":1,\"postTaskCode\":123451234,\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":\"{}\"}]"; @@ -818,6 +827,50 @@ public void testViewVariables_EmptyGlobalParams() { Assertions.assertTrue(globalParams.isEmpty(), "Global params list should be empty when input is empty string"); } + @Test + @SuppressWarnings("unchecked") + public void testViewVariablesShouldMaskSensitiveGlobalAndLocalParams() { + long projectCode = 1L; + User loginUser = getAdminUser(); + doNothing().when(projectService) + .checkProjectAndAuthThrowException(loginUser, projectCode, WORKFLOW_INSTANCE); + + WorkflowInstance workflowInstance = getProcessInstance(); + workflowInstance.setId(3); + workflowInstance.setCommandType(CommandType.START_PROCESS); + workflowInstance.setScheduleTime(new Date()); + workflowInstance.setWorkflowDefinitionCode(46L); + + WorkflowDefinition workflowDefinition = getProcessDefinition(); + workflowDefinition.setProjectCode(projectCode); + + TaskInstance taskInstance = getTaskInstance(); + taskInstance.setTaskCode(123456789L); + taskInstance.setTaskDefinitionVersion(1); + TaskDefinitionLog taskDefinitionLog = new TaskDefinitionLog(); + taskDefinitionLog.setCode(123456789L); + taskDefinitionLog.setName("shell"); + taskDefinitionLog.setTaskType("SHELL"); + + try (MockedStatic ignored = mockEncryptionEnabled()) { + workflowInstance.setGlobalParams(sensitiveGlobalParams(PasswordUtils.encodePassword("global_secret"))); + taskDefinitionLog.setTaskParams(sensitiveTaskParams(PasswordUtils.encodePassword("local_secret"))); + } + + when(workflowInstanceDao.queryDetailById(3)).thenReturn(workflowInstance); + when(workflowDefinitionDao.queryByCode(46L)).thenReturn(Optional.of(workflowDefinition)); + when(taskInstanceDao.queryValidTaskListByWorkflowInstanceId(3)).thenReturn(Lists.newArrayList(taskInstance)); + when(taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(123456789L, 1)).thenReturn(taskDefinitionLog); + + try (MockedStatic ignored = mockEncryptionEnabled()) { + WorkflowInstanceVariablesDTO result = workflowInstanceService.viewVariables(loginUser, projectCode, 3); + + Assertions.assertEquals(SENSITIVE_DATA_MASK, result.getGlobalParams().get(0).getValue()); + List localParams = (List) result.getLocalParams().get("shell").get("localParamsList"); + Assertions.assertEquals(SENSITIVE_DATA_MASK, localParams.get(0).getValue()); + } + } + @Test public void testViewGantt() throws Exception { long projectCode = 0L; @@ -955,6 +1008,30 @@ private WorkflowDefinition getProcessDefinition() { return workflowDefinition; } + private MockedStatic mockEncryptionEnabled() { + MockedStatic propertyUtilsMockedStatic = Mockito.mockStatic(PropertyUtils.class); + propertyUtilsMockedStatic.when(() -> PropertyUtils.getBoolean( + DataSourceConstants.DATASOURCE_ENCRYPTION_ENABLE, false)).thenReturn(Boolean.TRUE); + propertyUtilsMockedStatic.when(() -> PropertyUtils.getString( + DataSourceConstants.DATASOURCE_ENCRYPTION_SALT, + DataSourceConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT)) + .thenReturn(DataSourceConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); + return propertyUtilsMockedStatic; + } + + private String sensitiveGlobalParams(String value) { + return "[{\"prop\":\"secret_global\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"" + value + + "\",\"sensitive\":true}]"; + } + + private String sensitiveTaskParams(String localParamValue) { + return "{\"resourceList\":[],\"localParams\":[{\"prop\":\"secret_local\",\"direct\":\"IN\"," + + "\"type\":\"VARCHAR\",\"value\":\"" + localParamValue + "\",\"sensitive\":true}]," + + "\"rawScript\":\"echo ${secret_local}\",\"dependence\":{}," + + "\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{}," + + "\"switchResult\":{}}"; + } + private Tenant getTenant() { Tenant tenant = new Tenant(); tenant.setId(1); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RunWorkflowCommandHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RunWorkflowCommandHandler.java index 414921a6ba98..0e19adf54f04 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RunWorkflowCommandHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/handler/RunWorkflowCommandHandler.java @@ -28,6 +28,7 @@ import org.apache.dolphinscheduler.extract.master.command.RunWorkflowCommandParam; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.utils.GlobalParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.PropertySensitiveUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.engine.graph.IWorkflowGraph; import org.apache.dolphinscheduler.server.master.engine.graph.WorkflowExecutionGraph; @@ -35,6 +36,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.execution.TaskExecution; import org.apache.dolphinscheduler.server.master.engine.task.execution.TaskExecutionBuilder; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteContext.WorkflowExecuteContextBuilder; +import org.apache.dolphinscheduler.server.master.utils.MasterSensitivePropertyUtils; import org.apache.commons.collections4.CollectionUtils; @@ -131,9 +133,10 @@ private String mergeCommandParamsWithWorkflowParams(final Command command, globalParamsList.forEach(globalParam -> finalParams.put(globalParam.getProp(), globalParam)); } if (CollectionUtils.isNotEmpty(commandParams)) { - commandParams.forEach(commandParam -> finalParams.put(commandParam.getProp(), commandParam)); + PropertySensitiveUtils.mergeSensitiveValuePlaceholders(commandParams, globalParamsList) + .forEach(commandParam -> finalParams.put(commandParam.getProp(), commandParam)); } - return JSONUtils.toJsonString(finalParams.values()); + return JSONUtils.toJsonString(MasterSensitivePropertyUtils.decryptSensitiveValues(finalParams.values())); } @Override diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java index 653088911514..26bb7191315c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java @@ -46,6 +46,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.execution.ITaskExecution; import org.apache.dolphinscheduler.server.master.engine.task.execution.TaskExecutionContextBuilder; import org.apache.dolphinscheduler.server.master.engine.task.execution.TaskExecutionContextCreateRequest; +import org.apache.dolphinscheduler.server.master.utils.MasterSensitivePropertyUtils; import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -168,16 +169,18 @@ private Map getPrepareParams(final TaskInstance taskInstance, final WorkflowInstance workflowInstance, final WorkflowDefinition workflowDefinition, final Project project) { + taskInstance.setTaskParams( + MasterSensitivePropertyUtils.decryptLocalParamsInTaskParams(taskInstance.getTaskParams())); final AbstractParameters baseParam = TaskPluginManager.parseTaskParameters( taskInstance.getTaskType(), taskInstance.getTaskParams()); - return curingParamsService.paramParsingPreparation( + return MasterSensitivePropertyUtils.decryptPrepareParams(curingParamsService.paramParsingPreparation( taskInstance, baseParam, workflowInstance, project.getName(), - workflowDefinition.getName()); + workflowDefinition.getName())); } private Optional getEnvironmentConfigFromDB(final TaskInstance taskInstance) { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/MasterSensitivePropertyUtils.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/MasterSensitivePropertyUtils.java new file mode 100644 index 000000000000..be555f8a0776 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/MasterSensitivePropertyUtils.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.utils; + +import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.utils.GlobalParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.PropertySensitiveUtils; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import lombok.experimental.UtilityClass; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +@UtilityClass +public class MasterSensitivePropertyUtils { + + public String decryptGlobalParams(String globalParams) { + List properties = GlobalParameterUtils.deserializeGlobalParameter(globalParams); + if (CollectionUtils.isEmpty(properties)) { + return globalParams; + } + return GlobalParameterUtils.serializeGlobalParameter(decryptSensitiveValues(properties)); + } + + public String decryptLocalParamsInTaskParams(String taskParams) { + if (StringUtils.isEmpty(taskParams)) { + return taskParams; + } + ObjectNode taskParamsNode = JSONUtils.parseObject(taskParams); + JsonNode localParamsNode = taskParamsNode.findValue(LOCAL_PARAMS); + if (localParamsNode == null || localParamsNode.isNull()) { + return taskParams; + } + List localParams = JSONUtils.toList(localParamsNode.toString(), Property.class); + taskParamsNode.set(LOCAL_PARAMS, JSONUtils.toJsonNode(decryptSensitiveValues(localParams))); + return JSONUtils.toJsonString(taskParamsNode); + } + + public Map decryptPrepareParams(Map prepareParams) { + if (prepareParams == null || prepareParams.isEmpty()) { + return prepareParams; + } + prepareParams.replaceAll((key, property) -> decryptSensitiveValue(property)); + return prepareParams; + } + + public List decryptSensitiveValues(Collection properties) { + if (CollectionUtils.isEmpty(properties)) { + return Collections.emptyList(); + } + return properties.stream() + .map(MasterSensitivePropertyUtils::decryptSensitiveValue) + .collect(Collectors.toList()); + } + + private Property decryptSensitiveValue(Property property) { + Property decryptedProperty = PropertySensitiveUtils.copy(property); + if (!PropertySensitiveUtils.isSensitive(decryptedProperty) + || StringUtils.isEmpty(decryptedProperty.getValue()) + || PropertySensitiveUtils.isSensitiveValuePlaceholder(decryptedProperty.getValue())) { + return decryptedProperty; + } + decryptedProperty.setValue(PasswordUtils.decodePassword(decryptedProperty.getValue())); + return decryptedProperty; + } +} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java index 8a68b7579f97..2c025f32cb66 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java @@ -46,12 +46,14 @@ import org.apache.dolphinscheduler.plugin.task.api.utils.GlobalParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.PropertySensitiveUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.VarPoolUtils; import org.apache.dolphinscheduler.service.exceptions.ServiceException; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -189,7 +191,10 @@ public Map paramParsingPreparation(@NonNull TaskInstance taskI // 5. Command-line / complement parameters if (CollectionUtils.isNotEmpty(commandParam.getCommandParams())) { - Map commandParamsMap = commandParam.getCommandParams().stream() + Map commandParamsMap = PropertySensitiveUtils + .mergeSensitiveValuePlaceholders(commandParam.getCommandParams(), + new ArrayList<>(prepareParamsMap.values())) + .stream() .filter(prop -> StringUtils.isNotBlank(prop.getProp())) .collect(Collectors.toMap( Property::getProp, diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImplTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImplTest.java index 1f0beb349eee..0618edcc2820 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImplTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImplTest.java @@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; import org.apache.dolphinscheduler.dao.mapper.ProjectParameterMapper; import org.apache.dolphinscheduler.extract.master.command.BackfillWorkflowCommandParam; +import org.apache.dolphinscheduler.extract.master.command.RunWorkflowCommandParam; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; @@ -232,6 +233,45 @@ public void testParamParsingPreparation() { String.valueOf(workflowDefinition.getCode())); } + @Test + public void testParamParsingPreparationShouldKeepGlobalValueWhenSensitiveCommandParamIsPlaceholder() { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(1); + taskInstance.setExecutePath("home/path/execute"); + taskInstance.setProjectCode(3000001L); + taskInstance.setWorkflowInstanceId(2); + + WorkflowInstance workflowInstance = new WorkflowInstance(); + workflowInstance.setId(2); + workflowInstance.setProjectCode(3000001L); + workflowInstance.setWorkflowDefinitionCode(200001L); + workflowInstance.setCommandType(CommandType.START_PROCESS); + workflowInstance.setHistoryCmd(CommandType.START_PROCESS.toString()); + workflowInstance.setScheduleTime(new Date()); + + Property globalParam = new Property("var", Direct.IN, DataType.VARCHAR, "321"); + globalParam.setSensitive(true); + workflowInstance + .setGlobalParams(GlobalParameterUtils.serializeGlobalParameter(Lists.newArrayList(globalParam))); + + Property commandParam = new Property("var", Direct.IN, DataType.VARCHAR, "******"); + commandParam.setSensitive(true); + RunWorkflowCommandParam runWorkflowCommandParam = RunWorkflowCommandParam.builder() + .commandParams(Lists.newArrayList(commandParam)) + .timeZone("Asia/Shanghai") + .build(); + workflowInstance.setCommandParam(JSONUtils.toJsonString(runWorkflowCommandParam)); + + Mockito.when(projectParameterMapper.queryByProjectCode(Mockito.anyLong())).thenReturn(Collections.emptyList()); + + Map propertyMap = + curingParamsServiceImpl.paramParsingPreparation(taskInstance, new SubWorkflowParameters(), + workflowInstance, "ProjectName", "ProcessName-1"); + + Assertions.assertEquals("321", propertyMap.get("var").getValue()); + Assertions.assertTrue(propertyMap.get("var").isSensitive()); + } + @Test public void testParseGlobalParamsMap() throws Exception { WorkflowInstance workflowInstance = new WorkflowInstance(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverter.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverter.java index ea70aaef7678..737ae2b9fa57 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverter.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverter.java @@ -21,7 +21,9 @@ import org.apache.commons.lang3.StringUtils; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -34,8 +36,9 @@ */ public class SensitiveDataConverter extends MessageConverter { - private static Pattern multilinePattern; + private static volatile Pattern multilinePattern; private static final Set maskPatterns = new HashSet<>(); + private static final Map dynamicMaskPatternRefCount = new HashMap<>(); static { addMaskPattern(TaskConstants.DATASOURCE_PASSWORD_REGEX); @@ -56,7 +59,31 @@ public static synchronized void addMaskPattern(final String maskPattern) { return; } maskPatterns.add(maskPattern); - multilinePattern = Pattern.compile(String.join("|", maskPatterns), Pattern.MULTILINE); + refreshMultilinePattern(); + } + + public static synchronized void addDynamicMaskPattern(final String maskPattern) { + if (StringUtils.isEmpty(maskPattern)) { + return; + } + dynamicMaskPatternRefCount.put(maskPattern, dynamicMaskPatternRefCount.getOrDefault(maskPattern, 0) + 1); + refreshMultilinePattern(); + } + + public static synchronized void removeDynamicMaskPattern(final String maskPattern) { + if (StringUtils.isEmpty(maskPattern)) { + return; + } + Integer refCount = dynamicMaskPatternRefCount.get(maskPattern); + if (refCount == null) { + return; + } + if (refCount <= 1) { + dynamicMaskPatternRefCount.remove(maskPattern); + } else { + dynamicMaskPatternRefCount.put(maskPattern, refCount - 1); + } + refreshMultilinePattern(); } public static String maskSensitiveData(final String logMsg) { @@ -75,4 +102,10 @@ public static String maskSensitiveData(final String logMsg) { return sb.toString(); } + private static void refreshMultilinePattern() { + Set allMaskPatterns = new HashSet<>(maskPatterns); + allMaskPatterns.addAll(dynamicMaskPatternRefCount.keySet()); + multilinePattern = Pattern.compile(String.join("|", allMaskPatterns), Pattern.MULTILINE); + } + } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Property.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Property.java index e96f4020d3a0..350727d9b366 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Property.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Property.java @@ -51,4 +51,14 @@ public class Property implements Serializable { private String value; + @Builder.Default + private boolean sensitive = false; + + public Property(String prop, Direct direct, DataType type, String value) { + this.prop = prop; + this.direct = direct; + this.type = type; + this.value = value; + } + } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/PropertySensitiveUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/PropertySensitiveUtils.java new file mode 100644 index 000000000000..c10cd46f26b8 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/PropertySensitiveUtils.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.utils; + +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class PropertySensitiveUtils { + + public boolean isSensitive(Property property) { + return property != null && property.isSensitive(); + } + + public boolean isSensitiveValuePlaceholder(String value) { + return TaskConstants.SENSITIVE_DATA_MASK.equals(value); + } + + public Property copy(Property property) { + if (property == null) { + return null; + } + return Property.builder() + .prop(property.getProp()) + .direct(property.getDirect()) + .type(property.getType()) + .value(property.getValue()) + .sensitive(property.isSensitive()) + .build(); + } + + public List copy(List properties) { + if (CollectionUtils.isEmpty(properties)) { + return Collections.emptyList(); + } + return properties.stream() + .map(PropertySensitiveUtils::copy) + .collect(Collectors.toList()); + } + + public Property maskSensitiveValue(Property property) { + Property maskedProperty = copy(property); + if (isSensitive(maskedProperty) && maskedProperty.getValue() != null) { + maskedProperty.setValue(TaskConstants.SENSITIVE_DATA_MASK); + } + return maskedProperty; + } + + public List maskSensitiveValues(List properties) { + if (CollectionUtils.isEmpty(properties)) { + return Collections.emptyList(); + } + return properties.stream() + .map(PropertySensitiveUtils::maskSensitiveValue) + .collect(Collectors.toList()); + } + + public List mergeSensitiveValuePlaceholders(List submittedProperties, + List existingProperties) { + if (CollectionUtils.isEmpty(submittedProperties)) { + return Collections.emptyList(); + } + Map existingPropertyMap = CollectionUtils.emptyIfNull(existingProperties) + .stream() + .filter(Objects::nonNull) + .filter(property -> property.getProp() != null) + .collect(Collectors.toMap(Property::getProp, Function.identity(), (left, right) -> right)); + + return submittedProperties.stream() + .map(PropertySensitiveUtils::copy) + .peek(property -> mergeSensitiveValuePlaceholder(property, existingPropertyMap)) + .collect(Collectors.toList()); + } + + private void mergeSensitiveValuePlaceholder(Property submittedProperty, Map existingPropertyMap) { + if (!isSensitive(submittedProperty) || !isSensitiveValuePlaceholder(submittedProperty.getValue())) { + return; + } + Property existingProperty = existingPropertyMap.get(submittedProperty.getProp()); + if (existingProperty != null) { + submittedProperty.setValue(existingProperty.getValue()); + } + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverterTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverterTest.java index b1d242c1105a..0af880843670 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverterTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/log/SensitiveDataConverterTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.HashMap; +import java.util.regex.Pattern; import org.junit.jupiter.api.Test; @@ -124,4 +125,39 @@ void testK8SLogMsgConverter() { assertEquals(maskMsg, maskedLog); } + + @Test + void testDynamicMaskPatternCanBeRemoved() { + String sensitiveValue = "task_sensitive_value_remove"; + String maskPattern = Pattern.quote(sensitiveValue); + + try { + SensitiveDataConverter.addDynamicMaskPattern(maskPattern); + assertEquals("password=******", SensitiveDataConverter.maskSensitiveData("password=" + sensitiveValue)); + } finally { + SensitiveDataConverter.removeDynamicMaskPattern(maskPattern); + } + + assertEquals("password=" + sensitiveValue, + SensitiveDataConverter.maskSensitiveData("password=" + sensitiveValue)); + } + + @Test + void testDynamicMaskPatternUsesReferenceCount() { + String sensitiveValue = "task_sensitive_value_ref_count"; + String maskPattern = Pattern.quote(sensitiveValue); + + try { + SensitiveDataConverter.addDynamicMaskPattern(maskPattern); + SensitiveDataConverter.addDynamicMaskPattern(maskPattern); + + SensitiveDataConverter.removeDynamicMaskPattern(maskPattern); + assertEquals("password=******", SensitiveDataConverter.maskSensitiveData("password=" + sensitiveValue)); + } finally { + SensitiveDataConverter.removeDynamicMaskPattern(maskPattern); + } + + assertEquals("password=" + sensitiveValue, + SensitiveDataConverter.maskSensitiveData("password=" + sensitiveValue)); + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/PropertySensitiveUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/PropertySensitiveUtilsTest.java new file mode 100644 index 000000000000..d52ee4aed700 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/PropertySensitiveUtilsTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.utils; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; + +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import com.google.common.collect.Lists; + +class PropertySensitiveUtilsTest { + + @Test + void testDeserializePropertyWithoutSensitiveShouldUseDefaultFalse() { + Property property = JSONUtils.parseObject( + "{\"prop\":\"password\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"secret\"}", + Property.class); + + Assertions.assertNotNull(property); + Assertions.assertFalse(property.isSensitive()); + } + + @Test + void testMaskSensitiveValuesShouldNotMutateSourceProperties() { + Property sensitiveProperty = Property.builder() + .prop("password") + .direct(Direct.IN) + .type(DataType.VARCHAR) + .value("secret") + .sensitive(true) + .build(); + Property normalProperty = new Property("env", Direct.IN, DataType.VARCHAR, "prod"); + + List result = PropertySensitiveUtils.maskSensitiveValues( + Lists.newArrayList(sensitiveProperty, normalProperty)); + + Assertions.assertEquals(TaskConstants.SENSITIVE_DATA_MASK, result.get(0).getValue()); + Assertions.assertTrue(result.get(0).isSensitive()); + Assertions.assertEquals("prod", result.get(1).getValue()); + Assertions.assertFalse(result.get(1).isSensitive()); + Assertions.assertEquals("secret", sensitiveProperty.getValue()); + } + + @Test + void testMergeSensitiveValuePlaceholdersShouldKeepExistingValue() { + Property submittedProperty = Property.builder() + .prop("password") + .direct(Direct.IN) + .type(DataType.VARCHAR) + .value(TaskConstants.SENSITIVE_DATA_MASK) + .sensitive(true) + .build(); + Property existingProperty = Property.builder() + .prop("password") + .direct(Direct.IN) + .type(DataType.VARCHAR) + .value("encrypted-secret") + .sensitive(true) + .build(); + + List result = PropertySensitiveUtils.mergeSensitiveValuePlaceholders( + Lists.newArrayList(submittedProperty), + Lists.newArrayList(existingProperty)); + + Assertions.assertEquals("encrypted-secret", result.get(0).getValue()); + Assertions.assertEquals(TaskConstants.SENSITIVE_DATA_MASK, submittedProperty.getValue()); + } + + @Test + void testMergeSensitiveValuePlaceholdersShouldKeepSubmittedValueWhenNotPlaceholder() { + Property submittedProperty = Property.builder() + .prop("password") + .direct(Direct.IN) + .type(DataType.VARCHAR) + .value("new-secret") + .sensitive(true) + .build(); + Property existingProperty = Property.builder() + .prop("password") + .direct(Direct.IN) + .type(DataType.VARCHAR) + .value("old-secret") + .sensitive(true) + .build(); + + List result = PropertySensitiveUtils.mergeSensitiveValuePlaceholders( + Lists.newArrayList(submittedProperty), + Lists.newArrayList(existingProperty)); + + Assertions.assertEquals("new-secret", result.get(0).getValue()); + } +} diff --git a/dolphinscheduler-ui/src/components/form/fields/custom-parameters.ts b/dolphinscheduler-ui/src/components/form/fields/custom-parameters.ts index 49bb371b5f46..ae1f99a46537 100644 --- a/dolphinscheduler-ui/src/components/form/fields/custom-parameters.ts +++ b/dolphinscheduler-ui/src/components/form/fields/custom-parameters.ts @@ -92,7 +92,7 @@ const getDefaultValue = (children: IJsonItem[]) => { } return } else { - parent[mergedChild.field] = mergedChild.value || null + parent[mergedChild.field] = mergedChild.value ?? null if (mergedChild.validate) ruleParent[mergedChild.field] = formatValidate(mergedChild.validate) } diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index 40e553d4dd1d..46facef6bb42 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -347,6 +347,7 @@ export default { minute: 'Minute', key: 'Key', value: 'Value', + sensitive: 'Sensitive', success: 'Success', delete_cell: 'Delete selected edges and nodes', online_directly: 'Whether to go online the workflow definition', @@ -456,6 +457,7 @@ export default { prop_repeat: 'prop is repeat', value_tips: 'value(optional)', value_required_tips: 'value(required)', + sensitive: 'Sensitive', custom_labels: 'Customized labels', node_selectors: 'Node Selectors', label_repeat: 'repeated label', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index abaa84a34b1a..cf1926a4f751 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -341,6 +341,7 @@ export default { minute: '分', key: '键', value: '值', + sensitive: '敏感', success: '成功', delete_cell: '删除选中的线或节点', online_directly: '是否上线工作流定义', @@ -442,6 +443,7 @@ export default { prop_repeat: 'prop中有重复', value_tips: 'value(选填)', value_required_tips: 'value(必填)', + sensitive: '敏感', custom_labels: '自定义标签', node_selectors: '节点选择器', label_repeat: 'label中有重复', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-custom-params.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-custom-params.ts index 2052f524fe16..457901c7b282 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-custom-params.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-custom-params.ts @@ -44,7 +44,7 @@ export function useCustomParams({ { type: 'input', field: 'prop', - span: 6, + span: 5, class: 'input-param-key', props: { placeholder: t('project.node.prop_tips'), @@ -81,7 +81,7 @@ export function useCustomParams({ { type: 'select', field: 'type', - span: 6, + span: 5, options: TYPE_LIST, value: 'VARCHAR', props: { @@ -91,12 +91,19 @@ export function useCustomParams({ { type: 'input', field: 'value', - span: 6, + span: 5, class: 'input-param-value', props: { placeholder: t('project.node.value_tips'), maxLength: 256 } + }, + { + type: 'checkbox', + field: 'sensitive', + name: t('project.node.sensitive'), + span: 3, + value: false } ] } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts index 218f318db0d0..70d6428a3722 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts @@ -64,6 +64,7 @@ interface ILocalParam { direct?: string type?: string value?: string + sensitive?: boolean } interface ILabel { diff --git a/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag-save-modal.tsx b/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag-save-modal.tsx index 1e39a6d37e4e..e0594a118548 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag-save-modal.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/components/dag/dag-save-modal.tsx @@ -156,7 +156,8 @@ export default defineComponent({ key: param.prop, value: param.value, direct: param.direct, - type: param.type + type: param.type, + sensitive: param.sensitive || false }) ) } @@ -243,7 +244,8 @@ export default defineComponent({ key: '', direct: 'IN', type: 'VARCHAR', - value: '' + value: '', + sensitive: false } }} class='input-global-params' @@ -255,10 +257,11 @@ export default defineComponent({ direct: string type: string value: string + sensitive?: boolean } }) => ( - + - + - + + + + {t('project.dag.sensitive')} + + ) }} diff --git a/dolphinscheduler-ui/src/views/projects/workflow/components/dag/types.ts b/dolphinscheduler-ui/src/views/projects/workflow/components/dag/types.ts index f136fa5be08f..5e7497503eb7 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/components/dag/types.ts +++ b/dolphinscheduler-ui/src/views/projects/workflow/components/dag/types.ts @@ -140,6 +140,7 @@ export interface GlobalParam { direct: string type: string value: string + sensitive?: boolean } export interface SaveForm { diff --git a/dolphinscheduler-ui/src/views/projects/workflow/definition/create/index.tsx b/dolphinscheduler-ui/src/views/projects/workflow/definition/create/index.tsx index 511f830c110d..39f703aef2ea 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/definition/create/index.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/definition/create/index.tsx @@ -59,7 +59,8 @@ export default defineComponent({ prop: p.key, value: p.value, direct: p.direct, - type: p.type + type: p.type, + sensitive: p.sensitive || false } }) diff --git a/dolphinscheduler-ui/src/views/projects/workflow/definition/detail/index.tsx b/dolphinscheduler-ui/src/views/projects/workflow/definition/detail/index.tsx index 2831014c55a6..30ccc0df5b40 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/definition/detail/index.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/definition/detail/index.tsx @@ -85,7 +85,8 @@ export default defineComponent({ prop: p.key, value: p.value, direct: p.direct, - type: p.type + type: p.type, + sensitive: p.sensitive || false } }) diff --git a/dolphinscheduler-ui/src/views/projects/workflow/instance/components/variables-view.tsx b/dolphinscheduler-ui/src/views/projects/workflow/instance/components/variables-view.tsx index dcd37ad03a1f..0bcdf1404f8c 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/instance/components/variables-view.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/instance/components/variables-view.tsx @@ -66,17 +66,23 @@ export default defineComponent({ ctx.emit('copy', text) } + const shouldDisplayParamField = (key: string, taskType: string) => { + if (key === 'sensitive') { + return false + } + return ( + !(taskType === 'SQL' || taskType === 'PROCEDURE') || + (key !== 'direct' && key !== 'type') + ) + } + /** * Copyed text processing */ const rtClipboard = (el: any, taskType: string) => { const arr: Array = [] Object.keys(el).forEach((key) => { - if (taskType === 'SQL' || taskType === 'PROCEDURE') { - if (key !== 'direct' && key !== 'type') { - arr.push(`${key}=${el[key]}`) - } - } else { + if (shouldDisplayParamField(key, taskType)) { arr.push(`${key}=${el[key]}`) } }) @@ -92,21 +98,14 @@ export default defineComponent({ onClick={() => handleCopy(rtClipboard(el, taskType))} > {Object.keys(el).map((key: string) => { - if (taskType === 'SQL' || taskType === 'PROCEDURE') { - return key !== 'direct' && key !== 'type' ? ( - - {key} = {el[key]} - - ) : ( - '' - ) - } else { + if (shouldDisplayParamField(key, taskType)) { return ( {key} = {el[key]} ) } + return '' })} ) @@ -120,7 +119,8 @@ export default defineComponent({ globalParams, localParams, localButton, - handleCopy + handleCopy, + shouldDisplayParamField } }, render() { diff --git a/dolphinscheduler-ui/src/views/projects/workflow/instance/detail/index.tsx b/dolphinscheduler-ui/src/views/projects/workflow/instance/detail/index.tsx index 1957a6fe9120..7e9654067172 100644 --- a/dolphinscheduler-ui/src/views/projects/workflow/instance/detail/index.tsx +++ b/dolphinscheduler-ui/src/views/projects/workflow/instance/detail/index.tsx @@ -82,7 +82,8 @@ export default defineComponent({ prop: p.key, value: p.value, direct: p.direct, - type: p.type + type: p.type, + sensitive: p.sensitive || false } }) diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/executor/PhysicalTaskExecutor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/executor/PhysicalTaskExecutor.java index 7118a099d5f9..e6ac7121f3cd 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/executor/PhysicalTaskExecutor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/executor/PhysicalTaskExecutor.java @@ -23,9 +23,12 @@ import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; +import org.apache.dolphinscheduler.plugin.task.api.log.SensitiveDataConverter; import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogMarkers; import org.apache.dolphinscheduler.plugin.task.api.model.ApplicationInfo; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; +import org.apache.dolphinscheduler.plugin.task.api.utils.PropertySensitiveUtils; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.utils.TaskExecutionContextUtils; import org.apache.dolphinscheduler.server.worker.utils.TenantUtils; @@ -35,7 +38,13 @@ import org.apache.dolphinscheduler.task.executor.TaskExecutorStateMappings; import org.apache.dolphinscheduler.task.executor.events.TaskExecutorRuntimeContextChangedLifecycleEvent; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; + import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -52,6 +61,8 @@ public class PhysicalTaskExecutor extends AbstractTaskExecutor { private final PhysicalTaskPluginFactory physicalTaskPluginFactory; + private final List dynamicMaskPatterns = new ArrayList<>(); + public PhysicalTaskExecutor(final PhysicalTaskExecutorBuilder physicalTaskExecutorBuilder) { super(physicalTaskExecutorBuilder.getTaskExecutionContext(), physicalTaskExecutorBuilder.getTaskExecutorEventBus()); @@ -109,6 +120,7 @@ public void kill() { @Override public void finalizeTask() { clearTaskInstanceWorkingDirectoryIfNeeded(); + clearSensitiveDataMaskPatterns(); } private void clearTaskInstanceWorkingDirectoryIfNeeded() { @@ -120,6 +132,8 @@ private void clearTaskInstanceWorkingDirectoryIfNeeded() { @Override protected void initializeTaskContext() { + registerSensitiveDataMaskPatterns(); + super.initializeTaskContext(); taskExecutionContext.setTaskAppId(String.valueOf(taskExecutionContext.getTaskInstanceId())); @@ -142,6 +156,28 @@ protected void initializeTaskContext() { } + private void registerSensitiveDataMaskPatterns() { + Map prepareParamsMap = taskExecutionContext.getPrepareParamsMap(); + if (MapUtils.isEmpty(prepareParamsMap)) { + return; + } + prepareParamsMap.values().stream() + .filter(PropertySensitiveUtils::isSensitive) + .map(Property::getValue) + .filter(StringUtils::isNotEmpty) + .filter(value -> !PropertySensitiveUtils.isSensitiveValuePlaceholder(value)) + .map(Pattern::quote) + .forEach(maskPattern -> { + SensitiveDataConverter.addDynamicMaskPattern(maskPattern); + dynamicMaskPatterns.add(maskPattern); + }); + } + + private void clearSensitiveDataMaskPatterns() { + dynamicMaskPatterns.forEach(SensitiveDataConverter::removeDynamicMaskPattern); + dynamicMaskPatterns.clear(); + } + @Override public String toString() { return "PhysicalTaskExecutor{" + diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/PhysicalTaskExecutorOperatorImpl.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/PhysicalTaskExecutorOperatorImpl.java index e29ce197c02c..ec9b969670d7 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/PhysicalTaskExecutorOperatorImpl.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/PhysicalTaskExecutorOperatorImpl.java @@ -46,14 +46,15 @@ public class PhysicalTaskExecutorOperatorImpl implements IPhysicalTaskExecutorOp @Override public TaskExecutorDispatchResponse dispatchTask(final TaskExecutorDispatchRequest taskExecutorDispatchRequest) { - log.info("Receive TaskExecutorDispatchResponse: {}", taskExecutorDispatchRequest); final TaskExecutionContext taskExecutionContext = taskExecutorDispatchRequest.getTaskExecutionContext(); + final int taskInstanceId = taskExecutionContext.getTaskInstanceId(); + log.info("Receive TaskExecutorDispatchResponse, taskInstanceId: {}", taskInstanceId); try { physicalTaskEngineDelegator.dispatchLogicTask(taskExecutionContext); - log.info("Handle TaskExecutorDispatchResponse: {} success", taskExecutorDispatchRequest); + log.info("Handle TaskExecutorDispatchResponse, taskInstanceId: {} success", taskInstanceId); return TaskExecutorDispatchResponse.success(); } catch (Throwable throwable) { - log.error("Handle TaskExecutorDispatchResponse: {} failed", taskExecutorDispatchRequest, throwable); + log.error("Handle TaskExecutorDispatchResponse, taskInstanceId: {} failed", taskInstanceId, throwable); return TaskExecutorDispatchResponse.failed(ExceptionUtils.getMessage(throwable)); } } From e8dc15945605ccd18301061fa47eeec95716e0e1 Mon Sep 17 00:00:00 2001 From: luxl Date: Tue, 9 Jun 2026 11:07:22 +0800 Subject: [PATCH 2/3] add test case --- .../SensitiveWorkflowVariableAPITest.java | 179 ++++++++++++++++++ .../workflow/WorkflowDefinitionPage.java | 43 +++++ .../WorkflowDefinitionServiceTest.java | 69 +++++++ .../api/utils/SensitivePropertyUtilsTest.java | 178 +++++++++++++++++ 4 files changed, 469 insertions(+) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/SensitivePropertyUtilsTest.java diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/SensitiveWorkflowVariableAPITest.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/SensitiveWorkflowVariableAPITest.java index 3a136da67e9c..b1c5309aadbc 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/SensitiveWorkflowVariableAPITest.java +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/cases/SensitiveWorkflowVariableAPITest.java @@ -56,6 +56,8 @@ public class SensitiveWorkflowVariableAPITest { private static final String USERNAME = "admin"; private static final String PASSWORD = "dolphinscheduler123"; private static final String SECRET = "Ds_SeCrEt_2026_06_08_Case_A1"; + private static final String GLOBAL_SECRET = "Ds_Global_2026_A1"; + private static final String LOCAL_SECRET = "Ds_Local_2026_B2"; private static final String SENSITIVE_DATA_MASK = "******"; private static String sessionId; @@ -209,6 +211,99 @@ public void testSensitiveGlobalParamUsesRealValueAtRuntimeAndMasksLog() { assertTrue(plainLogContent.contains(SECRET), plainLogContent); } + @Test + @SuppressWarnings("unchecked") + public void testUpdateWorkflowWithPlaceholderKeepsSensitiveValues() { + String projectName = "sensitive-update-project-" + System.currentTimeMillis(); + String workflowName = "sensitive-update-workflow-" + System.currentTimeMillis(); + int expectedSecretLength = GLOBAL_SECRET.length() + LOCAL_SECRET.length(); + + HttpResponse createProjectResponse = projectPage.createProject(loginUser, projectName); + assertTrue(createProjectResponse.getBody().getSuccess()); + long projectCode = ((Number) ((LinkedHashMap) createProjectResponse.getBody().getData()) + .get("code")).longValue(); + + long taskCode = System.currentTimeMillis(); + HttpResponse createWorkflowResponse = workflowDefinitionPage.createWorkflowDefinition( + loginUser, + projectCode, + sensitiveWorkflowWithLocalJson(taskCode, GLOBAL_SECRET, LOCAL_SECRET), + workflowName); + assertTrue(createWorkflowResponse.getBody().getSuccess()); + long workflowDefinitionCode = + ((Number) ((LinkedHashMap) createWorkflowResponse.getBody().getData()).get("code")) + .longValue(); + + HttpResponse queryBeforeUpdateResponse = + workflowDefinitionPage.queryWorkflowDefinitionByCode(loginUser, projectCode, workflowDefinitionCode); + assertTrue(queryBeforeUpdateResponse.getBody().getSuccess()); + String beforeUpdateData = queryBeforeUpdateResponse.getBody().getData().toString(); + assertTrue(beforeUpdateData.contains(SENSITIVE_DATA_MASK)); + assertFalse(beforeUpdateData.contains(GLOBAL_SECRET)); + assertFalse(beforeUpdateData.contains(LOCAL_SECRET)); + + HttpResponse updateWorkflowResponse = workflowDefinitionPage.updateWorkflowDefinition( + loginUser, + projectCode, + workflowDefinitionCode, + sensitiveWorkflowPlaceholderUpdateJson(taskCode), + workflowName); + assertTrue(updateWorkflowResponse.getBody().getSuccess(), + () -> "update failed: " + updateWorkflowResponse.getBody()); + + HttpResponse queryAfterUpdateResponse = + workflowDefinitionPage.queryWorkflowDefinitionByCode(loginUser, projectCode, workflowDefinitionCode); + assertTrue(queryAfterUpdateResponse.getBody().getSuccess()); + String afterUpdateData = queryAfterUpdateResponse.getBody().getData().toString(); + assertTrue(afterUpdateData.contains(SENSITIVE_DATA_MASK)); + assertFalse(afterUpdateData.contains(GLOBAL_SECRET)); + assertFalse(afterUpdateData.contains(LOCAL_SECRET)); + + HttpResponse releaseWorkflowResponse = workflowDefinitionPage.releaseWorkflowDefinition( + loginUser, + projectCode, + workflowDefinitionCode, + ReleaseState.ONLINE); + assertTrue(releaseWorkflowResponse.getBody().getSuccess()); + + HttpResponse startWorkflowResponse = executorPage.startWorkflowInstance( + loginUser, + projectCode, + workflowDefinitionCode, + scheduleTime(), + FailureStrategy.END, + WarningType.NONE); + assertTrue(startWorkflowResponse.getBody().getSuccess()); + List workflowInstanceIds = (List) startWorkflowResponse.getBody().getData(); + assertEquals(1, workflowInstanceIds.size()); + int workflowInstanceId = workflowInstanceIds.get(0); + + Awaitility.await() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted(() -> { + HttpResponse instanceResponse = + workflowInstancePage.queryWorkflowInstanceById(loginUser, projectCode, workflowInstanceId); + assertTrue(instanceResponse.getBody().getSuccess()); + Map workflowInstance = (Map) instanceResponse.getBody().getData(); + assertEquals("SUCCESS", workflowInstance.get("state")); + }); + + HttpResponse taskListResponse = + workflowInstancePage.queryTaskInstanceList(loginUser, projectCode, workflowInstanceId); + assertTrue(taskListResponse.getBody().getSuccess()); + Map taskPage = (Map) taskListResponse.getBody().getData(); + List> taskInstances = (List>) taskPage.get("totalList"); + assertEquals(1, taskInstances.size()); + int taskInstanceId = (int) taskInstances.get(0).get("id"); + + HttpResponse logResponse = workflowInstancePage.queryTaskLog(loginUser, taskInstanceId, 0, 1000); + assertTrue(logResponse.getBody().getSuccess()); + String logContent = ((Map) logResponse.getBody().getData()).get("message").toString(); + assertTrue(logContent.contains(" -> " + expectedSecretLength), logContent); + assertFalse(logContent.contains(GLOBAL_SECRET), logContent); + assertFalse(logContent.contains(LOCAL_SECRET), logContent); + } + private String sensitiveWorkflowJson(long taskCode) { return "{" + "\"taskDefinitionJson\":[{" @@ -288,6 +383,90 @@ private String plainWorkflowJson(long taskCode) { + "}"; } + private String sensitiveWorkflowWithLocalJson(long taskCode, String globalSecret, String localSecret) { + return "{" + + "\"taskDefinitionJson\":[{" + + "\"code\":" + taskCode + "," + + "\"delayTime\":\"0\"," + + "\"description\":\"\"," + + "\"environmentCode\":-1," + + "\"failRetryInterval\":\"1\"," + + "\"failRetryTimes\":\"0\"," + + "\"flag\":\"YES\"," + + "\"name\":\"sensitive_shell\"," + + "\"taskParams\":{\"localParams\":[{\"prop\":\"localVar\",\"direct\":\"IN\",\"type\":\"VARCHAR\"," + + "\"value\":\"" + localSecret + "\",\"sensitive\":true}]," + + "\"rawScript\":\"echo -n ${var}${localVar} | wc -c\"," + + "\"resourceList\":[]}," + + "\"taskPriority\":\"MEDIUM\"," + + "\"taskType\":\"SHELL\"," + + "\"timeout\":0," + + "\"timeoutFlag\":\"CLOSE\"," + + "\"timeoutNotifyStrategy\":\"\"," + + "\"workerGroup\":\"default\"," + + "\"cpuQuota\":-1," + + "\"memoryMax\":-1," + + "\"taskExecuteType\":\"BATCH\"" + + "}]," + + "\"taskRelationJson\":[{" + + "\"name\":\"\"," + + "\"preTaskCode\":0," + + "\"preTaskVersion\":0," + + "\"postTaskCode\":" + taskCode + "," + + "\"postTaskVersion\":0," + + "\"conditionType\":\"NONE\"," + + "\"conditionParams\":{}" + + "}]," + + "\"executionType\":\"PARALLEL\"," + + "\"description\":\"\"," + + "\"globalParams\":[{\"prop\":\"var\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"" + + globalSecret + "\",\"sensitive\":true}]," + + "\"timeout\":0" + + "}"; + } + + private String sensitiveWorkflowPlaceholderUpdateJson(long taskCode) { + return "{" + + "\"taskDefinitionJson\":[{" + + "\"code\":" + taskCode + "," + + "\"delayTime\":\"0\"," + + "\"description\":\"updated\"," + + "\"environmentCode\":-1," + + "\"failRetryInterval\":\"1\"," + + "\"failRetryTimes\":\"0\"," + + "\"flag\":\"YES\"," + + "\"name\":\"sensitive_shell\"," + + "\"taskParams\":{\"localParams\":[{\"prop\":\"localVar\",\"direct\":\"IN\",\"type\":\"VARCHAR\"," + + "\"value\":\"" + SENSITIVE_DATA_MASK + "\",\"sensitive\":true}]," + + "\"rawScript\":\"echo -n ${var}${localVar} | wc -c\"," + + "\"resourceList\":[]}," + + "\"taskPriority\":\"MEDIUM\"," + + "\"taskType\":\"SHELL\"," + + "\"timeout\":0," + + "\"timeoutFlag\":\"CLOSE\"," + + "\"timeoutNotifyStrategy\":\"\"," + + "\"workerGroup\":\"default\"," + + "\"cpuQuota\":-1," + + "\"memoryMax\":-1," + + "\"taskExecuteType\":\"BATCH\"" + + "}]," + + "\"taskRelationJson\":[{" + + "\"name\":\"\"," + + "\"preTaskCode\":0," + + "\"preTaskVersion\":0," + + "\"postTaskCode\":" + taskCode + "," + + "\"postTaskVersion\":0," + + "\"conditionType\":\"NONE\"," + + "\"conditionParams\":{}" + + "}]," + + "\"executionType\":\"PARALLEL\"," + + "\"description\":\"updated\"," + + "\"globalParams\":[{\"prop\":\"var\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"" + + SENSITIVE_DATA_MASK + "\",\"sensitive\":true}]," + + "\"timeout\":0" + + "}"; + } + private String sensitiveStartParams() { return "[{\"prop\":\"var\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"" + SENSITIVE_DATA_MASK + "\",\"sensitive\":true}]"; diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/workflow/WorkflowDefinitionPage.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/workflow/WorkflowDefinitionPage.java index 2d82bd2b6938..83327495dfa4 100644 --- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/workflow/WorkflowDefinitionPage.java +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api/test/pages/workflow/WorkflowDefinitionPage.java @@ -136,6 +136,49 @@ public HttpResponse queryWorkflowDefinitionList(User loginUser, long projectCode return requestClient.get(url, headers, params); } + public HttpResponse updateWorkflowDefinition(User loginUser, + long projectCode, + long workflowDefinitionCode, + String jsonContent, + String workflowDefinitionName) { + return updateWorkflowDefinition(loginUser, projectCode, workflowDefinitionCode, jsonContent, + workflowDefinitionName, null); + } + + public HttpResponse updateWorkflowDefinition(User loginUser, + long projectCode, + long workflowDefinitionCode, + String jsonContent, + String workflowDefinitionName, + ReleaseState releaseState) { + Map params = new HashMap<>(); + params.put("loginUser", loginUser); + + Map fileContentMap = JSONUtils.parseObject(jsonContent, new TypeReference<>() { + }); + if (fileContentMap == null) { + throw new RuntimeException("file content parse error"); + } + fileContentMap.replaceAll((key, value) -> { + if (value instanceof List) { + return JSONUtils.toJsonString(value); + } + return value; + }); + params.putAll(fileContentMap); + params.put("name", workflowDefinitionName); + params.put("code", workflowDefinitionCode); + if (releaseState != null) { + params.put("releaseState", releaseState); + } + + Map headers = new HashMap<>(); + headers.put(Constants.SESSION_ID_KEY, sessionId); + RequestClient requestClient = new RequestClient(); + String url = String.format("/projects/%s/workflow-definition/%s", projectCode, workflowDefinitionCode); + return requestClient.put(url, headers, params); + } + public HttpResponse releaseWorkflowDefinition(User loginUser, long projectCode, long code, ReleaseState releaseState) { Map params = new HashMap<>(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java index 133e1c33e2b7..97cd10322abd 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkflowDefinitionServiceTest.java @@ -41,6 +41,7 @@ import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; import org.apache.dolphinscheduler.api.service.impl.WorkflowDefinitionServiceImpl; import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.SensitivePropertyUtils; import org.apache.dolphinscheduler.api.validator.GlobalParamsValidator; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.FailureStrategy; @@ -399,6 +400,42 @@ public void testQueryWorkflowDefinitionListPaging() { Assertions.assertEquals(pd1.getModifyBy(), user1); } + @Test + public void testQueryWorkflowDefinitionListPagingShouldMaskSensitiveGlobalParams() { + user.setId(1); + doNothing().when(projectService).checkProjectAndAuthThrowException(user, projectCode, WORKFLOW_DEFINITION); + + WorkflowDefinition workflowDefinition = WorkflowDefinition.builder() + .version(1) + .code(1L) + .build(); + try (MockedStatic ignored = mockEncryptionEnabled()) { + workflowDefinition.setGlobalParams(sensitiveGlobalParams(PasswordUtils.encodePassword("paging_secret"))); + } + + PageListingResult pageListingResult = PageListingResult.builder() + .records(Collections.singletonList(workflowDefinition)) + .currentPage(1) + .pageSize(10) + .totalCount(1) + .build(); + when(workflowDefinitionDao.listingWorkflowDefinition(eq(0), eq(10), eq(""), eq(1), eq(projectCode))) + .thenReturn(pageListingResult); + when(userDao.queryUserWithWorkflowDefinitionCode(Collections.singletonList(1L))) + .thenReturn(Collections.emptyList()); + when(schedulerService.queryScheduleByWorkflowDefinitionCodes(Collections.singletonList(1L))) + .thenReturn(Collections.emptyList()); + + try (MockedStatic ignored = mockEncryptionEnabled()) { + PageInfo pageInfo = workflowDefinitionService.queryWorkflowDefinitionListPaging( + user, projectCode, "", "", 1, 0, 10); + + Assertions.assertEquals(SENSITIVE_DATA_MASK, + pageInfo.getTotalList().get(0).getGlobalParamList().get(0).getValue()); + Assertions.assertFalse(JSONUtils.toJsonString(pageInfo).contains("paging_secret")); + } + } + @Test public void testQueryWorkflowDefinitionByCode() { Project project = getProject(projectCode); @@ -427,6 +464,38 @@ public void testQueryWorkflowDefinitionByCode() { Assertions.assertNotNull(successRes); } + @Test + public void testQueryWorkflowDefinitionByCodeShouldMaskSensitiveParams() { + Project project = getProject(projectCode); + when(projectDao.queryByCode(projectCode)).thenReturn(project); + Mockito.doNothing().when(projectService) + .checkProjectAndAuthThrowException(user, project, WORKFLOW_DEFINITION); + + WorkflowDefinition workflowDefinition = getWorkflowDefinition(); + TaskDefinition taskDefinition = new TaskDefinition(); + taskDefinition.setCode(123456789L); + try (MockedStatic ignored = mockEncryptionEnabled()) { + workflowDefinition.setGlobalParams(sensitiveGlobalParams(PasswordUtils.encodePassword("global_secret"))); + taskDefinition.setTaskParams(sensitiveTaskParams(PasswordUtils.encodePassword("local_secret"))); + } + when(workflowDefinitionDao.queryByCode(46L)).thenReturn(Optional.of(workflowDefinition)); + DagData dagData = new DagData(workflowDefinition, Collections.emptyList(), + Collections.singletonList(taskDefinition)); + when(processService.genDagData(any())).thenReturn(dagData); + + try (MockedStatic ignored = mockEncryptionEnabled()) { + DagData result = workflowDefinitionService.queryWorkflowDefinitionByCode(user, projectCode, 46L); + + Assertions.assertEquals(SENSITIVE_DATA_MASK, + result.getWorkflowDefinition().getGlobalParamList().get(0).getValue()); + List localParams = + SensitivePropertyUtils.getLocalParams(result.getTaskDefinitionList().get(0).getTaskParams()); + Assertions.assertEquals(SENSITIVE_DATA_MASK, localParams.get(0).getValue()); + Assertions.assertFalse(JSONUtils.toJsonString(result).contains("global_secret")); + Assertions.assertFalse(JSONUtils.toJsonString(result).contains("local_secret")); + } + } + @Test public void testQueryWorkflowDefinitionByName() { Project project = getProject(projectCode); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/SensitivePropertyUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/SensitivePropertyUtilsTest.java new file mode 100644 index 000000000000..d80615c17eb6 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/SensitivePropertyUtilsTest.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.utils; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.dao.entity.DagData; +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; +import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition; +import org.apache.dolphinscheduler.plugin.datasource.api.constants.DataSourceConstants; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.utils.GlobalParameterUtils; + +import java.util.Collections; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import com.google.common.collect.Lists; + +class SensitivePropertyUtilsTest { + + @Test + void testEncryptSensitiveValuesShouldEncryptOnlySensitiveNonPlaceholder() { + List properties = Lists.newArrayList( + sensitiveProperty("secret_a"), + new Property("env", Direct.IN, DataType.VARCHAR, "prod"), + sensitiveProperty(TaskConstants.SENSITIVE_DATA_MASK)); + + try (MockedStatic ignored = mockEncryptionEnabled()) { + List encrypted = SensitivePropertyUtils.encryptSensitiveValues(properties); + + Assertions.assertNotEquals("secret_a", encrypted.get(0).getValue()); + Assertions.assertEquals("prod", encrypted.get(1).getValue()); + Assertions.assertEquals(TaskConstants.SENSITIVE_DATA_MASK, encrypted.get(2).getValue()); + Assertions.assertEquals("secret_a", PasswordUtils.decodePassword(encrypted.get(0).getValue())); + } + } + + @Test + void testMergeAndEncryptGlobalParamsShouldAvoidDoubleEncryption() { + String existingGlobalParams; + try (MockedStatic ignored = mockEncryptionEnabled()) { + existingGlobalParams = GlobalParameterUtils.serializeGlobalParameter( + SensitivePropertyUtils.encryptSensitiveValues( + Lists.newArrayList(sensitiveProperty("old_secret")))); + } + + String submittedGlobalParams = GlobalParameterUtils.serializeGlobalParameter( + Lists.newArrayList(sensitiveProperty(TaskConstants.SENSITIVE_DATA_MASK))); + + try (MockedStatic ignored = mockEncryptionEnabled()) { + String merged = SensitivePropertyUtils.mergeAndEncryptGlobalParams(submittedGlobalParams, + existingGlobalParams); + Property saved = GlobalParameterUtils.deserializeGlobalParameter(merged).get(0); + + Assertions.assertNotEquals(TaskConstants.SENSITIVE_DATA_MASK, saved.getValue()); + Assertions.assertEquals("old_secret", PasswordUtils.decodePassword(saved.getValue())); + } + } + + @Test + void testDecryptAndMaskSensitiveValuesShouldReturnPlaceholder() { + String encryptedValue; + try (MockedStatic ignored = mockEncryptionEnabled()) { + encryptedValue = PasswordUtils.encodePassword("db_secret"); + } + + List properties = Lists.newArrayList(sensitiveProperty(encryptedValue)); + + try (MockedStatic ignored = mockEncryptionEnabled()) { + List masked = SensitivePropertyUtils.decryptAndMaskSensitiveValues(properties); + + Assertions.assertEquals(TaskConstants.SENSITIVE_DATA_MASK, masked.get(0).getValue()); + Assertions.assertTrue(masked.get(0).isSensitive()); + } + } + + @Test + void testDecryptAndMaskDagDataShouldMaskGlobalAndLocalParams() { + WorkflowDefinition workflowDefinition = new WorkflowDefinition(); + TaskDefinition taskDefinition = new TaskDefinition(); + String encryptedGlobal; + String encryptedLocal; + try (MockedStatic ignored = mockEncryptionEnabled()) { + encryptedGlobal = PasswordUtils.encodePassword("global_secret"); + encryptedLocal = PasswordUtils.encodePassword("local_secret"); + } + + workflowDefinition.setGlobalParams(GlobalParameterUtils.serializeGlobalParameter( + Lists.newArrayList(sensitiveProperty(encryptedGlobal)))); + taskDefinition.setTaskParams("{\"localParams\":[{\"prop\":\"secret_local\",\"direct\":\"IN\"," + + "\"type\":\"VARCHAR\",\"value\":\"" + encryptedLocal + "\",\"sensitive\":true}]," + + "\"rawScript\":\"echo 1\"}"); + + DagData dagData = new DagData(workflowDefinition, Collections.emptyList(), + Collections.singletonList(taskDefinition)); + + try (MockedStatic ignored = mockEncryptionEnabled()) { + SensitivePropertyUtils.decryptAndMaskDagData(dagData); + } + + Property maskedGlobal = workflowDefinition.getGlobalParamList().get(0); + Property maskedLocal = SensitivePropertyUtils.getLocalParams(taskDefinition.getTaskParams()).get(0); + Assertions.assertEquals(TaskConstants.SENSITIVE_DATA_MASK, maskedGlobal.getValue()); + Assertions.assertEquals(TaskConstants.SENSITIVE_DATA_MASK, maskedLocal.getValue()); + Assertions.assertFalse(workflowDefinition.getGlobalParams().contains("global_secret")); + Assertions.assertFalse(taskDefinition.getTaskParams().contains("local_secret")); + } + + @Test + void testMergeAndEncryptLocalParamsInTaskParamsShouldKeepExistingValue() { + String existingTaskParams; + try (MockedStatic ignored = mockEncryptionEnabled()) { + existingTaskParams = "{\"localParams\":[{\"prop\":\"secret_local\",\"direct\":\"IN\"," + + "\"type\":\"VARCHAR\",\"value\":\"" + + PasswordUtils.encodePassword("old_local_secret") + + "\",\"sensitive\":true}],\"rawScript\":\"echo 1\"}"; + } + + String submittedTaskParams = "{\"localParams\":[{\"prop\":\"secret_local\",\"direct\":\"IN\"," + + "\"type\":\"VARCHAR\",\"value\":\"" + TaskConstants.SENSITIVE_DATA_MASK + + "\",\"sensitive\":true}],\"rawScript\":\"echo 1\"}"; + + try (MockedStatic ignored = mockEncryptionEnabled()) { + String merged = SensitivePropertyUtils.mergeAndEncryptLocalParamsInTaskParams(submittedTaskParams, + existingTaskParams); + Property saved = SensitivePropertyUtils.getLocalParams(merged).get(0); + + Assertions.assertNotEquals(TaskConstants.SENSITIVE_DATA_MASK, saved.getValue()); + Assertions.assertEquals("old_local_secret", PasswordUtils.decodePassword(saved.getValue())); + Assertions.assertEquals("echo 1", JSONUtils.getNodeString(merged, "rawScript")); + } + } + + private Property sensitiveProperty(String value) { + return Property.builder() + .prop("secret") + .direct(Direct.IN) + .type(DataType.VARCHAR) + .value(value) + .sensitive(true) + .build(); + } + + private MockedStatic mockEncryptionEnabled() { + MockedStatic propertyUtilsMockedStatic = Mockito.mockStatic(PropertyUtils.class); + propertyUtilsMockedStatic.when(() -> PropertyUtils.getBoolean( + DataSourceConstants.DATASOURCE_ENCRYPTION_ENABLE, false)).thenReturn(Boolean.TRUE); + propertyUtilsMockedStatic.when(() -> PropertyUtils.getString( + DataSourceConstants.DATASOURCE_ENCRYPTION_SALT, + DataSourceConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT)) + .thenReturn(DataSourceConstants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); + return propertyUtilsMockedStatic; + } +} From 9f87226619e9e5d2c3051d8639c061423ca97a89 Mon Sep 17 00:00:00 2001 From: luxl Date: Tue, 9 Jun 2026 15:21:26 +0800 Subject: [PATCH 3/3] Explain that TaskExecutorDispatchRequest.toString() may include decrypted sensitive values in prepareParamsMap. --- .../server/worker/rpc/PhysicalTaskExecutorOperatorImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/PhysicalTaskExecutorOperatorImpl.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/PhysicalTaskExecutorOperatorImpl.java index ec9b969670d7..be0309c64157 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/PhysicalTaskExecutorOperatorImpl.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/PhysicalTaskExecutorOperatorImpl.java @@ -48,6 +48,7 @@ public class PhysicalTaskExecutorOperatorImpl implements IPhysicalTaskExecutorOp public TaskExecutorDispatchResponse dispatchTask(final TaskExecutorDispatchRequest taskExecutorDispatchRequest) { final TaskExecutionContext taskExecutionContext = taskExecutorDispatchRequest.getTaskExecutionContext(); final int taskInstanceId = taskExecutionContext.getTaskInstanceId(); + // Do not log the full dispatch request: prepareParamsMap may contain decrypted sensitive values. log.info("Receive TaskExecutorDispatchResponse, taskInstanceId: {}", taskInstanceId); try { physicalTaskEngineDelegator.dispatchLogicTask(taskExecutionContext);