From 7196b125d9a6612f2f175c2190d7758a09d7e0b9 Mon Sep 17 00:00:00 2001 From: zhuxt2015 <594754793@qq.com> Date: Fri, 15 Jul 2022 09:27:23 +0800 Subject: [PATCH] [Feature][Datax Task]Datax task plugin support custom parameters (#10950) * datax task plugin support custom parameters --- .../plugin/task/datax/DataxTask.java | 13 +++++++++++++ .../task/components/node/fields/use-datax.ts | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java index 49ec953725..b8801becbd 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java @@ -82,6 +82,8 @@ public class DataxTask extends AbstractTaskExecutor { * jvm parameters */ public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" "; + + public static final String CUSTOM_PARAM = " -D%s=%s"; /** * python process(datax only supports version 2.7 by default) */ @@ -392,6 +394,8 @@ public class DataxTask extends AbstractTaskExecutor { sbr.append(DATAX_PATH); sbr.append(" "); sbr.append(loadJvmEnv(dataXParameters)); + sbr.append(addCustomParameters(paramsMap)); + sbr.append(" "); sbr.append(jobConfigFilePath); // replace placeholder @@ -414,6 +418,15 @@ public class DataxTask extends AbstractTaskExecutor { return fileName; } + private StringBuilder addCustomParameters(Map paramsMap) { + StringBuilder customParameters = new StringBuilder("-p\""); + for (Map.Entry entry : paramsMap.entrySet()) { + customParameters.append(String.format(CUSTOM_PARAM, entry.getKey(), entry.getValue().getValue())); + } + customParameters.append("\""); + return customParameters; + } + public String getPythonCommand() { String pythonHome = System.getenv("PYTHON_HOME"); return getPythonCommand(pythonHome); diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts index ef80e05583..866b60f2cc 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts @@ -20,6 +20,7 @@ import type { IJsonItem } from '../types' import { find } from 'lodash' import { TypeReq } from '@/service/modules/data-source/types' import { queryDataSourceList } from '@/service/modules/data-source' +import {useCustomParams} from "." export function useDataX(model: { [field: string]: any }): IJsonItem[] { const { t } = useI18n() @@ -443,6 +444,7 @@ export function useDataX(model: { [field: string]: any }): IJsonItem[] { span: 12, options: memoryLimitOptions, value: 1 - } + }, + ...useCustomParams({ model, field: 'localParams', isSimple: true }) ] }