Browse Source

[Feature][Datax Task]Datax task plugin support custom parameters (#10950)

* datax task plugin support custom parameters
3.1.0-release
zhuxt2015 2 years ago committed by GitHub
parent
commit
7196b125d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 13
      dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
  2. 4
      dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts

13
dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java

@ -82,6 +82,8 @@ public class DataxTask extends AbstractTaskExecutor {
* jvm parameters * jvm parameters
*/ */
public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" "; public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" ";
public static final String CUSTOM_PARAM = " -D%s=%s";
/** /**
* python process(datax only supports version 2.7 by default) * python process(datax only supports version 2.7 by default)
*/ */
@ -392,6 +394,8 @@ public class DataxTask extends AbstractTaskExecutor {
sbr.append(DATAX_PATH); sbr.append(DATAX_PATH);
sbr.append(" "); sbr.append(" ");
sbr.append(loadJvmEnv(dataXParameters)); sbr.append(loadJvmEnv(dataXParameters));
sbr.append(addCustomParameters(paramsMap));
sbr.append(" ");
sbr.append(jobConfigFilePath); sbr.append(jobConfigFilePath);
// replace placeholder // replace placeholder
@ -414,6 +418,15 @@ public class DataxTask extends AbstractTaskExecutor {
return fileName; return fileName;
} }
private StringBuilder addCustomParameters(Map<String, Property> paramsMap) {
StringBuilder customParameters = new StringBuilder("-p\"");
for (Map.Entry<String, Property> entry : paramsMap.entrySet()) {
customParameters.append(String.format(CUSTOM_PARAM, entry.getKey(), entry.getValue().getValue()));
}
customParameters.append("\"");
return customParameters;
}
public String getPythonCommand() { public String getPythonCommand() {
String pythonHome = System.getenv("PYTHON_HOME"); String pythonHome = System.getenv("PYTHON_HOME");
return getPythonCommand(pythonHome); return getPythonCommand(pythonHome);

4
dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts

@ -20,6 +20,7 @@ import type { IJsonItem } from '../types'
import { find } from 'lodash' import { find } from 'lodash'
import { TypeReq } from '@/service/modules/data-source/types' import { TypeReq } from '@/service/modules/data-source/types'
import { queryDataSourceList } from '@/service/modules/data-source' import { queryDataSourceList } from '@/service/modules/data-source'
import {useCustomParams} from "."
export function useDataX(model: { [field: string]: any }): IJsonItem[] { export function useDataX(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n() const { t } = useI18n()
@ -443,6 +444,7 @@ export function useDataX(model: { [field: string]: any }): IJsonItem[] {
span: 12, span: 12,
options: memoryLimitOptions, options: memoryLimitOptions,
value: 1 value: 1
} },
...useCustomParams({ model, field: 'localParams', isSimple: true })
] ]
} }

Loading…
Cancel
Save