Browse Source

[Feature-10837][Task Plugin] Sqoop Task Add --split-by Parameter (#10960)

* [Feature-10837][Task Plugin] Sqoop Task Add --split-by Parameter

* checkstyle
3.1.0-release
Assert 2 years ago committed by GitHub
parent
commit
cdfe115247
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java
  2. 4
      dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java
  3. 30
      dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java
  4. 2
      dolphinscheduler-ui/src/locales/en_US/project.ts
  5. 2
      dolphinscheduler-ui/src/locales/zh_CN/project.ts
  6. 13
      dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sqoop.ts
  7. 1
      dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
  8. 1
      dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sqoop.ts
  9. 1
      dolphinscheduler-ui/src/views/projects/task/components/node/types.ts

2
dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java

@ -30,6 +30,8 @@ public final class SqoopConstants {
public static final String LINES_TERMINATED_BY = "--lines-terminated-by"; public static final String LINES_TERMINATED_BY = "--lines-terminated-by";
public static final String FIELD_NULL_PLACEHOLDER = "--null-non-string 'NULL' --null-string 'NULL'"; public static final String FIELD_NULL_PLACEHOLDER = "--null-non-string 'NULL' --null-string 'NULL'";
public static final String SPLIT_BY = "--split-by";
//sqoop db //sqoop db
public static final String DB_CONNECT = "--connect"; public static final String DB_CONNECT = "--connect";
public static final String DB_USERNAME = "--username"; public static final String DB_USERNAME = "--username";

4
dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java

@ -79,6 +79,10 @@ public class CommonGenerator {
if (sqoopParameters.getConcurrency() > 0) { if (sqoopParameters.getConcurrency() > 0) {
commonSb.append(SPACE).append(SqoopConstants.SQOOP_PARALLELISM) commonSb.append(SPACE).append(SqoopConstants.SQOOP_PARALLELISM)
.append(SPACE).append(sqoopParameters.getConcurrency()); .append(SPACE).append(sqoopParameters.getConcurrency());
if (sqoopParameters.getConcurrency() > 1) {
commonSb.append(SPACE).append(SqoopConstants.SPLIT_BY)
.append(SPACE).append(sqoopParameters.getSplitBy());
}
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(String.format("Sqoop task general param build failed: [%s]", e.getMessage())); logger.error(String.format("Sqoop task general param build failed: [%s]", e.getMessage()));

30
dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java

@ -35,7 +35,7 @@ import java.util.Objects;
/** /**
* sqoop parameters * sqoop parameters
*/ */
public class SqoopParameters extends AbstractParameters { public class SqoopParameters extends AbstractParameters {
/** /**
* sqoop job type: * sqoop job type:
@ -62,6 +62,10 @@ public class SqoopParameters extends AbstractParameters {
* concurrency * concurrency
*/ */
private int concurrency; private int concurrency;
/**
* split by
*/
private String splitBy;
/** /**
* source type * source type
*/ */
@ -105,6 +109,14 @@ public class SqoopParameters extends AbstractParameters {
this.concurrency = concurrency; this.concurrency = concurrency;
} }
public String getSplitBy() {
return splitBy;
}
public void setSplitBy(String splitBy) {
this.splitBy = splitBy;
}
public String getSourceType() { public String getSourceType() {
return sourceType; return sourceType;
} }
@ -188,16 +200,16 @@ public class SqoopParameters extends AbstractParameters {
if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) { if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) {
sqoopParamsCheck = StringUtils.isEmpty(customShell) sqoopParamsCheck = StringUtils.isEmpty(customShell)
&& StringUtils.isNotEmpty(modelType) && StringUtils.isNotEmpty(modelType)
&& StringUtils.isNotEmpty(jobName) && StringUtils.isNotEmpty(jobName)
&& concurrency != 0 && concurrency != 0
&& StringUtils.isNotEmpty(sourceType) && StringUtils.isNotEmpty(sourceType)
&& StringUtils.isNotEmpty(targetType) && StringUtils.isNotEmpty(targetType)
&& StringUtils.isNotEmpty(sourceParams) && StringUtils.isNotEmpty(sourceParams)
&& StringUtils.isNotEmpty(targetParams); && StringUtils.isNotEmpty(targetParams);
} else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) { } else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) {
sqoopParamsCheck = StringUtils.isNotEmpty(customShell) sqoopParamsCheck = StringUtils.isNotEmpty(customShell)
&& StringUtils.isEmpty(jobName); && StringUtils.isEmpty(jobName);
} }
return sqoopParamsCheck; return sqoopParamsCheck;

2
dolphinscheduler-ui/src/locales/en_US/project.ts

@ -497,6 +497,8 @@ export default {
allow_insert: 'AllowInsert', allow_insert: 'AllowInsert',
concurrency: 'Concurrency', concurrency: 'Concurrency',
concurrency_tips: 'Please enter Concurrency', concurrency_tips: 'Please enter Concurrency',
concurrency_column: 'Concurrency Column',
concurrency_column_tips: 'Please enter Concurrency Column',
sea_tunnel_master: 'Master', sea_tunnel_master: 'Master',
sea_tunnel_master_url: 'Master URL', sea_tunnel_master_url: 'Master URL',
sea_tunnel_queue: 'Queue', sea_tunnel_queue: 'Queue',

2
dolphinscheduler-ui/src/locales/zh_CN/project.ts

@ -491,6 +491,8 @@ export default {
allow_insert: '无更新便插入', allow_insert: '无更新便插入',
concurrency: '并发度', concurrency: '并发度',
concurrency_tips: '请输入并发度', concurrency_tips: '请输入并发度',
concurrency_column: '并发键',
concurrency_column_tips: '请输入并发键',
sea_tunnel_master: 'Master', sea_tunnel_master: 'Master',
sea_tunnel_master_url: 'Master URL', sea_tunnel_master_url: 'Master URL',
sea_tunnel_queue: '队列', sea_tunnel_queue: '队列',

13
dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sqoop.ts

@ -24,6 +24,8 @@ export function useSqoop(model: { [field: string]: any }): IJsonItem[] {
const customSpan = computed(() => (model.isCustomTask ? 24 : 0)) const customSpan = computed(() => (model.isCustomTask ? 24 : 0))
const unCustomSpan = computed(() => (model.isCustomTask ? 0 : 24)) const unCustomSpan = computed(() => (model.isCustomTask ? 0 : 24))
const unCustomHalfSpan = computed(() => (model.isCustomTask ? 0 : 12))
return [ return [
{ {
type: 'switch', type: 'switch',
@ -75,12 +77,21 @@ export function useSqoop(model: { [field: string]: any }): IJsonItem[] {
type: 'input-number', type: 'input-number',
field: 'concurrency', field: 'concurrency',
name: t('project.node.concurrency'), name: t('project.node.concurrency'),
span: unCustomSpan, span: unCustomHalfSpan,
props: { props: {
placeholder: t('project.node.concurrency_tips'), placeholder: t('project.node.concurrency_tips'),
min: 1 min: 1
} }
}, },
{
type: 'input',
field: 'splitBy',
name: t('project.node.concurrency_column'),
span: unCustomHalfSpan,
props: {
placeholder: t('project.node.concurrency_column_tips')
}
},
{ {
type: 'editor', type: 'editor',
field: 'customShell', field: 'customShell',

1
dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts

@ -88,6 +88,7 @@ export function formatParams(data: INodeData): {
taskParams.hadoopCustomParams = data.hadoopCustomParams taskParams.hadoopCustomParams = data.hadoopCustomParams
taskParams.sqoopAdvancedParams = data.sqoopAdvancedParams taskParams.sqoopAdvancedParams = data.sqoopAdvancedParams
taskParams.concurrency = data.concurrency taskParams.concurrency = data.concurrency
taskParams.splitBy = data.splitBy
taskParams.modelType = data.modelType taskParams.modelType = data.modelType
taskParams.sourceType = data.sourceType taskParams.sourceType = data.sourceType
taskParams.targetType = data.targetType taskParams.targetType = data.targetType

1
dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sqoop.ts

@ -65,6 +65,7 @@ export function useSqoop({
targetHiveDropDelimiter: false, targetHiveDropDelimiter: false,
targetHiveOverWrite: true, targetHiveOverWrite: true,
concurrency: 1, concurrency: 1,
splitBy: '',
timeoutNotifyStrategy: ['WARN'] timeoutNotifyStrategy: ['WARN']
} as INodeData) } as INodeData)

1
dolphinscheduler-ui/src/views/projects/task/components/node/types.ts

@ -265,6 +265,7 @@ interface ITaskParams {
hadoopCustomParams?: ILocalParam[] hadoopCustomParams?: ILocalParam[]
sqoopAdvancedParams?: ILocalParam[] sqoopAdvancedParams?: ILocalParam[]
concurrency?: number concurrency?: number
splitBy?: string
modelType?: ModelType modelType?: ModelType
sourceType?: SourceType sourceType?: SourceType
targetType?: SourceType targetType?: SourceType

Loading…
Cancel
Save