Browse Source

[Feature][UI Next] Add SQOOP into task. (#8494)

3.0.0/version-upgrade
Amy0104 3 years ago committed by GitHub
parent
commit
922283ee9c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      dolphinscheduler-ui-next/src/components/form/fields/get-field.ts
  2. 3
      dolphinscheduler-ui-next/src/components/form/types.ts
  3. 61
      dolphinscheduler-ui-next/src/locales/modules/en_US.ts
  4. 61
      dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts
  5. 4
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/index.ts
  6. 192
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-custom-params.ts
  7. 2
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource.ts
  8. 52
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-flink.ts
  9. 46
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-http.ts
  10. 52
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-mr.ts
  11. 108
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-shell.ts
  12. 50
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-spark.ts
  13. 85
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-datasource.ts
  14. 282
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-source-type.ts
  15. 384
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-target-type.ts
  16. 121
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop.ts
  17. 149
      dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts
  18. 32
      dolphinscheduler-ui-next/src/views/projects/task/components/node/index.module.scss
  19. 1
      dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-shell.ts
  20. 101
      dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-sqoop.ts
  21. 136
      dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts
  22. 10
      dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts

3
dolphinscheduler-ui-next/src/components/form/fields/get-field.ts

@ -26,6 +26,9 @@ const getField = (
) => {
const { type = 'input' } = item
const renderTypeName = `render${upperFirst(camelCase(type))}`
if (type === 'custom') {
return item.widget || null
}
// TODO Support other widgets later
if (type === 'custom-parameters') {
let fieldRules: { [key: string]: FormItemRule }[] = []

3
dolphinscheduler-ui-next/src/components/form/types.ts

@ -35,6 +35,7 @@ type IType =
| 'checkbox'
| 'tree-select'
| 'multi-input'
| 'custom'
interface IOption extends SelectOption, TreeSelectOption {
label: string
@ -46,6 +47,7 @@ interface IFormItem {
label?: string
widget: any
span?: number | Ref<number>
type?: 'custom'
}
interface IMeta extends Omit<FormProps, 'model'> {
@ -65,6 +67,7 @@ interface IJsonItem {
children?: IJsonItem[]
slots?: object
span?: number | Ref<number>
widget?: any
}
export {

61
dolphinscheduler-ui-next/src/locales/modules/en_US.ts

@ -674,7 +674,7 @@ const project = {
target_task_name: 'Target Task Name',
target_task_name_tips: 'Please enter the Pigeon task name',
datasource_type: 'Datasource types',
datasource: 'Datasource instances',
datasource_instances: 'Datasource instances',
sql_type: 'SQL Type',
sql_type_query: 'Query',
sql_type_non_query: 'Non Query',
@ -688,7 +688,64 @@ const project = {
start: 'Start',
edit: 'Edit',
copy: 'Copy',
delete: 'Delete'
delete: 'Delete',
custom_job: 'Custom Job',
custom_script: 'Custom Script',
sqoop_job_name: 'Job Name',
sqoop_job_name_tips: 'Please enter Job Name(required)',
direct: 'Direct',
hadoop_custom_params: 'Hadoop Params',
sqoop_advanced_parameters: 'Sqoop Advanced Parameters',
data_source: 'Data Source',
type: 'Type',
datasource: 'Datasource',
datasource_tips: 'Please select the datasource',
model_type: 'ModelType',
form: 'Form',
table: 'Table',
table_tips: 'Please enter Mysql Table(required)',
column_type: 'ColumnType',
all_columns: 'All Columns',
some_columns: 'Some Columns',
column: 'Column',
column_tips: 'Please enter Columns (Comma separated)',
database: 'Database',
database_tips: 'Please enter Hive Database(required)',
hive_table_tips: 'Please enter Hive Table(required)',
hive_partition_keys: 'Hive partition Keys',
hive_partition_keys_tips: 'Please enter Hive Partition Keys',
hive_partition_values: 'Hive partition Values',
hive_partition_values_tips: 'Please enter Hive Partition Values',
export_dir: 'Export Dir',
export_dir_tips: 'Please enter Export Dir(required)',
sql_statement_tips: 'SQL Statement(required)',
map_column_hive: 'Map Column Hive',
map_column_java: 'Map Column Java',
data_target: 'Data Target',
create_hive_table: 'CreateHiveTable',
drop_delimiter: 'DropDelimiter',
over_write_src: 'OverWriteSrc',
hive_target_dir: 'Hive Target Dir',
hive_target_dir_tips: 'Please enter hive target dir',
replace_delimiter: 'ReplaceDelimiter',
replace_delimiter_tips: 'Please enter Replace Delimiter',
target_dir: 'Target Dir',
target_dir_tips: 'Please enter Target Dir(required)',
delete_target_dir: 'DeleteTargetDir',
compression_codec: 'CompressionCodec',
file_type: 'FileType',
fields_terminated: 'FieldsTerminated',
fields_terminated_tips: 'Please enter Fields Terminated',
lines_terminated: 'LinesTerminated',
lines_terminated_tips: 'Please enter Lines Terminated',
is_update: 'IsUpdate',
update_key: 'UpdateKey',
update_key_tips: 'Please enter Update Key',
update_mode: 'UpdateMode',
only_update: 'OnlyUpdate',
allow_insert: 'AllowInsert',
concurrency: 'Concurrency',
concurrency_tips: 'Please enter Concurrency'
}
}

61
dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts

@ -666,7 +666,7 @@ const project = {
target_task_name: '目标任务名',
target_task_name_tips: '请输入Pigeon任务名',
datasource_type: '数据源类型',
datasource: '数据源实例',
datasource_instances: '数据源实例',
sql_type: 'SQL类型',
sql_type_query: '查询',
sql_type_non_query: '非查询',
@ -680,7 +680,64 @@ const project = {
start: '运行',
edit: '编辑',
copy: '复制节点',
delete: '删除'
delete: '删除',
custom_job: '自定义任务',
custom_script: '自定义脚本',
sqoop_job_name: '任务名称',
sqoop_job_name_tips: '请输入任务名称(必填)',
direct: '流向',
hadoop_custom_params: 'Hadoop参数',
sqoop_advanced_parameters: 'Sqoop参数',
data_source: '数据来源',
type: '类型',
datasource: '数据源',
datasource_tips: '请选择数据源',
model_type: '模式',
form: '表单',
table: '表名',
table_tips: '请输入Mysql表名(必填)',
column_type: '列类型',
all_columns: '全表导入',
some_columns: '选择列',
column: '列',
column_tips: '请输入列名,用 , 隔开',
database: '数据库',
database_tips: '请输入Hive数据库(必填)',
hive_table_tips: '请输入Hive表名(必填)',
hive_partition_keys: 'Hive 分区键',
hive_partition_keys_tips: '请输入分区键',
hive_partition_values: 'Hive 分区值',
hive_partition_values_tips: '请输入分区值',
export_dir: '数据源路径',
export_dir_tips: '请输入数据源路径(必填)',
sql_statement_tips: 'SQL语句(必填)',
map_column_hive: 'Hive类型映射',
map_column_java: 'Java类型映射',
data_target: '数据目的',
create_hive_table: '是否创建新表',
drop_delimiter: '是否删除分隔符',
over_write_src: '是否覆盖数据源',
hive_target_dir: 'Hive目标路径',
hive_target_dir_tips: '请输入Hive临时目录',
replace_delimiter: '替换分隔符',
replace_delimiter_tips: '请输入替换分隔符',
target_dir: '目标路径',
target_dir_tips: '请输入目标路径(必填)',
delete_target_dir: '是否删除目录',
compression_codec: '压缩类型',
file_type: '保存格式',
fields_terminated: '列分隔符',
fields_terminated_tips: '请输入列分隔符',
lines_terminated: '行分隔符',
lines_terminated_tips: '请输入行分隔符',
is_update: '是否更新',
update_key: '更新列',
update_key_tips: '请输入更新列',
update_mode: '更新类型',
only_update: '只更新',
allow_insert: '无更新便插入',
concurrency: '并发度',
concurrency_tips: '请输入并发度'
}
}

4
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/index.ts

@ -34,6 +34,9 @@ export { useDatasourceType } from './use-datasource-type'
export { useDatasource } from './use-datasource'
export { useSqlType } from './use-sql-type'
export { useProcedure } from './use-procedure'
export { useCustomParams } from './use-custom-params'
export { useSourceType } from './use-sqoop-source-type'
export { useTargetType } from './use-sqoop-target-type'
export { useShell } from './use-shell'
export { useSpark } from './use-spark'
@ -41,3 +44,4 @@ export { useMr } from './use-mr'
export { useFlink } from './use-flink'
export { useHttp } from './use-http'
export { useSql } from './use-sql'
export { useSqoop } from './use-sqoop'

192
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-custom-params.ts

@ -0,0 +1,192 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Ref } from 'vue'
import { useI18n } from 'vue-i18n'
import type { IJsonItem } from '../types'
export function useCustomParams({
model,
field,
isSimple,
name = 'custom_parameters',
span = 24
}: {
model: { [field: string]: any }
field: string
isSimple: boolean
name?: string
span?: Ref | number
}): IJsonItem[] {
const { t } = useI18n()
if (isSimple) {
return [
{
type: 'custom-parameters',
field: field,
name: t(`project.node.${name}`),
span,
children: [
{
type: 'input',
field: 'prop',
span: 10,
props: {
placeholder: t('project.node.prop_tips'),
maxLength: 256
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.prop_tips'))
}
const sameItems = model.localParams.filter(
(item: { prop: string }) => item.prop === value
)
if (sameItems.length > 1) {
return new Error(t('project.node.prop_repeat'))
}
}
}
},
{
type: 'input',
field: 'value',
span: 10,
props: {
placeholder: t('project.node.value_tips'),
maxLength: 256
}
}
]
}
]
} else {
return [
{
type: 'custom-parameters',
field: field,
name: t(`project.node.${name}`),
span,
children: [
{
type: 'input',
field: 'prop',
span: 6,
props: {
placeholder: t('project.node.prop_tips'),
maxLength: 256
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.prop_tips'))
}
const sameItems = model.localParams.filter(
(item: { prop: string }) => item.prop === value
)
if (sameItems.length > 1) {
return new Error(t('project.node.prop_repeat'))
}
}
}
},
{
type: 'select',
field: 'direct',
span: 4,
options: DIRECT_LIST,
value: 'IN'
},
{
type: 'select',
field: 'type',
span: 6,
options: TYPE_LIST,
value: 'VARCHAR'
},
{
type: 'input',
field: 'value',
span: 6,
props: {
placeholder: t('project.node.value_tips'),
maxLength: 256
}
}
]
}
]
}
}
export const TYPE_LIST = [
{
value: 'VARCHAR',
label: 'VARCHAR'
},
{
value: 'INTEGER',
label: 'INTEGER'
},
{
value: 'LONG',
label: 'LONG'
},
{
value: 'FLOAT',
label: 'FLOAT'
},
{
value: 'DOUBLE',
label: 'DOUBLE'
},
{
value: 'DATE',
label: 'DATE'
},
{
value: 'TIME',
label: 'TIME'
},
{
value: 'TIMESTAMP',
label: 'TIMESTAMP'
},
{
value: 'BOOLEAN',
label: 'BOOLEAN'
}
]
export const DIRECT_LIST = [
{
value: 'IN',
label: 'IN'
},
{
value: 'OUT',
label: 'OUT'
}
]

2
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource.ts

@ -73,7 +73,7 @@ export function useDatasource(model: { [field: string]: any }): IJsonItem {
type: 'select',
field: 'datasource',
span: 12,
name: t('project.node.datasource'),
name: t('project.node.datasource_instances'),
props: {
loading: loading
},

52
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-flink.ts

@ -18,7 +18,8 @@ import { ref, onMounted, computed } from 'vue'
import { useI18n } from 'vue-i18n'
import { queryResourceByProgramType } from '@/service/modules/resources'
import { removeUselessChildren } from './use-shell'
import type { IJsonItem } from '../types'
import { useCustomParams } from '.'
import type { IJsonItem, ProgramType } from '../types'
export function useFlink(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n()
@ -38,7 +39,7 @@ export function useFlink(model: { [field: string]: any }): IJsonItem[] {
const mainJarOptions = ref([])
const resources: { [field: string]: any } = {}
const getResourceList = async (programType: string) => {
const getResourceList = async (programType: ProgramType) => {
if (resources[programType] !== void 0) {
mainJarOptions.value = resources[programType]
return
@ -66,7 +67,7 @@ export function useFlink(model: { [field: string]: any }): IJsonItem[] {
name: t('project.node.program_type'),
options: PROGRAM_TYPES,
props: {
'on-update:value': (value: string) => {
'on-update:value': (value: ProgramType) => {
model.mainJar = null
model.mainClass = ''
getResourceList(value)
@ -260,48 +261,11 @@ export function useFlink(model: { [field: string]: any }): IJsonItem[] {
labelField: 'name'
}
},
{
type: 'custom-parameters',
...useCustomParams({
model,
field: 'localParams',
name: t('project.node.custom_parameters'),
children: [
{
type: 'input',
field: 'prop',
span: 10,
props: {
placeholder: t('project.node.prop_tips'),
maxLength: 256
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.prop_tips'))
}
const sameItems = model.localParams.filter(
(item: { prop: string }) => item.prop === value
)
if (sameItems.length > 1) {
return new Error(t('project.node.prop_repeat'))
}
}
}
},
{
type: 'input',
field: 'value',
span: 10,
props: {
placeholder: t('project.node.value_tips'),
maxLength: 256
}
}
]
}
isSimple: true
})
]
}

46
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-http.ts

@ -16,6 +16,7 @@
*/
import { computed } from 'vue'
import { useI18n } from 'vue-i18n'
import { useCustomParams } from '.'
import type { IJsonItem } from '../types'
export function useHttp(model: { [field: string]: any }): IJsonItem[] {
@ -190,48 +191,11 @@ export function useHttp(model: { [field: string]: any }): IJsonItem[] {
}
}
},
{
type: 'custom-parameters',
...useCustomParams({
model,
field: 'localParams',
name: t('project.node.custom_parameters'),
children: [
{
type: 'input',
field: 'prop',
span: 6,
props: {
placeholder: t('project.node.prop_tips'),
maxLength: 256
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.prop_tips'))
}
const sameItems = model.localParams.filter(
(item: { prop: string }) => item.prop === value
)
if (sameItems.length > 1) {
return new Error(t('project.node.prop_repeat'))
}
}
}
},
{
type: 'input',
field: 'value',
span: 6,
props: {
placeholder: t('project.node.value_tips'),
maxLength: 256
}
}
]
}
isSimple: true
})
]
}

52
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-mr.ts

@ -18,8 +18,9 @@ import { ref, onMounted, computed } from 'vue'
import { useI18n } from 'vue-i18n'
import { queryResourceByProgramType } from '@/service/modules/resources'
import { removeUselessChildren } from './use-shell'
import { PROGRAM_TYPES, SPARK_VERSIONS, DeployModes } from './use-spark'
import type { IJsonItem } from '../types'
import { PROGRAM_TYPES } from './use-spark'
import { useCustomParams } from '.'
import type { IJsonItem, ProgramType } from '../types'
export function useMr(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n()
@ -31,7 +32,7 @@ export function useMr(model: { [field: string]: any }): IJsonItem[] {
const mainJarOptions = ref([])
const resources: { [field: string]: any } = {}
const getResourceList = async (programType: string) => {
const getResourceList = async (programType: ProgramType) => {
if (resources[programType] !== void 0) {
mainJarOptions.value = resources[programType]
return
@ -59,7 +60,7 @@ export function useMr(model: { [field: string]: any }): IJsonItem[] {
name: t('project.node.program_type'),
options: PROGRAM_TYPES,
props: {
'on-update:value': (value: string) => {
'on-update:value': (value: ProgramType) => {
model.mainJar = null
model.mainClass = ''
getResourceList(value)
@ -150,47 +151,6 @@ export function useMr(model: { [field: string]: any }): IJsonItem[] {
labelField: 'name'
}
},
{
type: 'custom-parameters',
field: 'localParams',
name: t('project.node.custom_parameters'),
children: [
{
type: 'input',
field: 'prop',
span: 10,
props: {
placeholder: t('project.node.prop_tips'),
maxLength: 256
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.prop_tips'))
}
const sameItems = model.localParams.filter(
(item: { prop: string }) => item.prop === value
)
if (sameItems.length > 1) {
return new Error(t('project.node.prop_repeat'))
}
}
}
},
{
type: 'input',
field: 'value',
span: 10,
props: {
placeholder: t('project.node.value_tips'),
maxLength: 256
}
}
]
}
...useCustomParams({ model, field: 'localParams', isSimple: true })
]
}

108
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-shell.ts

@ -17,6 +17,7 @@
import { ref, onMounted } from 'vue'
import { useI18n } from 'vue-i18n'
import { queryResourceList } from '@/service/modules/resources'
import { useCustomParams } from './use-custom-params'
import type { IJsonItem } from '../types'
export function useShell(model: { [field: string]: any }): IJsonItem[] {
@ -70,62 +71,7 @@ export function useShell(model: { [field: string]: any }): IJsonItem[] {
loading
}
},
{
type: 'custom-parameters',
field: 'localParams',
name: t('project.node.custom_parameters'),
children: [
{
type: 'input',
field: 'prop',
span: 6,
props: {
placeholder: t('project.node.prop_tips'),
maxLength: 256
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.prop_tips'))
}
const sameItems = model.localParams.filter(
(item: { prop: string }) => item.prop === value
)
if (sameItems.length > 1) {
return new Error(t('project.node.prop_repeat'))
}
}
}
},
{
type: 'select',
field: 'direct',
span: 4,
options: DIRECT_LIST,
value: 'IN'
},
{
type: 'select',
field: 'type',
span: 6,
options: TYPE_LIST,
value: 'VARCHAR'
},
{
type: 'input',
field: 'value',
span: 6,
props: {
placeholder: t('project.node.value_tips'),
maxLength: 256
}
}
]
}
...useCustomParams({ model, field: 'localParams', isSimple: true })
]
}
@ -140,53 +86,3 @@ export function removeUselessChildren(list: { children?: [] }[]) {
removeUselessChildren(item.children)
})
}
export const TYPE_LIST = [
{
value: 'VARCHAR',
label: 'VARCHAR'
},
{
value: 'INTEGER',
label: 'INTEGER'
},
{
value: 'LONG',
label: 'LONG'
},
{
value: 'FLOAT',
label: 'FLOAT'
},
{
value: 'DOUBLE',
label: 'DOUBLE'
},
{
value: 'DATE',
label: 'DATE'
},
{
value: 'TIME',
label: 'TIME'
},
{
value: 'TIMESTAMP',
label: 'TIMESTAMP'
},
{
value: 'BOOLEAN',
label: 'BOOLEAN'
}
]
export const DIRECT_LIST = [
{
value: 'IN',
label: 'IN'
},
{
value: 'OUT',
label: 'OUT'
}
]

50
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-spark.ts

@ -18,7 +18,8 @@ import { ref, onMounted, computed } from 'vue'
import { useI18n } from 'vue-i18n'
import { queryResourceByProgramType } from '@/service/modules/resources'
import { removeUselessChildren } from './use-shell'
import type { IJsonItem } from '../types'
import { useCustomParams } from '.'
import type { IJsonItem, ProgramType } from '../types'
export function useSpark(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n()
@ -30,7 +31,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] {
const mainJarOptions = ref([])
const resources: { [field: string]: any } = {}
const getResourceList = async (programType: string) => {
const getResourceList = async (programType: ProgramType) => {
if (resources[programType] !== void 0) {
mainJarOptions.value = resources[programType]
return
@ -58,7 +59,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] {
name: t('project.node.program_type'),
options: PROGRAM_TYPES,
props: {
'on-update:value': (value: string) => {
'on-update:value': (value: ProgramType) => {
model.mainJar = null
model.mainClass = ''
getResourceList(value)
@ -272,48 +273,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] {
labelField: 'name'
}
},
{
type: 'custom-parameters',
field: 'localParams',
name: t('project.node.custom_parameters'),
children: [
{
type: 'input',
field: 'prop',
span: 10,
props: {
placeholder: t('project.node.prop_tips'),
maxLength: 256
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.prop_tips'))
}
const sameItems = model.localParams.filter(
(item: { prop: string }) => item.prop === value
)
if (sameItems.length > 1) {
return new Error(t('project.node.prop_repeat'))
}
}
}
},
{
type: 'input',
field: 'value',
span: 10,
props: {
placeholder: t('project.node.value_tips'),
maxLength: 256
}
}
]
}
...useCustomParams({ model, field: 'localParams', isSimple: true })
]
}

85
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-datasource.ts

@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { onMounted, ref, unref, Ref } from 'vue'
import { queryDataSourceList } from '@/service/modules/data-source'
import { useI18n } from 'vue-i18n'
import type { IJsonItem, IDataBase } from '../types'
export function useDatasource(
model: { [field: string]: any },
span: Ref,
fieldType: string,
fieldDatasource: string
): IJsonItem[] {
const { t } = useI18n()
const dataSourceList = ref([])
const loading = ref(false)
const getDataSource = async (type: IDataBase) => {
if (loading.value) return
loading.value = true
try {
const result = await queryDataSourceList({ type })
dataSourceList.value = result.map(
(item: { name: string; id: number }) => ({
label: item.name,
value: item.id
})
)
loading.value = false
} catch (err) {
loading.value = false
}
}
onMounted(() => {
getDataSource('MYSQL')
})
return [
{
type: 'select',
field: fieldType,
name: t('project.node.datasource'),
span: span,
options: [{ label: 'MYSQL', value: 'MYSQL' }],
validate: {
required: unref(span) !== 0
}
},
{
type: 'select',
field: fieldDatasource,
name: ' ',
span: span,
props: {
placeholder: t('project.node.datasource_tips'),
filterable: true,
loading
},
options: dataSourceList,
validate: {
trigger: ['blur', 'input'],
validator(validate, value) {
if (!value) {
return new Error(t('project.node.datasource_tips'))
}
}
}
}
]
}

282
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-source-type.ts

@ -0,0 +1,282 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ref, h, watch, computed, unref } from 'vue'
import { useI18n } from 'vue-i18n'
import { useDatasource } from './use-sqoop-datasource'
import { useCustomParams } from '.'
import styles from '../index.module.scss'
import type { IJsonItem, IOption, ModelType } from '../types'
export function useSourceType(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n()
const unCustomSpan = computed(() => (model.isCustomTask ? 0 : 24))
const tableSpan = computed(() =>
model.sourceType === 'MYSQL' && model.srcQueryType === '0' ? 24 : 0
)
const editorSpan = computed(() =>
model.sourceType === 'MYSQL' && model.srcQueryType === '1' ? 24 : 0
)
const columnSpan = computed(() =>
model.sourceType === 'MYSQL' && model.srcColumnType === '1' ? 24 : 0
)
const mysqlSpan = computed(() => (model.sourceType === 'MYSQL' ? 24 : 0))
const hiveSpan = computed(() => (model.sourceType === 'HIVE' ? 24 : 0))
const hdfsSpan = computed(() => (model.sourceType === 'HDFS' ? 24 : 0))
const datasourceSpan = computed(() => (model.sourceType === 'MYSQL' ? 12 : 0))
const sourceTypes = ref([
{
label: 'MYSQL',
value: 'MYSQL'
}
] as IOption[])
const getSourceTypesByModelType = (modelType: ModelType): IOption[] => {
switch (modelType) {
case 'import':
return [
{
label: 'MYSQL',
value: 'MYSQL'
}
]
case 'export':
return [
{
label: 'HDFS',
value: 'HDFS'
},
{
label: 'HIVE',
value: 'HIVE'
}
]
default:
return [
{
label: 'MYSQL',
value: 'MYSQL'
},
{
label: 'HDFS',
value: 'HDFS'
},
{
label: 'HIVE',
value: 'HIVE'
}
]
}
}
watch(
() => model.modelType,
(modelType: ModelType) => {
getSourceTypesByModelType(modelType)
}
)
return [
{
type: 'custom',
field: 'custom-title',
span: unCustomSpan,
widget: h(
'div',
{ class: styles['field-title'] },
t('project.node.data_source')
)
},
{
type: 'select',
field: 'sourceType',
name: t('project.node.type'),
span: unCustomSpan,
options: sourceTypes
},
...useDatasource(
model,
datasourceSpan,
'sourceMysqlType',
'sourceMysqlDatasource'
),
{
type: 'radio',
field: 'srcQueryType',
name: t('project.node.model_type'),
span: mysqlSpan,
options: [
{
label: t('project.node.form'),
value: '0'
},
{
label: 'SQL',
value: '1'
}
]
},
{
type: 'input',
field: 'srcTable',
name: t('project.node.table'),
span: tableSpan,
props: {
placeholder: t('project.node.table_tips')
},
validate: {
trigger: ['input', 'blur'],
required: !!unref(tableSpan),
validator(validate, value) {
if (!!unref(tableSpan) && !value) {
return new Error(t('project.node.table_tips'))
}
}
}
},
{
type: 'radio',
field: 'srcColumnType',
name: t('project.node.column_type'),
span: tableSpan,
options: [
{ label: t('project.node.all_columns'), value: '0' },
{ label: t('project.node.some_columns'), value: '1' }
]
},
{
type: 'input',
field: 'srcColumns',
name: t('project.node.column'),
span: columnSpan,
props: {
placeholder: t('project.node.column_tips')
},
validate: {
trigger: ['input', 'blur'],
required: !!unref(columnSpan),
validator(validate, value) {
if (!!unref(columnSpan) && !value) {
return new Error(t('project.node.column_tips'))
}
}
}
},
{
type: 'input',
field: 'sourceHiveDatabase',
name: t('project.node.database'),
span: hiveSpan,
props: {
placeholder: t('project.node.database_tips')
},
validate: {
trigger: ['blur', 'input'],
required: !!unref(hiveSpan),
validator(validate, value) {
if (!!unref(hiveSpan) && !value) {
return new Error(t('project.node.database_tips'))
}
}
}
},
{
type: 'input',
field: 'sourceHiveTable',
name: t('project.node.table'),
span: hiveSpan,
props: {
placeholder: t('project.node.hive_table_tips')
},
validate: {
trigger: ['blur', 'input'],
required: !!unref(hiveSpan),
validator(validate, value) {
if (!!unref(hiveSpan) && !value) {
return new Error(t('project.node.hive_table_tips'))
}
}
}
},
{
type: 'input',
field: 'sourceHivePartitionKey',
name: t('project.node.hive_partition_keys'),
span: hiveSpan,
props: {
placeholder: t('project.node.hive_partition_keys_tips')
}
},
{
type: 'input',
field: 'sourceHivePartitionValue',
name: t('project.node.hive_partition_values'),
span: hiveSpan,
props: {
placeholder: t('project.node.hive_partition_values_tips')
}
},
{
type: 'input',
field: 'sourceHdfsExportDir',
name: t('project.node.export_dir'),
span: hdfsSpan,
props: {
placeholder: t('project.node.export_dir_tips')
},
validate: {
trigger: ['blur', 'input'],
required: !!unref(hdfsSpan),
validator(validate, value) {
if (!!unref(hdfsSpan) && !value) {
return new Error(t('project.node.export_dir_tips'))
}
}
}
},
{
type: 'editor',
field: 'sourceMysqlSrcQuerySql',
name: t('project.node.sql_statement'),
span: editorSpan,
validate: {
trigger: ['blur', 'input'],
required: !!unref(editorSpan),
validator(validate, value) {
if (!!unref(editorSpan) && !value) {
return new Error(t('project.node.sql_statement_tips'))
}
}
}
},
...useCustomParams({
model,
field: 'mapColumnHive',
name: 'map_column_hive',
isSimple: true,
span: editorSpan
}),
...useCustomParams({
model,
field: 'mapColumnJava',
name: 'map_column_java',
isSimple: true,
span: editorSpan
})
]
}

384
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-target-type.ts

@ -0,0 +1,384 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ref, h, watch, computed, unref } from 'vue'
import { useI18n } from 'vue-i18n'
import { useDatasource } from './use-sqoop-datasource'
import styles from '../index.module.scss'
import type { IJsonItem, IOption, SourceType } from '../types'
export function useTargetType(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n()
const unCustomSpan = computed(() => (model.isCustomTask ? 0 : 24))
const hiveSpan = computed(() => (model.targetType === 'HIVE' ? 24 : 0))
const hdfsSpan = computed(() => (model.targetType === 'HDFS' ? 24 : 0))
const mysqlSpan = computed(() => (model.targetType === 'MYSQL' ? 24 : 0))
const dataSourceSpan = computed(() => (model.targetType === 'MYSQL' ? 12 : 0))
const updateSpan = computed(() =>
model.targetType === 'MYSQL' && model.isUpdate ? 24 : 0
)
const targetTypes = ref([
{
label: 'HIVE',
value: 'HIVE'
},
{
label: 'HDFS',
value: 'HDFS'
}
] as IOption[])
const getTargetTypesBySourceType = (
sourceType: SourceType,
srcQueryType: string
): IOption[] => {
switch (sourceType) {
case 'MYSQL':
if (srcQueryType === '1') {
return [
{
label: 'HDFS',
value: 'HDFS'
}
]
}
return [
{
label: 'HDFS',
value: 'HDFS'
},
{
label: 'HIVE',
value: 'HIVE'
}
]
case 'HDFS':
case 'HIVE':
return [
{
label: 'MYSQL',
value: 'MYSQL'
}
]
default:
return [
{
label: 'HDFS',
value: 'HDFS'
},
{
label: 'HIVE',
value: 'HIVE'
}
]
}
}
watch(
() => [model.sourceType, model.srcQueryType],
([sourceType, srcQueryType]) => {
console.log(sourceType, srcQueryType)
getTargetTypesBySourceType(sourceType, srcQueryType)
}
)
return [
{
type: 'custom',
field: 'custom-title',
span: unCustomSpan,
widget: h(
'div',
{ class: styles['field-title'] },
t('project.node.data_target')
)
},
{
type: 'select',
field: 'targetType',
name: t('project.node.type'),
span: unCustomSpan,
options: targetTypes
},
{
type: 'input',
field: 'targetHiveDatabase',
name: t('project.node.database'),
span: hiveSpan,
props: {
placeholder: t('project.node.database_tips')
},
validate: {
trigger: ['blur', 'input'],
required: !!unref(hiveSpan),
validator(validate, value) {
if (!!unref(hiveSpan) && !value) {
return new Error(t('project.node.database_tips'))
}
}
}
},
{
type: 'input',
field: 'targetHiveTable',
name: t('project.node.database'),
span: hiveSpan,
props: {
placeholder: t('project.node.table')
},
validate: {
trigger: ['blur', 'input'],
required: !!unref(hiveSpan),
validator(rule, value) {
if (!!unref(hiveSpan) && !value) {
return new Error(t('project.node.hive_table_tips'))
}
}
}
},
{
type: 'switch',
field: 'targetHiveCreateTable',
span: hiveSpan,
name: t('project.node.create_hive_table')
},
{
type: 'switch',
field: 'targetHiveDropDelimiter',
span: hiveSpan,
name: t('project.node.drop_delimiter')
},
{
type: 'switch',
field: 'targetHiveOverWrite',
span: hiveSpan,
name: t('project.node.over_write_src')
},
{
type: 'input',
field: 'targetHiveTargetDir',
name: t('project.node.hive_target_dir'),
span: hiveSpan,
props: {
placeholder: t('project.node.hive_target_dir_tips')
}
},
{
type: 'input',
field: 'targetHiveReplaceDelimiter',
name: t('project.node.replace_delimiter'),
span: hiveSpan,
props: {
placeholder: t('project.node.replace_delimiter_tips')
}
},
{
type: 'input',
field: 'targetHivePartitionKey',
name: t('project.node.hive_partition_keys'),
span: hiveSpan,
props: {
placeholder: t('project.node.hive_partition_keys_tips')
}
},
{
type: 'input',
field: 'targetHivePartitionValue',
name: t('project.node.hive_partition_values'),
span: hiveSpan,
props: {
placeholder: t('project.node.hive_partition_values_tips')
}
},
{
type: 'input',
field: 'targetHdfsTargetPath',
name: t('project.node.target_dir'),
span: hdfsSpan,
props: {
placeholder: t('project.node.target_dir_tips')
},
validate: {
trigger: ['blur', 'input'],
required: !!unref(hdfsSpan),
validator(rule, value) {
if (!!unref(hdfsSpan) && !value) {
return new Error(t('project.node.target_dir_tips'))
}
}
}
},
{
type: 'switch',
field: 'targetHdfsDeleteTargetDir',
name: t('project.node.delete_target_dir'),
span: hdfsSpan
},
{
type: 'radio',
field: 'targetHdfsCompressionCodec',
name: t('project.node.compression_codec'),
span: hdfsSpan,
options: COMPRESSIONCODECS
},
{
type: 'radio',
field: 'targetHdfsFileType',
name: t('project.node.file_type'),
span: hdfsSpan,
options: FILETYPES
},
{
type: 'input',
field: 'targetHdfsFieldsTerminated',
name: t('project.node.fields_terminated'),
span: hdfsSpan,
props: {
placeholder: t('project.node.fields_terminated_tips')
}
},
{
type: 'input',
field: 'targetHdfsLinesTerminated',
name: t('project.node.lines_terminated'),
span: hdfsSpan,
props: {
placeholder: t('project.node.lines_terminated_tips')
}
},
...useDatasource(
model,
dataSourceSpan,
'targetMysqlType',
'targetMysqlDatasource'
),
{
type: 'input',
field: 'targetMysqlTable',
name: t('project.node.table'),
span: mysqlSpan,
props: {
placeholder: t('project.node.hive_table_tips')
},
validate: {
trigger: ['blur', 'input'],
required: !!unref(mysqlSpan),
validator(validate, value) {
if (!!unref(mysqlSpan) && !value) {
return new Error(t('project.node.table_tips'))
}
}
}
},
{
type: 'input',
field: 'targetMysqlColumns',
name: t('project.node.column'),
span: mysqlSpan,
props: {
placeholder: t('project.node.column_tips')
}
},
{
type: 'input',
field: 'targetMysqlFieldsTerminated',
name: t('project.node.fields_terminated'),
span: mysqlSpan,
props: {
placeholder: t('project.node.fields_terminated_tips')
}
},
{
type: 'input',
field: 'targetMysqlLinesTerminated',
name: t('project.node.lines_terminated'),
span: mysqlSpan,
props: {
placeholder: t('project.node.lines_terminated_tips')
}
},
{
type: 'switch',
field: 'targetMysqlIsUpdate',
span: mysqlSpan,
name: t('project.node.is_update')
},
{
type: 'input',
field: 'targetMysqlTargetUpdateKey',
name: t('project.node.update_key'),
span: updateSpan,
props: {
placeholder: t('project.node.update_key_tips')
}
},
{
type: 'radio',
field: 'targetMysqlUpdateMode',
name: t('project.node.update_mode'),
span: updateSpan,
options: [
{
label: t('project.node.only_update'),
value: 'updateonly'
},
{
label: t('project.node.allow_insert'),
value: 'allowinsert'
}
]
}
]
}
const COMPRESSIONCODECS = [
{
label: 'snappy',
value: 'snappy'
},
{
label: 'lzo',
value: 'lzo'
},
{
label: 'gzip',
value: 'gzip'
},
{
label: 'no',
value: ''
}
]
const FILETYPES = [
{
label: 'avro',
value: '--as-avrodatafile'
},
{
label: 'sequence',
value: '--as-sequencefile'
},
{
label: 'text',
value: '--as-textfile'
},
{
label: 'parquet',
value: '--as-parquetfile'
}
]

121
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop.ts

@ -0,0 +1,121 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { watch, computed, unref } from 'vue'
import { useI18n } from 'vue-i18n'
import { useCustomParams, useSourceType, useTargetType } from '.'
import type { IJsonItem, ModelType } from '../types'
export function useSqoop(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n()
const customSpan = computed(() => (model.isCustomTask ? 24 : 0))
const unCustomSpan = computed(() => (model.isCustomTask ? 0 : 24))
watch(
() => model.srcQueryType,
() => {}
)
return [
{
type: 'switch',
field: 'isCustomTask',
name: t('project.node.custom_job')
},
{
type: 'input',
field: 'jobName',
name: t('project.node.sqoop_job_name'),
span: unCustomSpan,
props: {
placeholder: t('project.node.sqoop_job_name_tips')
},
validate: {
trigger: ['input', 'blur'],
required: !model.isCustomTask,
validator(validate, value) {
if (!model.isCustomTask && !value) {
return new Error(t('project.node.sqoop_job_name_tips'))
}
}
}
},
{
type: 'select',
field: 'modelType',
name: t('project.node.direct'),
span: unCustomSpan,
options: MODEL_TYPES
},
...useCustomParams({
model,
field: 'hadoopCustomParams',
name: 'hadoop_custom_params',
isSimple: true,
span: unCustomSpan
}),
...useCustomParams({
model,
field: 'sqoopAdvancedParams',
name: 'sqoop_advanced_parameters',
isSimple: true,
span: unCustomSpan
}),
...useSourceType(model),
...useTargetType(model),
{
type: 'input-number',
field: 'concurrency',
name: t('project.node.concurrency'),
span: unCustomSpan,
props: {
placeholder: t('project.node.concurrency_tips')
}
},
{
type: 'editor',
field: 'customShell',
name: t('project.node.custom_script'),
span: customSpan,
validate: {
trigger: ['input', 'trigger'],
required: !!unref(customSpan),
validator(rule, value) {
if (!!unref(customSpan) && !value) {
return new Error(t('project.node.custom_script'))
}
}
}
},
...useCustomParams({
model,
field: 'localParams',
name: 'custom_parameters',
isSimple: true
})
]
}
const MODEL_TYPES = [
{
label: 'import',
value: 'import'
},
{
label: 'export',
value: 'export'
}
] as { label: ModelType; value: ModelType }[]

149
dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts

@ -16,7 +16,13 @@
*/
import { omit } from 'lodash'
import type { INodeData, ITaskData, ITaskParams } from './types'
import type {
INodeData,
ITaskData,
ITaskParams,
ISqoopTargetParams,
ISqoopSourceParams
} from './types'
export function formatParams(data: INodeData): {
processDefinitionCode: string
@ -66,6 +72,99 @@ export function formatParams(data: INodeData): {
taskParams.connectTimeout = data.connectTimeout
taskParams.socketTimeout = data.socketTimeout
}
if (data.taskType === 'SQOOP') {
taskParams.jobType = data.isCustomTask ? 'CUSTOM' : 'TEMPLATE'
taskParams.localParams = data.localParams
if (data.isCustomTask) {
taskParams.customShell = data.customShell
} else {
taskParams.jobName = data.jobName
taskParams.hadoopCustomParams = data.hadoopCustomParams
taskParams.sqoopAdvancedParams = data.sqoopAdvancedParams
taskParams.concurrency = data.concurrency
taskParams.modelType = data.modelType
taskParams.sourceType = data.sourceType
taskParams.targetType = data.targetType
let targetParams: ISqoopTargetParams = {}
let sourceParams: ISqoopSourceParams = {}
switch (data.targetType) {
case 'HIVE':
targetParams = {
hiveDatabase: data.targetHiveDatabase,
hiveTable: data.targetHiveTable,
createHiveTable: data.targetHiveCreateTable,
dropDelimiter: data.targetHiveDropDelimiter,
hiveOverWrite: data.targetHiveOverWrite,
hiveTargetDir: data.targetHiveTargetDir,
replaceDelimiter: data.targetHiveReplaceDelimiter,
hivePartitionKey: data.targetHivePartitionKey,
hivePartitionValue: data.targetHivePartitionValue
}
break
case 'HDFS':
targetParams = {
targetPath: data.targetHdfsTargetPath,
deleteTargetDir: data.targetHdfsDeleteTargetDir,
compressionCodec: data.targetHdfsCompressionCodec,
fileType: data.targetHdfsFileType,
fieldsTerminated: data.targetHdfsFieldsTerminated,
linesTerminated: data.targetHdfsLinesTerminated
}
break
case 'MYSQL':
targetParams = {
targetType: data.targetMysqlType,
targetDatasource: data.targetMysqlDatasource,
targetTable: data.targetMysqlTable,
targetColumns: data.targetMysqlColumns,
fieldsTerminated: data.targetMysqlFieldsTerminated,
linesTerminated: data.targetMysqlLinesTerminated,
isUpdate: data.targetMysqlIsUpdate,
targetUpdateKey: data.targetMysqlTargetUpdateKey,
targetUpdateMode: data.targetMysqlUpdateMode
}
break
default:
break
}
switch (data.sourceType) {
case 'MYSQL':
sourceParams = {
srcTable: data.srcQueryType === '1' ? '' : data.srcTable,
srcColumnType: data.srcQueryType === '1' ? '0' : data.srcColumnType,
srcColumns:
data.srcQueryType === '1' || data.srcColumnType === '0'
? ''
: data.srcColumns,
srcQuerySql:
data.srcQueryType === '0' ? '' : data.sourceMysqlSrcQuerySql,
srcQueryType: data.srcQueryType,
srcType: data.sourceMysqlType,
srcDatasource: data.sourceMysqlDatasource,
mapColumnHive: data.mapColumnHive,
mapColumnJava: data.mapColumnJava
}
break
case 'HDFS':
sourceParams = {
exportDir: data.sourceHdfsExportDir
}
break
case 'HIVE':
sourceParams = {
hiveDatabase: data.sourceHiveDatabase,
hiveTable: data.sourceHiveTable,
hivePartitionKey: data.sourceHivePartitionKey,
hivePartitionValue: data.sourceHivePartitionValue
}
break
default:
break
}
taskParams.targetParams = JSON.stringify(targetParams)
taskParams.sourceParams = JSON.stringify(sourceParams)
}
}
if (data.taskType === 'SQL') {
taskParams.type = data.type
@ -162,6 +261,54 @@ export function formatModel(data: ITaskData) {
if (data.taskParams?.method) {
params.method = data.taskParams?.method
}
if (data.taskParams?.targetParams) {
const targetParams: ISqoopTargetParams = JSON.parse(
data.taskParams.targetParams
)
params.targetHiveDatabase = targetParams.hiveDatabase
params.targetHiveTable = targetParams.hiveTable
params.targetHiveCreateTable = targetParams.createHiveTable
params.targetHiveDropDelimiter = targetParams.dropDelimiter
params.targetHiveOverWrite = targetParams.hiveOverWrite
params.targetHiveTargetDir = targetParams.hiveTargetDir
params.targetHiveReplaceDelimiter = targetParams.replaceDelimiter
params.targetHivePartitionKey = targetParams.hivePartitionKey
params.targetHivePartitionValue = targetParams.hivePartitionValue
params.targetHdfsTargetPath = targetParams.targetPath
params.targetHdfsDeleteTargetDir = targetParams.deleteTargetDir
params.targetHdfsCompressionCodec = targetParams.compressionCodec
params.targetHdfsFileType = targetParams.fileType
params.targetHdfsFieldsTerminated = targetParams.fieldsTerminated
params.targetHdfsLinesTerminated = targetParams.linesTerminated
params.targetMysqlType = targetParams.targetType
params.targetMysqlDatasource = targetParams.targetDatasource
params.targetMysqlTable = targetParams.targetTable
params.targetMysqlColumns = targetParams.targetColumns
params.targetMysqlFieldsTerminated = targetParams.fieldsTerminated
params.targetMysqlLinesTerminated = targetParams.linesTerminated
params.targetMysqlIsUpdate = targetParams.isUpdate
params.targetMysqlTargetUpdateKey = targetParams.targetUpdateKey
params.targetMysqlUpdateMode = targetParams.targetUpdateMode
}
if (data.taskParams?.sourceParams) {
const sourceParams: ISqoopSourceParams = JSON.parse(
data.taskParams.sourceParams
)
params.srcTable = sourceParams.srcTable
params.srcColumnType = sourceParams.srcColumnType
params.srcColumns = sourceParams.srcColumns
params.sourceMysqlSrcQuerySql = sourceParams.srcQuerySql
params.srcQueryType = sourceParams.srcQueryType
params.sourceMysqlType = sourceParams.srcType
params.sourceMysqlDatasource = sourceParams.srcDatasource
params.mapColumnHive = sourceParams.mapColumnHive
params.mapColumnJava = sourceParams.mapColumnJava
params.sourceHdfsExportDir = sourceParams.exportDir
params.sourceHiveDatabase = sourceParams.hiveDatabase
params.sourceHiveTable = sourceParams.hiveTable
params.sourceHivePartitionKey = sourceParams.hivePartitionKey
params.sourceHivePartitionValue = sourceParams.hivePartitionValue
}
return params
}

32
dolphinscheduler-ui-next/src/views/projects/task/components/node/index.module.scss

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
.field-title {
font-weight: bold;
width: 100%;
&::before {
content: '';
display: inline-block;
vertical-align: -2px;
width: 4px;
height: 1em;
background-color: var(--n-color);
border-radius: 4px;
margin-right: 8px;
}
}

1
dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-shell.ts

@ -31,6 +31,7 @@ export function useShell({
data?: ITaskData
}) {
const model = reactive({
taskType: 'SHELL',
name: '',
flag: 'YES',
description: '',

101
dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-sqoop.ts

@ -0,0 +1,101 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { reactive } from 'vue'
import * as Fields from '../fields/index'
import type { IJsonItem, INodeData, ITaskData } from '../types'
export function useSqoop({
projectCode,
from = 0,
readonly,
data
}: {
projectCode: number
from?: number
readonly?: boolean
data?: ITaskData
}) {
const model = reactive({
taskType: 'SQOOP',
name: '',
flag: 'YES',
description: '',
timeoutFlag: false,
localParams: [],
environmentCode: null,
failRetryInterval: 1,
failRetryTimes: 0,
workerGroup: 'default',
delayTime: 0,
timeout: 30,
isCustomTask: false,
hadoopCustomParams: [],
sqoopAdvancedParams: [],
mapColumnHive: [],
mapColumnJava: [],
modelType: 'import',
sourceType: 'MYSQL',
srcQueryType: '1',
srcColumnType: '0',
targetType: 'HDFS',
sourceMysqlType: 'MYSQL',
targetHdfsDeleteTargetDir: true,
targetHdfsCompressionCodec: 'snappy',
targetHdfsFileType: '--as-avrodatafile',
targetMysqlType: 'MYSQL',
targetMysqlUpdateMode: 'allowinsert',
targetHiveCreateTable: false,
targetHiveDropDelimiter: false,
targetHiveOverWrite: true,
concurrency: 1
} as INodeData)
let extra: IJsonItem[] = []
if (from === 1) {
extra = [
Fields.useTaskType(model, readonly),
Fields.useProcessName({
model,
projectCode,
isCreate: !data?.id,
from,
processName: data?.processName,
code: data?.code
})
]
}
return {
json: [
Fields.useName(),
...extra,
Fields.useRunFlag(),
Fields.useDescription(),
Fields.useTaskPriority(),
Fields.useWorkerGroup(),
Fields.useEnvironmentName(model, !data?.id),
...Fields.useTaskGroup(model, projectCode),
...Fields.useFailed(),
Fields.useDelayTime(model),
...Fields.useTimeoutAlarm(model),
...Fields.useSqoop(model),
Fields.usePreTasks(model)
] as IJsonItem[],
model
}
}

136
dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts

@ -19,6 +19,16 @@ import { VNode } from 'vue'
import type { SelectOption } from 'naive-ui'
import type { IFormItem, IJsonItem } from '@/components/form/types'
import type { TaskType } from '@/views/projects/task/constants/task-type'
import type { IDataBase } from '@/service/modules/data-source/types'
type ProgramType = 'JAVA' | 'SCALA' | 'PYTHON'
type SourceType = 'MYSQL' | 'HDFS' | 'HIVE'
type ModelType = 'import' | 'export'
interface IOption {
label: string
value: string | number
}
interface ITaskPriorityOption extends SelectOption {
icon: VNode
@ -29,10 +39,11 @@ interface IEnvironmentNameOption {
value: string
workerGroups?: string[]
}
interface ILocalParam {
prop: string
direct: string
type: string
direct?: string
type?: string
value?: string
}
@ -40,6 +51,91 @@ interface ISourceItem {
id: number
}
interface ISqoopTargetData {
targetHiveDatabase?: string
targetHiveTable?: string
targetHiveCreateTable?: boolean
targetHiveDropDelimiter?: boolean
targetHiveOverWrite?: boolean
targetHiveTargetDir?: string
targetHiveReplaceDelimiter?: string
targetHivePartitionKey?: string
targetHivePartitionValue?: string
targetHdfsTargetPath?: string
targetHdfsDeleteTargetDir?: boolean
targetHdfsCompressionCodec?: string
targetHdfsFileType?: string
targetHdfsFieldsTerminated?: string
targetHdfsLinesTerminated?: string
targetMysqlType?: string
targetMysqlDatasource?: string
targetMysqlTable?: string
targetMysqlColumns?: string
targetMysqlFieldsTerminated?: string
targetMysqlLinesTerminated?: string
targetMysqlIsUpdate?: string
targetMysqlTargetUpdateKey?: string
targetMysqlUpdateMode?: string
}
interface ISqoopSourceData {
srcQueryType?: '1' | '0'
srcTable?: string
srcColumnType?: '1' | '0'
srcColumns?: string
sourceMysqlSrcQuerySql?: string
sourceMysqlType?: string
sourceMysqlDatasource?: string
mapColumnHive?: ILocalParam[]
mapColumnJava?: ILocalParam[]
sourceHdfsExportDir?: string
sourceHiveDatabase?: string
sourceHiveTable?: string
sourceHivePartitionKey?: string
sourceHivePartitionValue?: string
}
interface ISqoopTargetParams {
hiveDatabase?: string
hiveTable?: string
createHiveTable?: boolean
dropDelimiter?: boolean
hiveOverWrite?: boolean
hiveTargetDir?: string
replaceDelimiter?: string
hivePartitionKey?: string
hivePartitionValue?: string
targetPath?: string
deleteTargetDir?: boolean
compressionCodec?: string
fileType?: string
fieldsTerminated?: string
linesTerminated?: string
targetType?: string
targetDatasource?: string
targetTable?: string
targetColumns?: string
isUpdate?: string
targetUpdateKey?: string
targetUpdateMode?: string
}
interface ISqoopSourceParams {
srcTable?: string
srcColumnType?: '1' | '0'
srcColumns?: string
srcQuerySql?: string
srcQueryType?: '1' | '0'
srcType?: string
srcDatasource?: string
mapColumnHive?: ILocalParam[]
mapColumnJava?: ILocalParam[]
exportDir?: string
hiveDatabase?: string
hiveTable?: string
hivePartitionKey?: string
hivePartitionValue?: string
}
interface ITaskParams {
resourceList?: ISourceItem[]
mainJar?: ISourceItem
@ -77,11 +173,28 @@ interface ITaskParams {
preStatements?: string[]
postStatements?: string[]
method?: string
jobType?: 'CUSTOM' | 'TEMPLATE'
customShell?: string
jobName?: string
hadoopCustomParams?: ILocalParam[]
sqoopAdvancedParams?: ILocalParam[]
concurrency?: number
modelType?: ModelType
sourceType?: SourceType
targetType?: SourceType
targetParams?: string
sourceParams?: string
}
type ITaskType = TaskType
interface INodeData extends Omit<ITaskParams, 'resourceList' | 'mainJar'> {
interface INodeData
extends Omit<
ITaskParams,
'resourceList' | 'mainJar' | 'targetParams' | 'sourceParams'
>,
ISqoopTargetData,
ISqoopSourceData {
id?: string
taskType?: ITaskType
processName?: number
@ -106,13 +219,7 @@ interface INodeData extends Omit<ITaskParams, 'resourceList' | 'mainJar'> {
resourceList?: number[]
mainJar?: number
timeoutSetting?: boolean
type?: string
datasource?: string
sql?: string
sqlType?: string
preStatements?: string[]
postStatements?: string[]
method?: string
isCustomTask?: boolean
}
interface ITaskData
@ -136,5 +243,12 @@ export {
INodeData,
IFormItem,
IJsonItem,
ITaskParams
ITaskParams,
IOption,
IDataBase,
ProgramType,
ModelType,
SourceType,
ISqoopSourceParams,
ISqoopTargetParams
}

10
dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts

@ -25,6 +25,7 @@ import { useMr } from './tasks/use-mr'
import { useHttp } from './tasks/use-http'
import { useSql } from './tasks/use-sql'
import { useProcedure } from './tasks/use-procedure'
import { useSqoop } from './tasks/use-sqoop'
import { IJsonItem, INodeData, ITaskData } from './types'
export function useTask({
@ -120,5 +121,14 @@ export function useTask({
data
})
}
if (taskType === 'SQOOP') {
node = useSqoop({
projectCode,
from,
readonly,
data
})
}
return node
}

Loading…
Cancel
Save