diff --git a/dolphinscheduler-ui-next/src/components/form/fields/get-field.ts b/dolphinscheduler-ui-next/src/components/form/fields/get-field.ts index cffa65cdde..3cb5affef0 100644 --- a/dolphinscheduler-ui-next/src/components/form/fields/get-field.ts +++ b/dolphinscheduler-ui-next/src/components/form/fields/get-field.ts @@ -26,6 +26,9 @@ const getField = ( ) => { const { type = 'input' } = item const renderTypeName = `render${upperFirst(camelCase(type))}` + if (type === 'custom') { + return item.widget || null + } // TODO Support other widgets later if (type === 'custom-parameters') { let fieldRules: { [key: string]: FormItemRule }[] = [] diff --git a/dolphinscheduler-ui-next/src/components/form/types.ts b/dolphinscheduler-ui-next/src/components/form/types.ts index f7b350d5ce..923c07aebb 100644 --- a/dolphinscheduler-ui-next/src/components/form/types.ts +++ b/dolphinscheduler-ui-next/src/components/form/types.ts @@ -35,6 +35,7 @@ type IType = | 'checkbox' | 'tree-select' | 'multi-input' + | 'custom' interface IOption extends SelectOption, TreeSelectOption { label: string @@ -46,6 +47,7 @@ interface IFormItem { label?: string widget: any span?: number | Ref + type?: 'custom' } interface IMeta extends Omit { @@ -65,6 +67,7 @@ interface IJsonItem { children?: IJsonItem[] slots?: object span?: number | Ref + widget?: any } export { diff --git a/dolphinscheduler-ui-next/src/locales/modules/en_US.ts b/dolphinscheduler-ui-next/src/locales/modules/en_US.ts index c9d6e64c19..3fae07f128 100644 --- a/dolphinscheduler-ui-next/src/locales/modules/en_US.ts +++ b/dolphinscheduler-ui-next/src/locales/modules/en_US.ts @@ -674,7 +674,7 @@ const project = { target_task_name: 'Target Task Name', target_task_name_tips: 'Please enter the Pigeon task name', datasource_type: 'Datasource types', - datasource: 'Datasource instances', + datasource_instances: 'Datasource instances', sql_type: 'SQL Type', sql_type_query: 'Query', sql_type_non_query: 'Non Query', @@ -688,7 +688,64 @@ const project = { start: 'Start', edit: 'Edit', copy: 'Copy', - delete: 'Delete' + delete: 'Delete', + custom_job: 'Custom Job', + custom_script: 'Custom Script', + sqoop_job_name: 'Job Name', + sqoop_job_name_tips: 'Please enter Job Name(required)', + direct: 'Direct', + hadoop_custom_params: 'Hadoop Params', + sqoop_advanced_parameters: 'Sqoop Advanced Parameters', + data_source: 'Data Source', + type: 'Type', + datasource: 'Datasource', + datasource_tips: 'Please select the datasource', + model_type: 'ModelType', + form: 'Form', + table: 'Table', + table_tips: 'Please enter Mysql Table(required)', + column_type: 'ColumnType', + all_columns: 'All Columns', + some_columns: 'Some Columns', + column: 'Column', + column_tips: 'Please enter Columns (Comma separated)', + database: 'Database', + database_tips: 'Please enter Hive Database(required)', + hive_table_tips: 'Please enter Hive Table(required)', + hive_partition_keys: 'Hive partition Keys', + hive_partition_keys_tips: 'Please enter Hive Partition Keys', + hive_partition_values: 'Hive partition Values', + hive_partition_values_tips: 'Please enter Hive Partition Values', + export_dir: 'Export Dir', + export_dir_tips: 'Please enter Export Dir(required)', + sql_statement_tips: 'SQL Statement(required)', + map_column_hive: 'Map Column Hive', + map_column_java: 'Map Column Java', + data_target: 'Data Target', + create_hive_table: 'CreateHiveTable', + drop_delimiter: 'DropDelimiter', + over_write_src: 'OverWriteSrc', + hive_target_dir: 'Hive Target Dir', + hive_target_dir_tips: 'Please enter hive target dir', + replace_delimiter: 'ReplaceDelimiter', + replace_delimiter_tips: 'Please enter Replace Delimiter', + target_dir: 'Target Dir', + target_dir_tips: 'Please enter Target Dir(required)', + delete_target_dir: 'DeleteTargetDir', + compression_codec: 'CompressionCodec', + file_type: 'FileType', + fields_terminated: 'FieldsTerminated', + fields_terminated_tips: 'Please enter Fields Terminated', + lines_terminated: 'LinesTerminated', + lines_terminated_tips: 'Please enter Lines Terminated', + is_update: 'IsUpdate', + update_key: 'UpdateKey', + update_key_tips: 'Please enter Update Key', + update_mode: 'UpdateMode', + only_update: 'OnlyUpdate', + allow_insert: 'AllowInsert', + concurrency: 'Concurrency', + concurrency_tips: 'Please enter Concurrency' } } diff --git a/dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts b/dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts index 6e58969d88..222e71e920 100644 --- a/dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts +++ b/dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts @@ -666,7 +666,7 @@ const project = { target_task_name: '目标任务名', target_task_name_tips: '请输入Pigeon任务名', datasource_type: '数据源类型', - datasource: '数据源实例', + datasource_instances: '数据源实例', sql_type: 'SQL类型', sql_type_query: '查询', sql_type_non_query: '非查询', @@ -680,7 +680,64 @@ const project = { start: '运行', edit: '编辑', copy: '复制节点', - delete: '删除' + delete: '删除', + custom_job: '自定义任务', + custom_script: '自定义脚本', + sqoop_job_name: '任务名称', + sqoop_job_name_tips: '请输入任务名称(必填)', + direct: '流向', + hadoop_custom_params: 'Hadoop参数', + sqoop_advanced_parameters: 'Sqoop参数', + data_source: '数据来源', + type: '类型', + datasource: '数据源', + datasource_tips: '请选择数据源', + model_type: '模式', + form: '表单', + table: '表名', + table_tips: '请输入Mysql表名(必填)', + column_type: '列类型', + all_columns: '全表导入', + some_columns: '选择列', + column: '列', + column_tips: '请输入列名,用 , 隔开', + database: '数据库', + database_tips: '请输入Hive数据库(必填)', + hive_table_tips: '请输入Hive表名(必填)', + hive_partition_keys: 'Hive 分区键', + hive_partition_keys_tips: '请输入分区键', + hive_partition_values: 'Hive 分区值', + hive_partition_values_tips: '请输入分区值', + export_dir: '数据源路径', + export_dir_tips: '请输入数据源路径(必填)', + sql_statement_tips: 'SQL语句(必填)', + map_column_hive: 'Hive类型映射', + map_column_java: 'Java类型映射', + data_target: '数据目的', + create_hive_table: '是否创建新表', + drop_delimiter: '是否删除分隔符', + over_write_src: '是否覆盖数据源', + hive_target_dir: 'Hive目标路径', + hive_target_dir_tips: '请输入Hive临时目录', + replace_delimiter: '替换分隔符', + replace_delimiter_tips: '请输入替换分隔符', + target_dir: '目标路径', + target_dir_tips: '请输入目标路径(必填)', + delete_target_dir: '是否删除目录', + compression_codec: '压缩类型', + file_type: '保存格式', + fields_terminated: '列分隔符', + fields_terminated_tips: '请输入列分隔符', + lines_terminated: '行分隔符', + lines_terminated_tips: '请输入行分隔符', + is_update: '是否更新', + update_key: '更新列', + update_key_tips: '请输入更新列', + update_mode: '更新类型', + only_update: '只更新', + allow_insert: '无更新便插入', + concurrency: '并发度', + concurrency_tips: '请输入并发度' } } diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/index.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/index.ts index 1dd5292c84..70b0d9c43e 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/index.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/index.ts @@ -34,6 +34,9 @@ export { useDatasourceType } from './use-datasource-type' export { useDatasource } from './use-datasource' export { useSqlType } from './use-sql-type' export { useProcedure } from './use-procedure' +export { useCustomParams } from './use-custom-params' +export { useSourceType } from './use-sqoop-source-type' +export { useTargetType } from './use-sqoop-target-type' export { useShell } from './use-shell' export { useSpark } from './use-spark' @@ -41,3 +44,4 @@ export { useMr } from './use-mr' export { useFlink } from './use-flink' export { useHttp } from './use-http' export { useSql } from './use-sql' +export { useSqoop } from './use-sqoop' diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-custom-params.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-custom-params.ts new file mode 100644 index 0000000000..fb898a307a --- /dev/null +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-custom-params.ts @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Ref } from 'vue' +import { useI18n } from 'vue-i18n' +import type { IJsonItem } from '../types' + +export function useCustomParams({ + model, + field, + isSimple, + name = 'custom_parameters', + span = 24 +}: { + model: { [field: string]: any } + field: string + isSimple: boolean + name?: string + span?: Ref | number +}): IJsonItem[] { + const { t } = useI18n() + + if (isSimple) { + return [ + { + type: 'custom-parameters', + field: field, + name: t(`project.node.${name}`), + span, + children: [ + { + type: 'input', + field: 'prop', + span: 10, + props: { + placeholder: t('project.node.prop_tips'), + maxLength: 256 + }, + validate: { + trigger: ['input', 'blur'], + required: true, + validator(validate: any, value: string) { + if (!value) { + return new Error(t('project.node.prop_tips')) + } + + const sameItems = model.localParams.filter( + (item: { prop: string }) => item.prop === value + ) + + if (sameItems.length > 1) { + return new Error(t('project.node.prop_repeat')) + } + } + } + }, + { + type: 'input', + field: 'value', + span: 10, + props: { + placeholder: t('project.node.value_tips'), + maxLength: 256 + } + } + ] + } + ] + } else { + return [ + { + type: 'custom-parameters', + field: field, + name: t(`project.node.${name}`), + span, + children: [ + { + type: 'input', + field: 'prop', + span: 6, + props: { + placeholder: t('project.node.prop_tips'), + maxLength: 256 + }, + validate: { + trigger: ['input', 'blur'], + required: true, + validator(validate: any, value: string) { + if (!value) { + return new Error(t('project.node.prop_tips')) + } + + const sameItems = model.localParams.filter( + (item: { prop: string }) => item.prop === value + ) + + if (sameItems.length > 1) { + return new Error(t('project.node.prop_repeat')) + } + } + } + }, + { + type: 'select', + field: 'direct', + span: 4, + options: DIRECT_LIST, + value: 'IN' + }, + { + type: 'select', + field: 'type', + span: 6, + options: TYPE_LIST, + value: 'VARCHAR' + }, + { + type: 'input', + field: 'value', + span: 6, + props: { + placeholder: t('project.node.value_tips'), + maxLength: 256 + } + } + ] + } + ] + } +} +export const TYPE_LIST = [ + { + value: 'VARCHAR', + label: 'VARCHAR' + }, + { + value: 'INTEGER', + label: 'INTEGER' + }, + { + value: 'LONG', + label: 'LONG' + }, + { + value: 'FLOAT', + label: 'FLOAT' + }, + { + value: 'DOUBLE', + label: 'DOUBLE' + }, + { + value: 'DATE', + label: 'DATE' + }, + { + value: 'TIME', + label: 'TIME' + }, + { + value: 'TIMESTAMP', + label: 'TIMESTAMP' + }, + { + value: 'BOOLEAN', + label: 'BOOLEAN' + } +] + +export const DIRECT_LIST = [ + { + value: 'IN', + label: 'IN' + }, + { + value: 'OUT', + label: 'OUT' + } +] diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource.ts index f34636ae95..b49df646b1 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource.ts @@ -73,7 +73,7 @@ export function useDatasource(model: { [field: string]: any }): IJsonItem { type: 'select', field: 'datasource', span: 12, - name: t('project.node.datasource'), + name: t('project.node.datasource_instances'), props: { loading: loading }, diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-flink.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-flink.ts index 8b2c829d75..da1cad9178 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-flink.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-flink.ts @@ -18,7 +18,8 @@ import { ref, onMounted, computed } from 'vue' import { useI18n } from 'vue-i18n' import { queryResourceByProgramType } from '@/service/modules/resources' import { removeUselessChildren } from './use-shell' -import type { IJsonItem } from '../types' +import { useCustomParams } from '.' +import type { IJsonItem, ProgramType } from '../types' export function useFlink(model: { [field: string]: any }): IJsonItem[] { const { t } = useI18n() @@ -38,7 +39,7 @@ export function useFlink(model: { [field: string]: any }): IJsonItem[] { const mainJarOptions = ref([]) const resources: { [field: string]: any } = {} - const getResourceList = async (programType: string) => { + const getResourceList = async (programType: ProgramType) => { if (resources[programType] !== void 0) { mainJarOptions.value = resources[programType] return @@ -66,7 +67,7 @@ export function useFlink(model: { [field: string]: any }): IJsonItem[] { name: t('project.node.program_type'), options: PROGRAM_TYPES, props: { - 'on-update:value': (value: string) => { + 'on-update:value': (value: ProgramType) => { model.mainJar = null model.mainClass = '' getResourceList(value) @@ -260,48 +261,11 @@ export function useFlink(model: { [field: string]: any }): IJsonItem[] { labelField: 'name' } }, - { - type: 'custom-parameters', + ...useCustomParams({ + model, field: 'localParams', - name: t('project.node.custom_parameters'), - children: [ - { - type: 'input', - field: 'prop', - span: 10, - props: { - placeholder: t('project.node.prop_tips'), - maxLength: 256 - }, - validate: { - trigger: ['input', 'blur'], - required: true, - validator(validate: any, value: string) { - if (!value) { - return new Error(t('project.node.prop_tips')) - } - - const sameItems = model.localParams.filter( - (item: { prop: string }) => item.prop === value - ) - - if (sameItems.length > 1) { - return new Error(t('project.node.prop_repeat')) - } - } - } - }, - { - type: 'input', - field: 'value', - span: 10, - props: { - placeholder: t('project.node.value_tips'), - maxLength: 256 - } - } - ] - } + isSimple: true + }) ] } diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-http.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-http.ts index c7fd79b680..4a8396b646 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-http.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-http.ts @@ -16,6 +16,7 @@ */ import { computed } from 'vue' import { useI18n } from 'vue-i18n' +import { useCustomParams } from '.' import type { IJsonItem } from '../types' export function useHttp(model: { [field: string]: any }): IJsonItem[] { @@ -190,48 +191,11 @@ export function useHttp(model: { [field: string]: any }): IJsonItem[] { } } }, - { - type: 'custom-parameters', + ...useCustomParams({ + model, field: 'localParams', - name: t('project.node.custom_parameters'), - children: [ - { - type: 'input', - field: 'prop', - span: 6, - props: { - placeholder: t('project.node.prop_tips'), - maxLength: 256 - }, - validate: { - trigger: ['input', 'blur'], - required: true, - validator(validate: any, value: string) { - if (!value) { - return new Error(t('project.node.prop_tips')) - } - - const sameItems = model.localParams.filter( - (item: { prop: string }) => item.prop === value - ) - - if (sameItems.length > 1) { - return new Error(t('project.node.prop_repeat')) - } - } - } - }, - { - type: 'input', - field: 'value', - span: 6, - props: { - placeholder: t('project.node.value_tips'), - maxLength: 256 - } - } - ] - } + isSimple: true + }) ] } diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-mr.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-mr.ts index c0ea15ede0..a13f32e84e 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-mr.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-mr.ts @@ -18,8 +18,9 @@ import { ref, onMounted, computed } from 'vue' import { useI18n } from 'vue-i18n' import { queryResourceByProgramType } from '@/service/modules/resources' import { removeUselessChildren } from './use-shell' -import { PROGRAM_TYPES, SPARK_VERSIONS, DeployModes } from './use-spark' -import type { IJsonItem } from '../types' +import { PROGRAM_TYPES } from './use-spark' +import { useCustomParams } from '.' +import type { IJsonItem, ProgramType } from '../types' export function useMr(model: { [field: string]: any }): IJsonItem[] { const { t } = useI18n() @@ -31,7 +32,7 @@ export function useMr(model: { [field: string]: any }): IJsonItem[] { const mainJarOptions = ref([]) const resources: { [field: string]: any } = {} - const getResourceList = async (programType: string) => { + const getResourceList = async (programType: ProgramType) => { if (resources[programType] !== void 0) { mainJarOptions.value = resources[programType] return @@ -59,7 +60,7 @@ export function useMr(model: { [field: string]: any }): IJsonItem[] { name: t('project.node.program_type'), options: PROGRAM_TYPES, props: { - 'on-update:value': (value: string) => { + 'on-update:value': (value: ProgramType) => { model.mainJar = null model.mainClass = '' getResourceList(value) @@ -150,47 +151,6 @@ export function useMr(model: { [field: string]: any }): IJsonItem[] { labelField: 'name' } }, - { - type: 'custom-parameters', - field: 'localParams', - name: t('project.node.custom_parameters'), - children: [ - { - type: 'input', - field: 'prop', - span: 10, - props: { - placeholder: t('project.node.prop_tips'), - maxLength: 256 - }, - validate: { - trigger: ['input', 'blur'], - required: true, - validator(validate: any, value: string) { - if (!value) { - return new Error(t('project.node.prop_tips')) - } - - const sameItems = model.localParams.filter( - (item: { prop: string }) => item.prop === value - ) - - if (sameItems.length > 1) { - return new Error(t('project.node.prop_repeat')) - } - } - } - }, - { - type: 'input', - field: 'value', - span: 10, - props: { - placeholder: t('project.node.value_tips'), - maxLength: 256 - } - } - ] - } + ...useCustomParams({ model, field: 'localParams', isSimple: true }) ] } diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-shell.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-shell.ts index 502b00e9bd..c36933b5c4 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-shell.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-shell.ts @@ -17,6 +17,7 @@ import { ref, onMounted } from 'vue' import { useI18n } from 'vue-i18n' import { queryResourceList } from '@/service/modules/resources' +import { useCustomParams } from './use-custom-params' import type { IJsonItem } from '../types' export function useShell(model: { [field: string]: any }): IJsonItem[] { @@ -70,62 +71,7 @@ export function useShell(model: { [field: string]: any }): IJsonItem[] { loading } }, - { - type: 'custom-parameters', - field: 'localParams', - name: t('project.node.custom_parameters'), - children: [ - { - type: 'input', - field: 'prop', - span: 6, - props: { - placeholder: t('project.node.prop_tips'), - maxLength: 256 - }, - validate: { - trigger: ['input', 'blur'], - required: true, - validator(validate: any, value: string) { - if (!value) { - return new Error(t('project.node.prop_tips')) - } - - const sameItems = model.localParams.filter( - (item: { prop: string }) => item.prop === value - ) - - if (sameItems.length > 1) { - return new Error(t('project.node.prop_repeat')) - } - } - } - }, - { - type: 'select', - field: 'direct', - span: 4, - options: DIRECT_LIST, - value: 'IN' - }, - { - type: 'select', - field: 'type', - span: 6, - options: TYPE_LIST, - value: 'VARCHAR' - }, - { - type: 'input', - field: 'value', - span: 6, - props: { - placeholder: t('project.node.value_tips'), - maxLength: 256 - } - } - ] - } + ...useCustomParams({ model, field: 'localParams', isSimple: true }) ] } @@ -140,53 +86,3 @@ export function removeUselessChildren(list: { children?: [] }[]) { removeUselessChildren(item.children) }) } - -export const TYPE_LIST = [ - { - value: 'VARCHAR', - label: 'VARCHAR' - }, - { - value: 'INTEGER', - label: 'INTEGER' - }, - { - value: 'LONG', - label: 'LONG' - }, - { - value: 'FLOAT', - label: 'FLOAT' - }, - { - value: 'DOUBLE', - label: 'DOUBLE' - }, - { - value: 'DATE', - label: 'DATE' - }, - { - value: 'TIME', - label: 'TIME' - }, - { - value: 'TIMESTAMP', - label: 'TIMESTAMP' - }, - { - value: 'BOOLEAN', - label: 'BOOLEAN' - } -] - -export const DIRECT_LIST = [ - { - value: 'IN', - label: 'IN' - }, - { - value: 'OUT', - label: 'OUT' - } -] diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-spark.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-spark.ts index ca4bf86d8d..c17717f957 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-spark.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-spark.ts @@ -18,7 +18,8 @@ import { ref, onMounted, computed } from 'vue' import { useI18n } from 'vue-i18n' import { queryResourceByProgramType } from '@/service/modules/resources' import { removeUselessChildren } from './use-shell' -import type { IJsonItem } from '../types' +import { useCustomParams } from '.' +import type { IJsonItem, ProgramType } from '../types' export function useSpark(model: { [field: string]: any }): IJsonItem[] { const { t } = useI18n() @@ -30,7 +31,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] { const mainJarOptions = ref([]) const resources: { [field: string]: any } = {} - const getResourceList = async (programType: string) => { + const getResourceList = async (programType: ProgramType) => { if (resources[programType] !== void 0) { mainJarOptions.value = resources[programType] return @@ -58,7 +59,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] { name: t('project.node.program_type'), options: PROGRAM_TYPES, props: { - 'on-update:value': (value: string) => { + 'on-update:value': (value: ProgramType) => { model.mainJar = null model.mainClass = '' getResourceList(value) @@ -272,48 +273,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] { labelField: 'name' } }, - { - type: 'custom-parameters', - field: 'localParams', - name: t('project.node.custom_parameters'), - children: [ - { - type: 'input', - field: 'prop', - span: 10, - props: { - placeholder: t('project.node.prop_tips'), - maxLength: 256 - }, - validate: { - trigger: ['input', 'blur'], - required: true, - validator(validate: any, value: string) { - if (!value) { - return new Error(t('project.node.prop_tips')) - } - - const sameItems = model.localParams.filter( - (item: { prop: string }) => item.prop === value - ) - - if (sameItems.length > 1) { - return new Error(t('project.node.prop_repeat')) - } - } - } - }, - { - type: 'input', - field: 'value', - span: 10, - props: { - placeholder: t('project.node.value_tips'), - maxLength: 256 - } - } - ] - } + ...useCustomParams({ model, field: 'localParams', isSimple: true }) ] } diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-datasource.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-datasource.ts new file mode 100644 index 0000000000..5904fe3ea4 --- /dev/null +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-datasource.ts @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { onMounted, ref, unref, Ref } from 'vue' +import { queryDataSourceList } from '@/service/modules/data-source' +import { useI18n } from 'vue-i18n' +import type { IJsonItem, IDataBase } from '../types' + +export function useDatasource( + model: { [field: string]: any }, + span: Ref, + fieldType: string, + fieldDatasource: string +): IJsonItem[] { + const { t } = useI18n() + const dataSourceList = ref([]) + const loading = ref(false) + + const getDataSource = async (type: IDataBase) => { + if (loading.value) return + loading.value = true + try { + const result = await queryDataSourceList({ type }) + dataSourceList.value = result.map( + (item: { name: string; id: number }) => ({ + label: item.name, + value: item.id + }) + ) + loading.value = false + } catch (err) { + loading.value = false + } + } + onMounted(() => { + getDataSource('MYSQL') + }) + + return [ + { + type: 'select', + field: fieldType, + name: t('project.node.datasource'), + span: span, + options: [{ label: 'MYSQL', value: 'MYSQL' }], + validate: { + required: unref(span) !== 0 + } + }, + { + type: 'select', + field: fieldDatasource, + name: ' ', + span: span, + props: { + placeholder: t('project.node.datasource_tips'), + filterable: true, + loading + }, + options: dataSourceList, + validate: { + trigger: ['blur', 'input'], + validator(validate, value) { + if (!value) { + return new Error(t('project.node.datasource_tips')) + } + } + } + } + ] +} diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-source-type.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-source-type.ts new file mode 100644 index 0000000000..6398a17440 --- /dev/null +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-source-type.ts @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ref, h, watch, computed, unref } from 'vue' +import { useI18n } from 'vue-i18n' +import { useDatasource } from './use-sqoop-datasource' +import { useCustomParams } from '.' +import styles from '../index.module.scss' +import type { IJsonItem, IOption, ModelType } from '../types' + +export function useSourceType(model: { [field: string]: any }): IJsonItem[] { + const { t } = useI18n() + const unCustomSpan = computed(() => (model.isCustomTask ? 0 : 24)) + const tableSpan = computed(() => + model.sourceType === 'MYSQL' && model.srcQueryType === '0' ? 24 : 0 + ) + const editorSpan = computed(() => + model.sourceType === 'MYSQL' && model.srcQueryType === '1' ? 24 : 0 + ) + const columnSpan = computed(() => + model.sourceType === 'MYSQL' && model.srcColumnType === '1' ? 24 : 0 + ) + const mysqlSpan = computed(() => (model.sourceType === 'MYSQL' ? 24 : 0)) + const hiveSpan = computed(() => (model.sourceType === 'HIVE' ? 24 : 0)) + const hdfsSpan = computed(() => (model.sourceType === 'HDFS' ? 24 : 0)) + const datasourceSpan = computed(() => (model.sourceType === 'MYSQL' ? 12 : 0)) + + const sourceTypes = ref([ + { + label: 'MYSQL', + value: 'MYSQL' + } + ] as IOption[]) + + const getSourceTypesByModelType = (modelType: ModelType): IOption[] => { + switch (modelType) { + case 'import': + return [ + { + label: 'MYSQL', + value: 'MYSQL' + } + ] + case 'export': + return [ + { + label: 'HDFS', + value: 'HDFS' + }, + { + label: 'HIVE', + value: 'HIVE' + } + ] + default: + return [ + { + label: 'MYSQL', + value: 'MYSQL' + }, + { + label: 'HDFS', + value: 'HDFS' + }, + { + label: 'HIVE', + value: 'HIVE' + } + ] + } + } + + watch( + () => model.modelType, + (modelType: ModelType) => { + getSourceTypesByModelType(modelType) + } + ) + + return [ + { + type: 'custom', + field: 'custom-title', + span: unCustomSpan, + widget: h( + 'div', + { class: styles['field-title'] }, + t('project.node.data_source') + ) + }, + { + type: 'select', + field: 'sourceType', + name: t('project.node.type'), + span: unCustomSpan, + options: sourceTypes + }, + ...useDatasource( + model, + datasourceSpan, + 'sourceMysqlType', + 'sourceMysqlDatasource' + ), + { + type: 'radio', + field: 'srcQueryType', + name: t('project.node.model_type'), + span: mysqlSpan, + options: [ + { + label: t('project.node.form'), + value: '0' + }, + { + label: 'SQL', + value: '1' + } + ] + }, + { + type: 'input', + field: 'srcTable', + name: t('project.node.table'), + span: tableSpan, + props: { + placeholder: t('project.node.table_tips') + }, + validate: { + trigger: ['input', 'blur'], + required: !!unref(tableSpan), + validator(validate, value) { + if (!!unref(tableSpan) && !value) { + return new Error(t('project.node.table_tips')) + } + } + } + }, + { + type: 'radio', + field: 'srcColumnType', + name: t('project.node.column_type'), + span: tableSpan, + options: [ + { label: t('project.node.all_columns'), value: '0' }, + { label: t('project.node.some_columns'), value: '1' } + ] + }, + { + type: 'input', + field: 'srcColumns', + name: t('project.node.column'), + span: columnSpan, + props: { + placeholder: t('project.node.column_tips') + }, + validate: { + trigger: ['input', 'blur'], + required: !!unref(columnSpan), + validator(validate, value) { + if (!!unref(columnSpan) && !value) { + return new Error(t('project.node.column_tips')) + } + } + } + }, + { + type: 'input', + field: 'sourceHiveDatabase', + name: t('project.node.database'), + span: hiveSpan, + props: { + placeholder: t('project.node.database_tips') + }, + validate: { + trigger: ['blur', 'input'], + required: !!unref(hiveSpan), + validator(validate, value) { + if (!!unref(hiveSpan) && !value) { + return new Error(t('project.node.database_tips')) + } + } + } + }, + { + type: 'input', + field: 'sourceHiveTable', + name: t('project.node.table'), + span: hiveSpan, + props: { + placeholder: t('project.node.hive_table_tips') + }, + validate: { + trigger: ['blur', 'input'], + required: !!unref(hiveSpan), + validator(validate, value) { + if (!!unref(hiveSpan) && !value) { + return new Error(t('project.node.hive_table_tips')) + } + } + } + }, + { + type: 'input', + field: 'sourceHivePartitionKey', + name: t('project.node.hive_partition_keys'), + span: hiveSpan, + props: { + placeholder: t('project.node.hive_partition_keys_tips') + } + }, + { + type: 'input', + field: 'sourceHivePartitionValue', + name: t('project.node.hive_partition_values'), + span: hiveSpan, + props: { + placeholder: t('project.node.hive_partition_values_tips') + } + }, + { + type: 'input', + field: 'sourceHdfsExportDir', + name: t('project.node.export_dir'), + span: hdfsSpan, + props: { + placeholder: t('project.node.export_dir_tips') + }, + validate: { + trigger: ['blur', 'input'], + required: !!unref(hdfsSpan), + validator(validate, value) { + if (!!unref(hdfsSpan) && !value) { + return new Error(t('project.node.export_dir_tips')) + } + } + } + }, + { + type: 'editor', + field: 'sourceMysqlSrcQuerySql', + name: t('project.node.sql_statement'), + span: editorSpan, + validate: { + trigger: ['blur', 'input'], + required: !!unref(editorSpan), + validator(validate, value) { + if (!!unref(editorSpan) && !value) { + return new Error(t('project.node.sql_statement_tips')) + } + } + } + }, + ...useCustomParams({ + model, + field: 'mapColumnHive', + name: 'map_column_hive', + isSimple: true, + span: editorSpan + }), + ...useCustomParams({ + model, + field: 'mapColumnJava', + name: 'map_column_java', + isSimple: true, + span: editorSpan + }) + ] +} diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-target-type.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-target-type.ts new file mode 100644 index 0000000000..2797511f3f --- /dev/null +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop-target-type.ts @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ref, h, watch, computed, unref } from 'vue' +import { useI18n } from 'vue-i18n' +import { useDatasource } from './use-sqoop-datasource' +import styles from '../index.module.scss' +import type { IJsonItem, IOption, SourceType } from '../types' + +export function useTargetType(model: { [field: string]: any }): IJsonItem[] { + const { t } = useI18n() + const unCustomSpan = computed(() => (model.isCustomTask ? 0 : 24)) + const hiveSpan = computed(() => (model.targetType === 'HIVE' ? 24 : 0)) + const hdfsSpan = computed(() => (model.targetType === 'HDFS' ? 24 : 0)) + const mysqlSpan = computed(() => (model.targetType === 'MYSQL' ? 24 : 0)) + const dataSourceSpan = computed(() => (model.targetType === 'MYSQL' ? 12 : 0)) + const updateSpan = computed(() => + model.targetType === 'MYSQL' && model.isUpdate ? 24 : 0 + ) + + const targetTypes = ref([ + { + label: 'HIVE', + value: 'HIVE' + }, + { + label: 'HDFS', + value: 'HDFS' + } + ] as IOption[]) + + const getTargetTypesBySourceType = ( + sourceType: SourceType, + srcQueryType: string + ): IOption[] => { + switch (sourceType) { + case 'MYSQL': + if (srcQueryType === '1') { + return [ + { + label: 'HDFS', + value: 'HDFS' + } + ] + } + return [ + { + label: 'HDFS', + value: 'HDFS' + }, + { + label: 'HIVE', + value: 'HIVE' + } + ] + case 'HDFS': + case 'HIVE': + return [ + { + label: 'MYSQL', + value: 'MYSQL' + } + ] + default: + return [ + { + label: 'HDFS', + value: 'HDFS' + }, + { + label: 'HIVE', + value: 'HIVE' + } + ] + } + } + + watch( + () => [model.sourceType, model.srcQueryType], + ([sourceType, srcQueryType]) => { + console.log(sourceType, srcQueryType) + getTargetTypesBySourceType(sourceType, srcQueryType) + } + ) + + return [ + { + type: 'custom', + field: 'custom-title', + span: unCustomSpan, + widget: h( + 'div', + { class: styles['field-title'] }, + t('project.node.data_target') + ) + }, + { + type: 'select', + field: 'targetType', + name: t('project.node.type'), + span: unCustomSpan, + options: targetTypes + }, + { + type: 'input', + field: 'targetHiveDatabase', + name: t('project.node.database'), + span: hiveSpan, + props: { + placeholder: t('project.node.database_tips') + }, + validate: { + trigger: ['blur', 'input'], + required: !!unref(hiveSpan), + validator(validate, value) { + if (!!unref(hiveSpan) && !value) { + return new Error(t('project.node.database_tips')) + } + } + } + }, + { + type: 'input', + field: 'targetHiveTable', + name: t('project.node.database'), + span: hiveSpan, + props: { + placeholder: t('project.node.table') + }, + validate: { + trigger: ['blur', 'input'], + required: !!unref(hiveSpan), + validator(rule, value) { + if (!!unref(hiveSpan) && !value) { + return new Error(t('project.node.hive_table_tips')) + } + } + } + }, + { + type: 'switch', + field: 'targetHiveCreateTable', + span: hiveSpan, + name: t('project.node.create_hive_table') + }, + { + type: 'switch', + field: 'targetHiveDropDelimiter', + span: hiveSpan, + name: t('project.node.drop_delimiter') + }, + { + type: 'switch', + field: 'targetHiveOverWrite', + span: hiveSpan, + name: t('project.node.over_write_src') + }, + { + type: 'input', + field: 'targetHiveTargetDir', + name: t('project.node.hive_target_dir'), + span: hiveSpan, + props: { + placeholder: t('project.node.hive_target_dir_tips') + } + }, + { + type: 'input', + field: 'targetHiveReplaceDelimiter', + name: t('project.node.replace_delimiter'), + span: hiveSpan, + props: { + placeholder: t('project.node.replace_delimiter_tips') + } + }, + { + type: 'input', + field: 'targetHivePartitionKey', + name: t('project.node.hive_partition_keys'), + span: hiveSpan, + props: { + placeholder: t('project.node.hive_partition_keys_tips') + } + }, + { + type: 'input', + field: 'targetHivePartitionValue', + name: t('project.node.hive_partition_values'), + span: hiveSpan, + props: { + placeholder: t('project.node.hive_partition_values_tips') + } + }, + { + type: 'input', + field: 'targetHdfsTargetPath', + name: t('project.node.target_dir'), + span: hdfsSpan, + props: { + placeholder: t('project.node.target_dir_tips') + }, + validate: { + trigger: ['blur', 'input'], + required: !!unref(hdfsSpan), + validator(rule, value) { + if (!!unref(hdfsSpan) && !value) { + return new Error(t('project.node.target_dir_tips')) + } + } + } + }, + { + type: 'switch', + field: 'targetHdfsDeleteTargetDir', + name: t('project.node.delete_target_dir'), + span: hdfsSpan + }, + { + type: 'radio', + field: 'targetHdfsCompressionCodec', + name: t('project.node.compression_codec'), + span: hdfsSpan, + options: COMPRESSIONCODECS + }, + { + type: 'radio', + field: 'targetHdfsFileType', + name: t('project.node.file_type'), + span: hdfsSpan, + options: FILETYPES + }, + { + type: 'input', + field: 'targetHdfsFieldsTerminated', + name: t('project.node.fields_terminated'), + span: hdfsSpan, + props: { + placeholder: t('project.node.fields_terminated_tips') + } + }, + { + type: 'input', + field: 'targetHdfsLinesTerminated', + name: t('project.node.lines_terminated'), + span: hdfsSpan, + props: { + placeholder: t('project.node.lines_terminated_tips') + } + }, + ...useDatasource( + model, + dataSourceSpan, + 'targetMysqlType', + 'targetMysqlDatasource' + ), + { + type: 'input', + field: 'targetMysqlTable', + name: t('project.node.table'), + span: mysqlSpan, + props: { + placeholder: t('project.node.hive_table_tips') + }, + validate: { + trigger: ['blur', 'input'], + required: !!unref(mysqlSpan), + validator(validate, value) { + if (!!unref(mysqlSpan) && !value) { + return new Error(t('project.node.table_tips')) + } + } + } + }, + { + type: 'input', + field: 'targetMysqlColumns', + name: t('project.node.column'), + span: mysqlSpan, + props: { + placeholder: t('project.node.column_tips') + } + }, + { + type: 'input', + field: 'targetMysqlFieldsTerminated', + name: t('project.node.fields_terminated'), + span: mysqlSpan, + props: { + placeholder: t('project.node.fields_terminated_tips') + } + }, + { + type: 'input', + field: 'targetMysqlLinesTerminated', + name: t('project.node.lines_terminated'), + span: mysqlSpan, + props: { + placeholder: t('project.node.lines_terminated_tips') + } + }, + { + type: 'switch', + field: 'targetMysqlIsUpdate', + span: mysqlSpan, + name: t('project.node.is_update') + }, + { + type: 'input', + field: 'targetMysqlTargetUpdateKey', + name: t('project.node.update_key'), + span: updateSpan, + props: { + placeholder: t('project.node.update_key_tips') + } + }, + { + type: 'radio', + field: 'targetMysqlUpdateMode', + name: t('project.node.update_mode'), + span: updateSpan, + options: [ + { + label: t('project.node.only_update'), + value: 'updateonly' + }, + { + label: t('project.node.allow_insert'), + value: 'allowinsert' + } + ] + } + ] +} + +const COMPRESSIONCODECS = [ + { + label: 'snappy', + value: 'snappy' + }, + { + label: 'lzo', + value: 'lzo' + }, + { + label: 'gzip', + value: 'gzip' + }, + { + label: 'no', + value: '' + } +] +const FILETYPES = [ + { + label: 'avro', + value: '--as-avrodatafile' + }, + { + label: 'sequence', + value: '--as-sequencefile' + }, + { + label: 'text', + value: '--as-textfile' + }, + { + label: 'parquet', + value: '--as-parquetfile' + } +] diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop.ts new file mode 100644 index 0000000000..a1a074c76a --- /dev/null +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sqoop.ts @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { watch, computed, unref } from 'vue' +import { useI18n } from 'vue-i18n' +import { useCustomParams, useSourceType, useTargetType } from '.' +import type { IJsonItem, ModelType } from '../types' + +export function useSqoop(model: { [field: string]: any }): IJsonItem[] { + const { t } = useI18n() + const customSpan = computed(() => (model.isCustomTask ? 24 : 0)) + const unCustomSpan = computed(() => (model.isCustomTask ? 0 : 24)) + + watch( + () => model.srcQueryType, + () => {} + ) + + return [ + { + type: 'switch', + field: 'isCustomTask', + name: t('project.node.custom_job') + }, + { + type: 'input', + field: 'jobName', + name: t('project.node.sqoop_job_name'), + span: unCustomSpan, + props: { + placeholder: t('project.node.sqoop_job_name_tips') + }, + validate: { + trigger: ['input', 'blur'], + required: !model.isCustomTask, + validator(validate, value) { + if (!model.isCustomTask && !value) { + return new Error(t('project.node.sqoop_job_name_tips')) + } + } + } + }, + { + type: 'select', + field: 'modelType', + name: t('project.node.direct'), + span: unCustomSpan, + options: MODEL_TYPES + }, + ...useCustomParams({ + model, + field: 'hadoopCustomParams', + name: 'hadoop_custom_params', + isSimple: true, + span: unCustomSpan + }), + ...useCustomParams({ + model, + field: 'sqoopAdvancedParams', + name: 'sqoop_advanced_parameters', + isSimple: true, + span: unCustomSpan + }), + ...useSourceType(model), + ...useTargetType(model), + { + type: 'input-number', + field: 'concurrency', + name: t('project.node.concurrency'), + span: unCustomSpan, + props: { + placeholder: t('project.node.concurrency_tips') + } + }, + { + type: 'editor', + field: 'customShell', + name: t('project.node.custom_script'), + span: customSpan, + validate: { + trigger: ['input', 'trigger'], + required: !!unref(customSpan), + validator(rule, value) { + if (!!unref(customSpan) && !value) { + return new Error(t('project.node.custom_script')) + } + } + } + }, + ...useCustomParams({ + model, + field: 'localParams', + name: 'custom_parameters', + isSimple: true + }) + ] +} + +const MODEL_TYPES = [ + { + label: 'import', + value: 'import' + }, + { + label: 'export', + value: 'export' + } +] as { label: ModelType; value: ModelType }[] diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts index 6d373b583c..0c2f0e3b26 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts @@ -16,7 +16,13 @@ */ import { omit } from 'lodash' -import type { INodeData, ITaskData, ITaskParams } from './types' +import type { + INodeData, + ITaskData, + ITaskParams, + ISqoopTargetParams, + ISqoopSourceParams +} from './types' export function formatParams(data: INodeData): { processDefinitionCode: string @@ -66,6 +72,99 @@ export function formatParams(data: INodeData): { taskParams.connectTimeout = data.connectTimeout taskParams.socketTimeout = data.socketTimeout } + if (data.taskType === 'SQOOP') { + taskParams.jobType = data.isCustomTask ? 'CUSTOM' : 'TEMPLATE' + taskParams.localParams = data.localParams + if (data.isCustomTask) { + taskParams.customShell = data.customShell + } else { + taskParams.jobName = data.jobName + taskParams.hadoopCustomParams = data.hadoopCustomParams + taskParams.sqoopAdvancedParams = data.sqoopAdvancedParams + taskParams.concurrency = data.concurrency + taskParams.modelType = data.modelType + taskParams.sourceType = data.sourceType + taskParams.targetType = data.targetType + let targetParams: ISqoopTargetParams = {} + let sourceParams: ISqoopSourceParams = {} + switch (data.targetType) { + case 'HIVE': + targetParams = { + hiveDatabase: data.targetHiveDatabase, + hiveTable: data.targetHiveTable, + createHiveTable: data.targetHiveCreateTable, + dropDelimiter: data.targetHiveDropDelimiter, + hiveOverWrite: data.targetHiveOverWrite, + hiveTargetDir: data.targetHiveTargetDir, + replaceDelimiter: data.targetHiveReplaceDelimiter, + hivePartitionKey: data.targetHivePartitionKey, + hivePartitionValue: data.targetHivePartitionValue + } + break + case 'HDFS': + targetParams = { + targetPath: data.targetHdfsTargetPath, + deleteTargetDir: data.targetHdfsDeleteTargetDir, + compressionCodec: data.targetHdfsCompressionCodec, + fileType: data.targetHdfsFileType, + fieldsTerminated: data.targetHdfsFieldsTerminated, + linesTerminated: data.targetHdfsLinesTerminated + } + break + case 'MYSQL': + targetParams = { + targetType: data.targetMysqlType, + targetDatasource: data.targetMysqlDatasource, + targetTable: data.targetMysqlTable, + targetColumns: data.targetMysqlColumns, + fieldsTerminated: data.targetMysqlFieldsTerminated, + linesTerminated: data.targetMysqlLinesTerminated, + isUpdate: data.targetMysqlIsUpdate, + targetUpdateKey: data.targetMysqlTargetUpdateKey, + targetUpdateMode: data.targetMysqlUpdateMode + } + break + default: + break + } + switch (data.sourceType) { + case 'MYSQL': + sourceParams = { + srcTable: data.srcQueryType === '1' ? '' : data.srcTable, + srcColumnType: data.srcQueryType === '1' ? '0' : data.srcColumnType, + srcColumns: + data.srcQueryType === '1' || data.srcColumnType === '0' + ? '' + : data.srcColumns, + srcQuerySql: + data.srcQueryType === '0' ? '' : data.sourceMysqlSrcQuerySql, + srcQueryType: data.srcQueryType, + srcType: data.sourceMysqlType, + srcDatasource: data.sourceMysqlDatasource, + mapColumnHive: data.mapColumnHive, + mapColumnJava: data.mapColumnJava + } + break + case 'HDFS': + sourceParams = { + exportDir: data.sourceHdfsExportDir + } + break + case 'HIVE': + sourceParams = { + hiveDatabase: data.sourceHiveDatabase, + hiveTable: data.sourceHiveTable, + hivePartitionKey: data.sourceHivePartitionKey, + hivePartitionValue: data.sourceHivePartitionValue + } + break + default: + break + } + taskParams.targetParams = JSON.stringify(targetParams) + taskParams.sourceParams = JSON.stringify(sourceParams) + } + } if (data.taskType === 'SQL') { taskParams.type = data.type @@ -162,6 +261,54 @@ export function formatModel(data: ITaskData) { if (data.taskParams?.method) { params.method = data.taskParams?.method } + if (data.taskParams?.targetParams) { + const targetParams: ISqoopTargetParams = JSON.parse( + data.taskParams.targetParams + ) + params.targetHiveDatabase = targetParams.hiveDatabase + params.targetHiveTable = targetParams.hiveTable + params.targetHiveCreateTable = targetParams.createHiveTable + params.targetHiveDropDelimiter = targetParams.dropDelimiter + params.targetHiveOverWrite = targetParams.hiveOverWrite + params.targetHiveTargetDir = targetParams.hiveTargetDir + params.targetHiveReplaceDelimiter = targetParams.replaceDelimiter + params.targetHivePartitionKey = targetParams.hivePartitionKey + params.targetHivePartitionValue = targetParams.hivePartitionValue + params.targetHdfsTargetPath = targetParams.targetPath + params.targetHdfsDeleteTargetDir = targetParams.deleteTargetDir + params.targetHdfsCompressionCodec = targetParams.compressionCodec + params.targetHdfsFileType = targetParams.fileType + params.targetHdfsFieldsTerminated = targetParams.fieldsTerminated + params.targetHdfsLinesTerminated = targetParams.linesTerminated + params.targetMysqlType = targetParams.targetType + params.targetMysqlDatasource = targetParams.targetDatasource + params.targetMysqlTable = targetParams.targetTable + params.targetMysqlColumns = targetParams.targetColumns + params.targetMysqlFieldsTerminated = targetParams.fieldsTerminated + params.targetMysqlLinesTerminated = targetParams.linesTerminated + params.targetMysqlIsUpdate = targetParams.isUpdate + params.targetMysqlTargetUpdateKey = targetParams.targetUpdateKey + params.targetMysqlUpdateMode = targetParams.targetUpdateMode + } + if (data.taskParams?.sourceParams) { + const sourceParams: ISqoopSourceParams = JSON.parse( + data.taskParams.sourceParams + ) + params.srcTable = sourceParams.srcTable + params.srcColumnType = sourceParams.srcColumnType + params.srcColumns = sourceParams.srcColumns + params.sourceMysqlSrcQuerySql = sourceParams.srcQuerySql + params.srcQueryType = sourceParams.srcQueryType + params.sourceMysqlType = sourceParams.srcType + params.sourceMysqlDatasource = sourceParams.srcDatasource + params.mapColumnHive = sourceParams.mapColumnHive + params.mapColumnJava = sourceParams.mapColumnJava + params.sourceHdfsExportDir = sourceParams.exportDir + params.sourceHiveDatabase = sourceParams.hiveDatabase + params.sourceHiveTable = sourceParams.hiveTable + params.sourceHivePartitionKey = sourceParams.hivePartitionKey + params.sourceHivePartitionValue = sourceParams.hivePartitionValue + } return params } diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/index.module.scss b/dolphinscheduler-ui-next/src/views/projects/task/components/node/index.module.scss new file mode 100644 index 0000000000..5d4d0c30e9 --- /dev/null +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/index.module.scss @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.field-title { + font-weight: bold; + width: 100%; + + &::before { + content: ''; + display: inline-block; + vertical-align: -2px; + width: 4px; + height: 1em; + background-color: var(--n-color); + border-radius: 4px; + margin-right: 8px; + } +} diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-shell.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-shell.ts index 5c4b99d7aa..fefec0339b 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-shell.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-shell.ts @@ -31,6 +31,7 @@ export function useShell({ data?: ITaskData }) { const model = reactive({ + taskType: 'SHELL', name: '', flag: 'YES', description: '', diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-sqoop.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-sqoop.ts new file mode 100644 index 0000000000..8efe6df8bd --- /dev/null +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-sqoop.ts @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { reactive } from 'vue' +import * as Fields from '../fields/index' +import type { IJsonItem, INodeData, ITaskData } from '../types' + +export function useSqoop({ + projectCode, + from = 0, + readonly, + data +}: { + projectCode: number + from?: number + readonly?: boolean + data?: ITaskData +}) { + const model = reactive({ + taskType: 'SQOOP', + name: '', + flag: 'YES', + description: '', + timeoutFlag: false, + localParams: [], + environmentCode: null, + failRetryInterval: 1, + failRetryTimes: 0, + workerGroup: 'default', + delayTime: 0, + timeout: 30, + isCustomTask: false, + hadoopCustomParams: [], + sqoopAdvancedParams: [], + mapColumnHive: [], + mapColumnJava: [], + modelType: 'import', + sourceType: 'MYSQL', + srcQueryType: '1', + srcColumnType: '0', + targetType: 'HDFS', + sourceMysqlType: 'MYSQL', + targetHdfsDeleteTargetDir: true, + targetHdfsCompressionCodec: 'snappy', + targetHdfsFileType: '--as-avrodatafile', + targetMysqlType: 'MYSQL', + targetMysqlUpdateMode: 'allowinsert', + targetHiveCreateTable: false, + targetHiveDropDelimiter: false, + targetHiveOverWrite: true, + concurrency: 1 + } as INodeData) + + let extra: IJsonItem[] = [] + if (from === 1) { + extra = [ + Fields.useTaskType(model, readonly), + Fields.useProcessName({ + model, + projectCode, + isCreate: !data?.id, + from, + processName: data?.processName, + code: data?.code + }) + ] + } + + return { + json: [ + Fields.useName(), + ...extra, + Fields.useRunFlag(), + Fields.useDescription(), + Fields.useTaskPriority(), + Fields.useWorkerGroup(), + Fields.useEnvironmentName(model, !data?.id), + ...Fields.useTaskGroup(model, projectCode), + ...Fields.useFailed(), + Fields.useDelayTime(model), + ...Fields.useTimeoutAlarm(model), + ...Fields.useSqoop(model), + Fields.usePreTasks(model) + ] as IJsonItem[], + model + } +} diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts index ba537bc1ab..4385a68912 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts @@ -19,6 +19,16 @@ import { VNode } from 'vue' import type { SelectOption } from 'naive-ui' import type { IFormItem, IJsonItem } from '@/components/form/types' import type { TaskType } from '@/views/projects/task/constants/task-type' +import type { IDataBase } from '@/service/modules/data-source/types' + +type ProgramType = 'JAVA' | 'SCALA' | 'PYTHON' +type SourceType = 'MYSQL' | 'HDFS' | 'HIVE' +type ModelType = 'import' | 'export' + +interface IOption { + label: string + value: string | number +} interface ITaskPriorityOption extends SelectOption { icon: VNode @@ -29,10 +39,11 @@ interface IEnvironmentNameOption { value: string workerGroups?: string[] } + interface ILocalParam { prop: string - direct: string - type: string + direct?: string + type?: string value?: string } @@ -40,6 +51,91 @@ interface ISourceItem { id: number } +interface ISqoopTargetData { + targetHiveDatabase?: string + targetHiveTable?: string + targetHiveCreateTable?: boolean + targetHiveDropDelimiter?: boolean + targetHiveOverWrite?: boolean + targetHiveTargetDir?: string + targetHiveReplaceDelimiter?: string + targetHivePartitionKey?: string + targetHivePartitionValue?: string + targetHdfsTargetPath?: string + targetHdfsDeleteTargetDir?: boolean + targetHdfsCompressionCodec?: string + targetHdfsFileType?: string + targetHdfsFieldsTerminated?: string + targetHdfsLinesTerminated?: string + targetMysqlType?: string + targetMysqlDatasource?: string + targetMysqlTable?: string + targetMysqlColumns?: string + targetMysqlFieldsTerminated?: string + targetMysqlLinesTerminated?: string + targetMysqlIsUpdate?: string + targetMysqlTargetUpdateKey?: string + targetMysqlUpdateMode?: string +} + +interface ISqoopSourceData { + srcQueryType?: '1' | '0' + srcTable?: string + srcColumnType?: '1' | '0' + srcColumns?: string + sourceMysqlSrcQuerySql?: string + sourceMysqlType?: string + sourceMysqlDatasource?: string + mapColumnHive?: ILocalParam[] + mapColumnJava?: ILocalParam[] + sourceHdfsExportDir?: string + sourceHiveDatabase?: string + sourceHiveTable?: string + sourceHivePartitionKey?: string + sourceHivePartitionValue?: string +} + +interface ISqoopTargetParams { + hiveDatabase?: string + hiveTable?: string + createHiveTable?: boolean + dropDelimiter?: boolean + hiveOverWrite?: boolean + hiveTargetDir?: string + replaceDelimiter?: string + hivePartitionKey?: string + hivePartitionValue?: string + targetPath?: string + deleteTargetDir?: boolean + compressionCodec?: string + fileType?: string + fieldsTerminated?: string + linesTerminated?: string + targetType?: string + targetDatasource?: string + targetTable?: string + targetColumns?: string + isUpdate?: string + targetUpdateKey?: string + targetUpdateMode?: string +} +interface ISqoopSourceParams { + srcTable?: string + srcColumnType?: '1' | '0' + srcColumns?: string + srcQuerySql?: string + srcQueryType?: '1' | '0' + srcType?: string + srcDatasource?: string + mapColumnHive?: ILocalParam[] + mapColumnJava?: ILocalParam[] + exportDir?: string + hiveDatabase?: string + hiveTable?: string + hivePartitionKey?: string + hivePartitionValue?: string +} + interface ITaskParams { resourceList?: ISourceItem[] mainJar?: ISourceItem @@ -77,11 +173,28 @@ interface ITaskParams { preStatements?: string[] postStatements?: string[] method?: string + jobType?: 'CUSTOM' | 'TEMPLATE' + customShell?: string + jobName?: string + hadoopCustomParams?: ILocalParam[] + sqoopAdvancedParams?: ILocalParam[] + concurrency?: number + modelType?: ModelType + sourceType?: SourceType + targetType?: SourceType + targetParams?: string + sourceParams?: string } type ITaskType = TaskType -interface INodeData extends Omit { +interface INodeData + extends Omit< + ITaskParams, + 'resourceList' | 'mainJar' | 'targetParams' | 'sourceParams' + >, + ISqoopTargetData, + ISqoopSourceData { id?: string taskType?: ITaskType processName?: number @@ -106,13 +219,7 @@ interface INodeData extends Omit { resourceList?: number[] mainJar?: number timeoutSetting?: boolean - type?: string - datasource?: string - sql?: string - sqlType?: string - preStatements?: string[] - postStatements?: string[] - method?: string + isCustomTask?: boolean } interface ITaskData @@ -136,5 +243,12 @@ export { INodeData, IFormItem, IJsonItem, - ITaskParams + ITaskParams, + IOption, + IDataBase, + ProgramType, + ModelType, + SourceType, + ISqoopSourceParams, + ISqoopTargetParams } diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts index 69618b24c9..a8eb2f0291 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts @@ -25,6 +25,7 @@ import { useMr } from './tasks/use-mr' import { useHttp } from './tasks/use-http' import { useSql } from './tasks/use-sql' import { useProcedure } from './tasks/use-procedure' +import { useSqoop } from './tasks/use-sqoop' import { IJsonItem, INodeData, ITaskData } from './types' export function useTask({ @@ -120,5 +121,14 @@ export function useTask({ data }) } + if (taskType === 'SQOOP') { + node = useSqoop({ + projectCode, + from, + readonly, + data + }) + } + return node }