From 79733643aab64af78c13f59fcaf2f9b3a4e3a068 Mon Sep 17 00:00:00 2001 From: Amy0104 <97265214+Amy0104@users.noreply.github.com> Date: Fri, 4 Mar 2022 18:29:12 +0800 Subject: [PATCH] [Feature-8565][Fix-8672][UI Next][V1.0.0-Alpha]: Add DATA_QUALITY and modify task type switching bug. (#8698) --- .../src/components/form/fields/get-field.ts | 16 ++ .../src/components/form/index.tsx | 1 + .../src/components/form/types.ts | 3 +- .../src/locales/modules/en_US.ts | 45 +++- .../src/locales/modules/zh_CN.ts | 45 +++- .../src/service/modules/data-quality/index.ts | 27 +++ .../src/service/modules/data-source/index.ts | 24 ++ .../task/components/node/detail-modal.tsx | 71 +++--- .../projects/task/components/node/detail.tsx | 90 +++----- .../task/components/node/fields/index.ts | 7 + .../components/node/fields/use-deploy-mode.ts | 45 ++++ .../node/fields/use-driver-cores.ts | 42 ++++ .../node/fields/use-driver-memory.ts | 47 ++++ .../node/fields/use-executor-cores.ts | 42 ++++ .../node/fields/use-executor-memory.ts | 47 ++++ .../node/fields/use-executor-number.ts | 42 ++++ .../task/components/node/fields/use-flink.ts | 20 +- .../task/components/node/fields/use-http.ts | 4 +- .../task/components/node/fields/use-rules.ts | 214 ++++++++++++++++++ .../components/node/fields/use-sea-tunnel.ts | 24 +- .../task/components/node/fields/use-spark.ts | 146 ++---------- .../task/components/node/format-data.ts | 53 +++++ .../task/components/node/tasks/index.ts | 54 +++++ .../components/node/tasks/use-data-quality.ts | 116 ++++++++++ .../task/components/node/tasks/use-flink.ts | 1 + .../task/components/node/tasks/use-pigeon.ts | 1 + .../components/node/tasks/use-sub-process.ts | 1 + .../projects/task/components/node/types.ts | 58 ++++- .../projects/task/components/node/use-task.ts | 181 +++------------ .../projects/task/constants/task-type.ts | 3 + .../projects/task/definition/use-task.ts | 2 +- 31 files changed, 1038 insertions(+), 434 deletions(-) create mode 100644 dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-deploy-mode.ts create mode 100644 dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-driver-cores.ts create mode 100644 dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-driver-memory.ts create mode 100644 dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-executor-cores.ts create mode 100644 dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-executor-memory.ts create mode 100644 dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-executor-number.ts create mode 100644 dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-rules.ts create mode 100644 dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/index.ts create mode 100644 dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-data-quality.ts diff --git a/dolphinscheduler-ui-next/src/components/form/fields/get-field.ts b/dolphinscheduler-ui-next/src/components/form/fields/get-field.ts index 94139bb1b4..8508985ec9 100644 --- a/dolphinscheduler-ui-next/src/components/form/fields/get-field.ts +++ b/dolphinscheduler-ui-next/src/components/form/fields/get-field.ts @@ -19,12 +19,28 @@ import { camelCase, upperFirst, isFunction } from 'lodash' import type { FormRules, FormItemRule } from 'naive-ui' import type { IJsonItem } from '../types' +const TYPES = [ + 'input', + 'radio', + 'editor', + 'custom-parameters', + 'switch', + 'input-number', + 'select', + 'checkbox', + 'tree-select', + 'multi-input', + 'custom', + 'multi-condition' +] + const getField = ( item: IJsonItem, fields: { [field: string]: any }, rules?: FormRules ) => { const { type = 'input', widget, field } = isFunction(item) ? item() : item + if (!TYPES.includes(type)) return null const renderTypeName = `render${upperFirst(camelCase(type))}` if (type === 'custom') { return widget || null diff --git a/dolphinscheduler-ui-next/src/components/form/index.tsx b/dolphinscheduler-ui-next/src/components/form/index.tsx index 66829dbdb3..d4962df889 100644 --- a/dolphinscheduler-ui-next/src/components/form/index.tsx +++ b/dolphinscheduler-ui-next/src/components/form/index.tsx @@ -60,6 +60,7 @@ const Form = defineComponent({ {...formItemProps} span={unref(span) === void 0 ? 24 : unref(span)} path={path} + key={path} > {h(widget)} diff --git a/dolphinscheduler-ui-next/src/components/form/types.ts b/dolphinscheduler-ui-next/src/components/form/types.ts index 8f7c266a62..c1a7d8a77f 100644 --- a/dolphinscheduler-ui-next/src/components/form/types.ts +++ b/dolphinscheduler-ui-next/src/components/form/types.ts @@ -85,5 +85,6 @@ export { FormItemRule, FormRules, IFormItem, - GridProps + GridProps, + IJsonItemParams } diff --git a/dolphinscheduler-ui-next/src/locales/modules/en_US.ts b/dolphinscheduler-ui-next/src/locales/modules/en_US.ts index 376a058672..e0e68a4c32 100644 --- a/dolphinscheduler-ui-next/src/locales/modules/en_US.ts +++ b/dolphinscheduler-ui-next/src/locales/modules/en_US.ts @@ -787,7 +787,6 @@ const project = { allow_insert: 'AllowInsert', concurrency: 'Concurrency', concurrency_tips: 'Please enter Concurrency', - sea_tunnel_deploy_mode: 'Deploy Mode', sea_tunnel_master: 'Master', sea_tunnel_master_url: 'Master URL', sea_tunnel_queue: 'Queue', @@ -844,7 +843,49 @@ const project = { add_dependency: 'Add dependency', waiting_dependent_start: 'Waiting Dependent start', check_interval: 'Check interval', - waiting_dependent_complete: 'Waiting Dependent complete' + waiting_dependent_complete: 'Waiting Dependent complete', + rule_name: 'Rule Name', + null_check: 'NullCheck', + custom_sql: 'CustomSql', + multi_table_accuracy: 'MulTableAccuracy', + multi_table_value_comparison: 'MulTableCompare', + field_length_check: 'FieldLengthCheck', + uniqueness_check: 'UniquenessCheck', + regexp_check: 'RegexpCheck', + timeliness_check: 'TimelinessCheck', + enumeration_check: 'EnumerationCheck', + table_count_check: 'TableCountCheck', + src_connector_type: 'SrcConnType', + src_datasource_id: 'SrcSource', + src_table: 'SrcTable', + src_filter: 'SrcFilter', + src_field: 'SrcField', + statistics_name: 'ActualValName', + check_type: 'CheckType', + operator: 'Operator', + threshold: 'Threshold', + failure_strategy: 'FailureStrategy', + target_connector_type: 'TargetConnType', + target_datasource_id: 'TargetSourceId', + target_table: 'TargetTable', + target_filter: 'TargetFilter', + mapping_columns: 'OnClause', + statistics_execute_sql: 'ActualValExecSql', + comparison_name: 'ExceptedValName', + comparison_execute_sql: 'ExceptedValExecSql', + comparison_type: 'ExceptedValType', + writer_connector_type: 'WriterConnType', + writer_datasource_id: 'WriterSourceId', + target_field: 'TargetField', + field_length: 'FieldLength', + logic_operator: 'LogicOperator', + regexp_pattern: 'RegexpPattern', + deadline: 'Deadline', + datetime_format: 'DatetimeFormat', + enum_list: 'EnumList', + begin_time: 'BeginTime', + fix_value: 'FixValue', + required: 'required' } } diff --git a/dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts b/dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts index 6c633cd675..08a31f9f7b 100644 --- a/dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts +++ b/dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts @@ -778,7 +778,6 @@ const project = { allow_insert: '无更新便插入', concurrency: '并发度', concurrency_tips: '请输入并发度', - sea_tunnel_deploy_mode: '部署方式', sea_tunnel_master: 'Master', sea_tunnel_master_url: 'Master URL', sea_tunnel_queue: '队列', @@ -834,7 +833,49 @@ const project = { add_dependency: '添加依赖', waiting_dependent_start: '等待依赖启动', check_interval: '检查间隔', - waiting_dependent_complete: '等待依赖完成' + waiting_dependent_complete: '等待依赖完成', + rule_name: '规则名称', + null_check: '空值检测', + custom_sql: '自定义SQL', + multi_table_accuracy: '多表准确性', + multi_table_value_comparison: '两表值比对', + field_length_check: '字段长度校验', + uniqueness_check: '唯一性校验', + regexp_check: '正则表达式', + timeliness_check: '及时性校验', + enumeration_check: '枚举值校验', + table_count_check: '表行数校验', + src_connector_type: '源数据类型', + src_datasource_id: '源数据源', + src_table: '源数据表', + src_filter: '源表过滤条件', + src_field: '源表检测列', + statistics_name: '实际值名', + check_type: '校验方式', + operator: '校验操作符', + threshold: '阈值', + failure_strategy: '失败策略', + target_connector_type: '目标数据类型', + target_datasource_id: '目标数据源', + target_table: '目标数据表', + target_filter: '目标表过滤条件', + mapping_columns: 'ON语句', + statistics_execute_sql: '实际值计算SQL', + comparison_name: '期望值名', + comparison_execute_sql: '期望值计算SQL', + comparison_type: '期望值类型', + writer_connector_type: '输出数据类型', + writer_datasource_id: '输出数据源', + target_field: '目标表检测列', + field_length: '字段长度限制', + logic_operator: '逻辑操作符', + regexp_pattern: '正则表达式', + deadline: '截止时间', + datetime_format: '时间格式', + enum_list: '枚举值列表', + begin_time: '起始时间', + fix_value: '固定值', + required: '必填' } } diff --git a/dolphinscheduler-ui-next/src/service/modules/data-quality/index.ts b/dolphinscheduler-ui-next/src/service/modules/data-quality/index.ts index 2250b633a1..c58a62e910 100644 --- a/dolphinscheduler-ui-next/src/service/modules/data-quality/index.ts +++ b/dolphinscheduler-ui-next/src/service/modules/data-quality/index.ts @@ -33,3 +33,30 @@ export function queryExecuteResultListPaging(params: ResultListReq): any { params }) } + +export function queryRuleList(): any { + return axios({ + url: '/data-quality/ruleList', + method: 'get' + }) +} + +export function getRuleFormCreateJson(ruleId: number): any { + return axios({ + url: '/data-quality/getRuleFormCreateJson', + method: 'get', + params: { + ruleId + } + }) +} + +export function getDatasourceOptionsById(datasourceId: number): any { + return axios({ + url: '/data-quality/getDatasourceOptionsById', + method: 'get', + params: { + datasourceId + } + }) +} diff --git a/dolphinscheduler-ui-next/src/service/modules/data-source/index.ts b/dolphinscheduler-ui-next/src/service/modules/data-source/index.ts index c24fa821cc..956ff3d3fb 100644 --- a/dolphinscheduler-ui-next/src/service/modules/data-source/index.ts +++ b/dolphinscheduler-ui-next/src/service/modules/data-source/index.ts @@ -128,3 +128,27 @@ export function connectionTest(id: IdReq): any { method: 'get' }) } + +export function getDatasourceTablesById(datasourceId: number): any { + return axios({ + url: '/datasources/tables', + method: 'get', + params: { + datasourceId + } + }) +} + +export function getDatasourceTableColumnsById( + datasourceId: number, + tableName: string +): any { + return axios({ + url: '/datasources/tableColumns', + method: 'get', + params: { + datasourceId, + tableName + } + }) +} diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/detail-modal.tsx b/dolphinscheduler-ui-next/src/views/projects/task/components/node/detail-modal.tsx index c69c56bfbf..c547a403a0 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/detail-modal.tsx +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/detail-modal.tsx @@ -20,15 +20,17 @@ import { PropType, ref, reactive, - toRefs, watch, - nextTick + nextTick, + provide, + computed } from 'vue' import { useI18n } from 'vue-i18n' +import { omit } from 'lodash' import Modal from '@/components/modal' import Detail from './detail' import { formatModel } from './format-data' -import type { ITaskData } from './types' +import type { ITaskData, ITaskType } from './types' const props = { show: { @@ -59,17 +61,17 @@ const NodeDetailModal = defineComponent({ emits: ['cancel', 'submit'], setup(props, { emit }) { const { t } = useI18n() + const detailRef = ref() const state = reactive({ saving: false, - detailRef: ref(), linkEventShowRef: ref(), linkEventTextRef: ref(), linkUrlRef: ref() }) const onConfirm = async () => { - await state.detailRef.form.validate() - emit('submit', { data: state.detailRef.form.getValues() }) + await detailRef.value.value.validate() + emit('submit', { data: detailRef.value.value.getValues() }) } const onCancel = () => { emit('cancel') @@ -85,54 +87,45 @@ const NodeDetailModal = defineComponent({ state.linkUrlRef = url } + const onTaskTypeChange = (taskType: ITaskType) => { + props.data.taskType = taskType + } + + provide( + 'data', + computed(() => ({ + projectCode: props.projectCode, + data: props.data, + from: props.from, + readonly: props.readonly + })) + ) + watch( () => props.data, async () => { + if (!props.show) return await nextTick() - state.detailRef.form.setValues(formatModel(props.data)) + detailRef.value.value.setValues(formatModel(props.data)) } ) - return { - t, - ...toRefs(state), - getLinkEventText, - onConfirm, - onCancel, - onJumpLink - } - }, - render() { - const { - t, - show, - onConfirm, - onCancel, - projectCode, - data, - readonly, - from, - onJumpLink - } = this - return ( + return () => ( ) diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/detail.tsx b/dolphinscheduler-ui-next/src/views/projects/task/components/node/detail.tsx index 237f524486..0392ced341 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/detail.tsx +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/detail.tsx @@ -15,92 +15,58 @@ * limitations under the License. */ -import { defineComponent, PropType, ref, watch } from 'vue' +import { defineComponent, ref, watch, inject, Ref, unref } from 'vue' import Form from '@/components/form' import { useTask } from './use-task' -import getElementByJson from '@/components/form/get-elements-by-json' import type { ITaskData } from './types' -import { useI18n } from 'vue-i18n' -const props = { - projectCode: { - type: Number as PropType, - default: 0 - }, - data: { - type: Object as PropType, - default: { taskType: 'SHELL' } - }, - readonly: { - type: Boolean as PropType, - default: false - }, - loading: { - type: Boolean as PropType, - default: false - }, - from: { - type: Number as PropType, - default: 0 - } +interface IDetailPanel { + projectCode: number + data: ITaskData + readonly: false + from: number + detailRef?: Ref } const NodeDetail = defineComponent({ name: 'NodeDetail', - props, - emits: ['linkEventText'], + emits: ['taskTypeChange'], setup(props, { expose, emit }) { - const { data, projectCode, from, readonly } = props - const { t } = useI18n() + const formRef = ref() + const detailData: IDetailPanel = inject('data') || { + projectCode: 0, + data: { + taskType: 'SHELL' + }, + readonly: false, + from: 0 + } + const { data, projectCode, from, readonly } = unref(detailData) - const { json, model } = useTask({ + const { elementsRef, rulesRef, model } = useTask({ data, projectCode, from, readonly }) - - const jsonRef = ref(json) - const formRef = ref() - - const { rules, elements } = getElementByJson(jsonRef.value, model) - - expose({ - form: formRef - }) - watch( () => model.taskType, - (taskType) => { - // TODO: Change task type - if (taskType === 'SUB_PROCESS') { - // TODO: add linkUrl - emit( - 'linkEventText', - true, - `${t('project.node.enter_child_node')}`, - '' - ) - } else { - emit('linkEventText', false, '', '') - } + async (taskType) => { + emit('taskTypeChange', taskType) } ) - return { rules, elements, model, formRef } - }, - render(props: { readonly: boolean; loading: boolean }) { - const { rules, elements, model } = this - return ( + expose(formRef) + + return () => (
=1.10' } ] - -const DeployModes = [ - { - label: 'cluster', - value: 'cluster' - }, - { - label: 'local', - value: 'local' - } -] diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-http.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-http.ts index 963cb19b78..9f21ba75a4 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-http.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-http.ts @@ -23,7 +23,7 @@ export function useHttp(model: { [field: string]: any }): IJsonItem[] { const { t } = useI18n() const timeoutSpan = computed(() => (model.timeoutSetting ? 12 : 0)) - const HTTP_CHECK_CONDITIONs = [ + const HTTP_CHECK_CONDITIONS = [ { label: t('project.node.status_code_default'), value: 'STATUS_CODE_DEFAULT' @@ -129,7 +129,7 @@ export function useHttp(model: { [field: string]: any }): IJsonItem[] { type: 'select', field: 'httpCheckCondition', name: t('project.node.http_check_condition'), - options: HTTP_CHECK_CONDITIONs + options: HTTP_CHECK_CONDITIONS }, { type: 'input', diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-rules.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-rules.ts new file mode 100644 index 0000000000..51653eb950 --- /dev/null +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-rules.ts @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ref, onMounted } from 'vue' +import { useI18n } from 'vue-i18n' +import { + queryRuleList, + getRuleFormCreateJson, + getDatasourceOptionsById +} from '@/service/modules/data-quality' +import { + getDatasourceTablesById, + getDatasourceTableColumnsById +} from '@/service/modules/data-source' +import type { IJsonItem, IResponseJsonItem, IJsonItemParams } from '../types' + +export function useRules( + model: { [field: string]: any }, + updateRules: (items: IJsonItem[], len: number) => void +): IJsonItem[] { + const { t } = useI18n() + const rules = ref([]) + const ruleLoading = ref(false) + const srcDatasourceOptions = ref([] as { label: string; value: number }[]) + const srcTableOptions = ref([] as { label: string; value: number }[]) + const srcTableColumnOptions = ref([] as { label: string; value: number }[]) + const targetDatasourceOptions = ref([] as { label: string; value: number }[]) + const targetTableOptions = ref([] as { label: string; value: string }[]) + const targetTableColumnOptions = ref([] as { label: string; value: number }[]) + const writerDatasourceOptions = ref([] as { label: string; value: number }[]) + + let preItemLen = 0 + + const getRuleList = async () => { + if (ruleLoading.value) return + ruleLoading.value = true + try { + const result = await queryRuleList() + rules.value = result.map((item: { id: number; name: string }) => { + let name = '' + if (item.name) { + name = item.name.replace('$t(', '').replace(')', '') + } + return { + value: item.id, + label: name ? t(`project.node.${name}`) : '' + } + }) + ruleLoading.value = false + } catch (err) { + ruleLoading.value = false + } + } + + const getRuleById = async (ruleId: number) => { + if (ruleLoading.value) return + ruleLoading.value = true + try { + const result = await getRuleFormCreateJson(ruleId) + const items = JSON.parse(result).map((item: IResponseJsonItem) => + formatResponseJson(item) + ) + updateRules(items, preItemLen) + preItemLen = items.length + ruleLoading.value = false + } catch (err) { + ruleLoading.value = false + } + } + + const formatResponseJson = ( + responseItem: IResponseJsonItem + ): IJsonItemParams => { + const item: IJsonItemParams = { + field: responseItem.field, + options: responseItem.options, + validate: responseItem.validate, + props: responseItem.props + } + let name = responseItem.name?.replace('$t(', '').replace(')', '') + item.name = name ? t(`project.node.${name}`) : '' + + if (responseItem.type !== 'group') { + item.type = responseItem.type + } else { + item.type = 'custom-parameters' + item.children = item.props.rules.map((child: IJsonItemParams) => { + child.span = Math.floor(22 / item.props.rules.length) + return child + }) + model[item.field] = [] + delete item.props.rules + } + if (responseItem.emit) { + responseItem.emit.forEach((emit) => { + if (emit === 'change') { + item.props.onUpdateValue = (value: string | number) => { + onFieldChange(value, item.field) + } + } + }) + } + if (item.field === 'src_datasource_id') { + item.options = srcDatasourceOptions + } + if (item.field === 'target_datasource_id') { + item.options = targetDatasourceOptions + } + if (item.field === 'writer_datasource_id') { + item.options = writerDatasourceOptions + } + if (item.field === 'src_table') { + item.options = srcTableOptions + } + if (item.field === 'target_table') { + item.options = targetTableOptions + } + if (item.field === 'src_field') { + item.options = srcTableColumnOptions + } + if (item.field === 'target_field') { + item.options = targetTableColumnOptions + } + return item + } + + const onFieldChange = async (value: string | number, field: string) => { + try { + if (field === 'src_connector_type' && typeof value === 'number') { + const result = await getDatasourceOptionsById(value) + srcDatasourceOptions.value = result || [] + srcTableOptions.value = [] + model.src_datasource_id = null + model.src_table = null + model.src_field = null + return + } + if (field === 'target_connector_type' && typeof value === 'number') { + const result = await getDatasourceOptionsById(value) + targetDatasourceOptions.value = result || [] + targetTableOptions.value = [] + model.target_datasource_id = null + model.target_table = null + model.target_field = null + return + } + if (field === 'writer_connector_type' && typeof value === 'number') { + const result = await getDatasourceOptionsById(value) + writerDatasourceOptions.value = result || [] + model.writer_datasource_id = null + return + } + if (field === 'src_datasource_id' && typeof value === 'number') { + const result = await getDatasourceTablesById(value) + srcTableOptions.value = result || [] + model.src_table = null + model.src_field = null + } + if (field === 'target_datasource_id' && typeof value === 'number') { + const result = await getDatasourceTablesById(value) + targetTableOptions.value = result || [] + model.target_table = null + model.target_field = null + } + if (field === 'src_table' && typeof value === 'string') { + const result = await getDatasourceTableColumnsById( + model.src_datasource_id, + value + ) + srcTableColumnOptions.value = result || [] + model.src_field = null + } + if (field === 'target_table' && typeof value === 'string') { + const result = await getDatasourceTableColumnsById( + model.target_datasource_id, + value + ) + targetTableColumnOptions.value = result || [] + model.target_field = null + } + } catch (err) {} + } + + onMounted(async () => { + await getRuleList() + await getRuleById(model.ruleId) + }) + + return [ + { + type: 'select', + field: 'ruleId', + name: t('project.node.rule_name'), + props: { + loading: ruleLoading, + onUpdateValue: getRuleById + }, + options: rules + } + ] +} diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sea-tunnel.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sea-tunnel.ts index eacb9118e4..c8dfdb6d14 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sea-tunnel.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sea-tunnel.ts @@ -17,27 +17,13 @@ import { ref, onMounted, watch, computed } from 'vue' import { useI18n } from 'vue-i18n' import { queryResourceList } from '@/service/modules/resources' +import { useDeployMode } from '.' import type { IJsonItem } from '../types' export function useSeaTunnel(model: { [field: string]: any }): IJsonItem[] { const { t } = useI18n() const options = ref([]) - const deployModeOptions = [ - { - label: 'client', - value: 'client' - }, - { - label: 'cluster', - value: 'cluster' - }, - { - label: 'local', - value: 'local' - } - ] - const masterTypeOptions = [ { label: 'yarn', @@ -148,13 +134,7 @@ export function useSeaTunnel(model: { [field: string]: any }): IJsonItem[] { ) return [ - { - type: 'radio', - field: 'deployMode', - name: t('project.node.sea_tunnel_deploy_mode'), - options: deployModeOptions, - value: model.deployMode - }, + useDeployMode(), { type: 'select', field: 'master', diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-spark.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-spark.ts index c17717f957..c6f87ee1ec 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-spark.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-spark.ts @@ -18,7 +18,15 @@ import { ref, onMounted, computed } from 'vue' import { useI18n } from 'vue-i18n' import { queryResourceByProgramType } from '@/service/modules/resources' import { removeUselessChildren } from './use-shell' -import { useCustomParams } from '.' +import { + useCustomParams, + useDeployMode, + useDriverCores, + useDriverMemory, + useExecutorNumber, + useExecutorMemory, + useExecutorCores +} from '.' import type { IJsonItem, ProgramType } from '../types' export function useSpark(model: { [field: string]: any }): IJsonItem[] { @@ -116,12 +124,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] { }, options: mainJarOptions }, - { - type: 'radio', - field: 'deployMode', - name: t('project.node.deploy_mode'), - options: DeployModes - }, + useDeployMode(), { type: 'input', field: 'appName', @@ -130,115 +133,11 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] { placeholder: t('project.node.app_name_tips') } }, - { - type: 'input-number', - field: 'driverCores', - name: t('project.node.driver_cores'), - span: 12, - props: { - placeholder: t('project.node.driver_cores_tips'), - min: 1 - }, - validate: { - trigger: ['input', 'blur'], - required: true, - validator(validate: any, value: string) { - if (!value) { - return new Error(t('project.node.driver_cores_tips')) - } - } - } - }, - { - type: 'input', - field: 'driverMemory', - name: t('project.node.driver_memory'), - span: 12, - props: { - placeholder: t('project.node.driver_memory_tips') - }, - validate: { - trigger: ['input', 'blur'], - required: true, - validator(validate: any, value: string) { - if (!value) { - return new Error(t('project.node.driver_memory_tips')) - } - if (!Number.isInteger(parseInt(value))) { - return new Error( - t('project.node.driver_memory') + - t('project.node.positive_integer_tips') - ) - } - } - }, - value: model.driverMemory - }, - { - type: 'input-number', - field: 'numExecutors', - name: t('project.node.executor_number'), - span: 12, - props: { - placeholder: t('project.node.executor_number_tips'), - min: 1 - }, - validate: { - trigger: ['input', 'blur'], - required: true, - validator(validate: any, value: string) { - if (!value) { - return new Error(t('project.node.executor_number_tips')) - } - } - }, - value: model.numExecutors - }, - { - type: 'input', - field: 'executorMemory', - name: t('project.node.executor_memory'), - span: 12, - props: { - placeholder: t('project.node.executor_memory_tips') - }, - validate: { - trigger: ['input', 'blur'], - required: true, - validator(validate: any, value: string) { - if (!value) { - return new Error(t('project.node.executor_memory_tips')) - } - if (!Number.isInteger(parseInt(value))) { - return new Error( - t('project.node.executor_memory_tips') + - t('project.node.positive_integer_tips') - ) - } - } - }, - value: model.executorMemory - }, - { - type: 'input-number', - field: 'executorCores', - name: t('project.node.executor_cores'), - span: 12, - props: { - placeholder: t('project.node.executor_cores_tips'), - min: 1 - }, - validate: { - trigger: ['input', 'blur'], - required: true, - validator(validate: any, value: string) { - if (!value) { - return new Error(t('project.node.executor_cores_tips')) - } - } - }, - value: model.executorCores - }, + useDriverCores(), + useDriverMemory(), + useExecutorNumber(), + useExecutorMemory(), + useExecutorCores(), { type: 'input', field: 'mainArgs', @@ -302,18 +201,3 @@ export const SPARK_VERSIONS = [ value: 'SPARK1' } ] - -export const DeployModes = [ - { - label: 'cluster', - value: 'cluster' - }, - { - label: 'client', - value: 'client' - }, - { - label: 'local', - value: 'local' - } -] diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts index 32bb970618..cb1c50e4e6 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts @@ -247,6 +247,34 @@ export function formatParams(data: INodeData): { dependTaskList: dependTaskList } } + if (data.taskType === 'DATA_QUALITY') { + taskParams.ruleId = data.ruleId + taskParams.ruleInputParameter = { + check_type: data.check_type, + comparison_execute_sql: data.comparison_execute_sql, + comparison_name: data.comparison_name, + failure_strategy: data.failure_strategy, + operator: data.operator, + src_connector_type: data.src_connector_type, + src_datasource_id: data.src_datasource_id, + src_table: data.src_table, + statistics_execute_sql: data.statistics_execute_sql, + statistics_name: data.statistics_name, + target_connector_type: data.target_connector_type, + target_datasource_id: data.target_datasource_id, + target_table: data.target_table, + threshold: data.threshold + } + taskParams.sparkParameters = { + deployMode: data.deployMode, + driverCores: data.driverCores, + driverMemory: data.driverMemory, + executorCores: data.executorCores, + executorMemory: data.executorMemory, + numExecutors: data.numExecutors, + others: data.others + } + } const params = { processDefinitionCode: data.processName ? String(data.processName) : '', @@ -394,6 +422,31 @@ export function formatModel(data: ITaskData) { params.dependTaskList = data.taskParams?.dependence.dependTaskList || [] params.relation = data.taskParams?.dependence.relation } + if (data.taskParams?.ruleInputParameter) { + params.check_type = data.check_type + params.comparison_execute_sql = data.comparison_execute_sql + params.comparison_name = data.comparison_name + params.failure_strategy = data.failure_strategy + params.operator = data.operator + params.src_connector_type = data.src_connector_type + params.src_datasource_id = data.src_datasource_id + params.src_table = data.src_table + params.statistics_execute_sql = data.statistics_execute_sql + params.statistics_name = data.statistics_name + params.target_connector_type = data.target_connector_type + params.target_datasource_id = data.target_datasource_id + params.target_table = data.target_table + params.threshold = data.threshold + } + if (data.taskParams?.sparkParameters) { + params.deployMode = data.deployMode + params.driverCores = data.driverCores + params.driverMemory = data.driverMemory + params.executorCores = data.executorCores + params.executorMemory = data.executorMemory + params.numExecutors = data.numExecutors + params.others = data.others + } return params } diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/index.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/index.ts new file mode 100644 index 0000000000..86202c3d47 --- /dev/null +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/index.ts @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { useFlink } from './use-flink' +import { useShell } from './use-shell' +import { useSubProcess } from './use-sub-process' +import { usePigeon } from './use-pigeon' +import { usePython } from './use-python' +import { useSpark } from './use-spark' +import { useMr } from './use-mr' +import { useHttp } from './use-http' +import { useSql } from './use-sql' +import { useProcedure } from './use-procedure' +import { useSqoop } from './use-sqoop' +import { useSeaTunnel } from './use-sea-tunnel' +import { useSwitch } from './use-switch' +import { useConditions } from './use-conditions' +import { useDataX } from './use-datax' +import { useDependent } from './use-dependent' +import { useDataQuality } from './use-data-quality' + +export default { + SHELL: useShell, + SUB_PROCESS: useSubProcess, + PYTHON: usePython, + SPARK: useSpark, + MR: useMr, + FLINK: useFlink, + HTTP: useHttp, + PIGEON: usePigeon, + SQL: useSql, + PROCEDURE: useProcedure, + SQOOP: useSqoop, + SEATUNNEL: useSeaTunnel, + SWITCH: useSwitch, + CONDITIONS: useConditions, + DATAX: useDataX, + DEPENDENT: useDependent, + DATA_QUALITY: useDataQuality +} diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-data-quality.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-data-quality.ts new file mode 100644 index 0000000000..aea55ff0da --- /dev/null +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-data-quality.ts @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Ref, reactive } from 'vue' +import { useI18n } from 'vue-i18n' +import * as Fields from '../fields/index' +import type { IJsonItem, INodeData, ITaskData } from '../types' + +export function useDataQuality({ + projectCode, + from = 0, + readonly, + data, + jsonRef +}: { + projectCode: number + from?: number + readonly?: boolean + data?: ITaskData + jsonRef: Ref +}) { + const { t } = useI18n() + const model = reactive({ + taskType: 'DATA_QUALITY', + name: '', + flag: 'YES', + description: '', + timeoutFlag: false, + timeoutNotifyStrategy: ['WARN'], + timeout: 30, + localParams: [], + environmentCode: null, + failRetryInterval: 1, + failRetryTimes: 0, + workerGroup: 'default', + delayTime: 0, + ruleId: 1, + deployMode: 'cluster', + driverCores: 1, + driverMemory: '512M', + numExecutors: 2, + executorMemory: '2G', + executorCores: 2, + others: '--conf spark.yarn.maxAppAttempts=1' + } as INodeData) + + let extra: IJsonItem[] = [] + if (from === 1) { + extra = [ + Fields.useTaskType(model, readonly), + Fields.useProcessName({ + model, + projectCode, + isCreate: !data?.id, + from, + processName: data?.processName, + code: data?.code + }) + ] + } + + return { + json: [ + Fields.useName(), + ...extra, + Fields.useRunFlag(), + Fields.useDescription(), + Fields.useTaskPriority(), + Fields.useWorkerGroup(), + Fields.useEnvironmentName(model, !data?.id), + ...Fields.useTaskGroup(model, projectCode), + ...Fields.useFailed(), + Fields.useDelayTime(model), + ...Fields.useTimeoutAlarm(model), + ...Fields.useRules(model, (items: IJsonItem[], len: number) => { + jsonRef.value.splice(17, len, ...items) + }), + Fields.useDeployMode(), + Fields.useDriverCores(), + Fields.useDriverMemory(), + Fields.useExecutorNumber(), + Fields.useExecutorMemory(), + Fields.useExecutorCores(), + { + type: 'input', + field: 'others', + name: t('project.node.option_parameters'), + props: { + type: 'textarea', + placeholder: t('project.node.option_parameters_tips') + } + }, + ...Fields.useCustomParams({ + model, + field: 'localParams', + isSimple: true + }), + Fields.usePreTasks(model) + ] as IJsonItem[], + model + } +} diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-flink.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-flink.ts index 5b0f7e0023..4196247f00 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-flink.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-flink.ts @@ -31,6 +31,7 @@ export function useFlink({ data?: ITaskData }) { const model = reactive({ + taskType: 'FLINK', name: '', flag: 'YES', description: '', diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-pigeon.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-pigeon.ts index a3eb0aca50..78c09ab85d 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-pigeon.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-pigeon.ts @@ -31,6 +31,7 @@ export function usePigeon({ data?: ITaskData }) { const model = reactive({ + taskType: 'PIGEON', name: '', flag: 'YES', description: '', diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-sub-process.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-sub-process.ts index f21cbf78ce..8c9d8d0119 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-sub-process.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-sub-process.ts @@ -31,6 +31,7 @@ export function useSubProcess({ data?: ITaskData }) { const model = reactive({ + taskType: 'SUB_PROCESS', name: '', flag: 'YES', description: '', diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts index 2e89b4579d..17df919fe2 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts @@ -17,9 +17,15 @@ import { VNode } from 'vue' import type { SelectOption } from 'naive-ui' -import type { IFormItem, IJsonItem } from '@/components/form/types' + import type { TaskType } from '@/views/projects/task/constants/task-type' import type { IDataBase } from '@/service/modules/data-source/types' +import type { + IFormItem, + IJsonItem, + FormRules, + IJsonItemParams +} from '@/components/form/types' type ProgramType = 'JAVA' | 'SCALA' | 'PYTHON' type SourceType = 'MYSQL' | 'HDFS' | 'HIVE' @@ -49,6 +55,11 @@ interface ILocalParam { value?: string } +interface IResponseJsonItem extends Omit { + type: 'input' | 'select' | 'radio' | 'group' + emit: 'change'[] +} + interface IDependpendItem { depTaskCode?: number status?: 'SUCCESS' | 'FAILURE' @@ -161,6 +172,33 @@ interface ISqoopSourceParams { hivePartitionKey?: string hivePartitionValue?: string } +interface ISparkParameters { + deployMode?: string + driverCores?: number + driverMemory?: string + executorCores?: number + executorMemory?: string + numExecutors?: number + others?: string +} + +interface IRuleParameters { + check_type?: string + comparison_execute_sql?: string + comparison_name?: string + failure_strategy?: string + operator?: string + src_connector_type?: number + src_datasource_id?: number + src_table?: string + statistics_execute_sql?: string + statistics_name?: string + target_connector_type?: number + target_datasource_id?: number + target_table?: string + threshold?: string +} + interface ITaskParams { resourceList?: ISourceItem[] mainJar?: ISourceItem @@ -229,6 +267,9 @@ interface ITaskParams { jobSpeedRecord?: number xms?: number xmx?: number + sparkParameters?: ISparkParameters + ruleId?: number + ruleInputParameter?: IRuleParameters } interface INodeData @@ -239,9 +280,11 @@ interface INodeData | 'targetParams' | 'sourceParams' | 'dependence' + | 'sparkParameters' >, ISqoopTargetData, - ISqoopSourceData { + ISqoopSourceData, + IRuleParameters { id?: string taskType?: ITaskType processName?: number @@ -280,7 +323,7 @@ interface ITaskData > { name?: string taskPriority?: string - timeoutFlag: 'OPEN' | 'CLOSE' + timeoutFlag?: 'OPEN' | 'CLOSE' timeoutNotifyStrategy?: string | [] taskParams?: ITaskParams } @@ -292,8 +335,6 @@ export { ITaskType, ITaskData, INodeData, - IFormItem, - IJsonItem, ITaskParams, IOption, IDataBase, @@ -303,5 +344,10 @@ export { ISqoopSourceParams, ISqoopTargetParams, IDependTask, - IDependpendItem + IDependpendItem, + IFormItem, + IJsonItem, + FormRules, + IJsonItemParams, + IResponseJsonItem } diff --git a/dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts b/dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts index d23fbabf1f..0812375726 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts @@ -14,24 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import { useFlink } from './tasks/use-flink' -import { useShell } from './tasks/use-shell' -import { useSubProcess } from './tasks/use-sub-process' -import { usePigeon } from './tasks/use-pigeon' -import { usePython } from './tasks/use-python' -import { useSpark } from './tasks/use-spark' -import { useMr } from './tasks/use-mr' -import { useHttp } from './tasks/use-http' -import { useSql } from './tasks/use-sql' -import { useProcedure } from './tasks/use-procedure' -import { useSqoop } from './tasks/use-sqoop' -import { useSeaTunnel } from './tasks/use-sea-tunnel' -import { useSwitch } from './tasks/use-switch' -import { useConditions } from './tasks/use-conditions' -import { useDataX } from './tasks/use-datax' -import { useDependent } from './tasks/use-dependent' -import { IJsonItem, INodeData, ITaskData } from './types' +import { ref, Ref, watch } from 'vue' +import nodes from './tasks' +import getElementByJson from '@/components/form/get-elements-by-json' +import { IFormItem, IJsonItem, INodeData, ITaskData, FormRules } from './types' export function useTask({ data, @@ -43,140 +29,39 @@ export function useTask({ projectCode: number from?: number readonly?: boolean -}): { json: IJsonItem[]; model: INodeData } { - const { taskType = 'SHELL' } = data - let node = {} as { json: IJsonItem[]; model: INodeData } - if (taskType === 'SHELL') { - node = useShell({ - projectCode, - from, - readonly, - data - }) - } - if (taskType === 'SUB_PROCESS') { - node = useSubProcess({ - projectCode, - from, - readonly, - data - }) - } - if (taskType === 'PYTHON') { - node = usePython({ - projectCode, - from, - readonly, - data - }) - } - if (taskType === 'SPARK') { - node = useSpark({ - projectCode, - from, - readonly, - data - }) - } - if (taskType === 'MR') { - node = useMr({ - projectCode, - from, - readonly, - data - }) - } - if (taskType === 'FLINK') { - node = useFlink({ - projectCode, - from, - readonly, - data - }) - } - if (taskType === 'HTTP') { - node = useHttp({ - projectCode, - from, - readonly, - data - }) - } - if (taskType === 'PIGEON') { - node = usePigeon({ - projectCode, - from, - readonly, - data - }) - } - if (taskType === 'SQL') { - node = useSql({ - projectCode, - from, - readonly, - data - }) - } - if (taskType === 'PROCEDURE') { - node = useProcedure({ - projectCode, - from, - readonly, - data - }) - } - if (taskType === 'SQOOP') { - node = useSqoop({ - projectCode, - from, - readonly, - data - }) - } - if (taskType === 'SEATUNNEL') { - node = useSeaTunnel({ - projectCode, - from, - readonly, - data - }) +}): { + elementsRef: Ref + rulesRef: Ref + model: INodeData +} { + const jsonRef = ref([]) as Ref + const elementsRef = ref([]) as Ref + const rulesRef = ref({}) + const params = { + projectCode, + from, + readonly, + data, + jsonRef } - if (taskType === 'SWITCH') { - node = useSwitch({ - projectCode, - from, - readonly, - data - }) - } + const { model, json } = nodes[data.taskType || 'SHELL'](params) + jsonRef.value = json - if (taskType === 'CONDITIONS') { - node = useConditions({ - projectCode, - from, - readonly, - data - }) + const getElements = () => { + const { rules, elements } = getElementByJson(jsonRef.value, model) + elementsRef.value = elements + rulesRef.value = rules } - if (taskType === 'DATAX') { - node = useDataX({ - projectCode, - from, - readonly, - data - }) - } - if (taskType === 'DEPENDENT') { - node = useDependent({ - projectCode, - from, - readonly, - data - }) - } + getElements() + + watch( + () => jsonRef.value.length, + () => { + getElements() + } + ) - return node + return { elementsRef, rulesRef, model } } diff --git a/dolphinscheduler-ui-next/src/views/projects/task/constants/task-type.ts b/dolphinscheduler-ui-next/src/views/projects/task/constants/task-type.ts index 52878ab096..3c2d104bc6 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/constants/task-type.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/constants/task-type.ts @@ -58,6 +58,9 @@ export const TASK_TYPES_MAP = { CONDITIONS: { alias: 'CONDITIONS' }, + DATA_QUALITY: { + alias: 'DATA_QUALITY' + }, SWITCH: { alias: 'SWITCH' }, diff --git a/dolphinscheduler-ui-next/src/views/projects/task/definition/use-task.ts b/dolphinscheduler-ui-next/src/views/projects/task/definition/use-task.ts index 2e887461e5..69e64cadec 100644 --- a/dolphinscheduler-ui-next/src/views/projects/task/definition/use-task.ts +++ b/dolphinscheduler-ui-next/src/views/projects/task/definition/use-task.ts @@ -27,7 +27,7 @@ import type { ITaskData, INodeData, ISingleSaveReq, IRecord } from './types' export function useTask(projectCode: number) { const initalTask = { - taskType: 'DEPENDENT' + taskType: 'SHELL' } as ITaskData const task = reactive({ taskShow: false,