Browse Source

[Feature][UI Next] Add the form of the datax task to the module of next ui. (#8554)

* add the component of the use-datax

* developed the form of the datax task

* prettier the codes
3.0.0/version-upgrade
calvin 3 years ago committed by GitHub
parent
commit
e84964f1f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 19
      dolphinscheduler-ui-next/src/locales/modules/en_US.ts
  2. 19
      dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts
  3. 1
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/index.ts
  4. 28
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource-type.ts
  5. 7
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource.ts
  6. 454
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datax.ts
  7. 12
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-environment-name.ts
  8. 27
      dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts
  9. 102
      dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-datax.ts
  10. 11
      dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts
  11. 10
      dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts
  12. 8
      dolphinscheduler-ui-next/src/views/resource/file/edit/index.tsx

19
dolphinscheduler-ui-next/src/locales/modules/en_US.ts

@ -773,7 +773,24 @@ const project = {
switch_condition: 'Condition', switch_condition: 'Condition',
switch_branch_flow: 'Branch Flow', switch_branch_flow: 'Branch Flow',
and: 'and', and: 'and',
or: 'or' or: 'or',
datax_custom_template: 'Custom Template Switch',
datax_json_template: 'JSON',
datax_target_datasource_type: 'Target Datasource Type',
datax_target_database: 'Target Database',
datax_target_table: 'Target Table',
datax_target_table_tips: 'Please enter the name of the target table',
datax_target_database_pre_sql: 'Pre SQL Statement',
datax_target_database_post_sql: 'Post SQL Statement',
datax_non_query_sql_tips: 'Please enter the non-query sql statement',
datax_job_speed_byte: 'Speed(Byte count)',
datax_job_speed_byte_info: '(0 means unlimited)',
datax_job_speed_record: 'Speed(Record count)',
datax_job_speed_record_info: '(0 means unlimited)',
datax_job_runtime_memory: 'Runtime Memory Limits',
datax_job_runtime_memory_xms: 'Low Limit Value',
datax_job_runtime_memory_xmx: 'High Limit Value',
datax_job_runtime_memory_unit: 'G'
} }
} }

19
dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts

@ -764,7 +764,24 @@ const project = {
switch_condition: '条件', switch_condition: '条件',
switch_branch_flow: '分支流转', switch_branch_flow: '分支流转',
and: '且', and: '且',
or: '或' or: '或',
datax_custom_template: '自定义模板',
datax_json_template: 'JSON',
datax_target_datasource_type: '目标源类型',
datax_target_database: '目标源实例',
datax_target_table: '目标表',
datax_target_table_tips: '请输入目标表名',
datax_target_database_pre_sql: '目标库前置SQL',
datax_target_database_post_sql: '目标库后置SQL',
datax_non_query_sql_tips: '请输入非查询SQL语句',
datax_job_speed_byte: '限流(字节数)',
datax_job_speed_byte_info: '(KB,0代表不限制)',
datax_job_speed_record: '限流(记录数)',
datax_job_speed_record_info: '(0代表不限制)',
datax_job_runtime_memory: '运行内存',
datax_job_runtime_memory_xms: '最小内存',
datax_job_runtime_memory_xmx: '最大内存',
datax_job_runtime_memory_unit: 'G'
} }
} }

1
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/index.ts

@ -48,3 +48,4 @@ export { useSql } from './use-sql'
export { useSqoop } from './use-sqoop' export { useSqoop } from './use-sqoop'
export { useSeaTunnel } from './use-sea-tunnel' export { useSeaTunnel } from './use-sea-tunnel'
export { useSwitch } from './use-switch' export { useSwitch } from './use-switch'
export { useDataX } from './use-datax'

28
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource-type.ts

@ -18,8 +18,13 @@
import { ref, onMounted } from 'vue' import { ref, onMounted } from 'vue'
import { useI18n } from 'vue-i18n' import { useI18n } from 'vue-i18n'
import type { IJsonItem } from '../types' import type { IJsonItem } from '../types'
import { indexOf } from 'lodash'
export function useDatasourceType(model: { [field: string]: any }): IJsonItem { export function useDatasourceType(
model: { [field: string]: any },
supportedDatasourceType?: string[],
field?: string
): IJsonItem {
const { t } = useI18n() const { t } = useI18n()
const options = ref([] as { label: string; value: string }[]) const options = ref([] as { label: string; value: string }[])
@ -78,7 +83,15 @@ export function useDatasourceType(model: { [field: string]: any }): IJsonItem {
loading.value = true loading.value = true
try { try {
options.value = datasourceTypes options.value = datasourceTypes
.filter((item) => !item.disabled) .filter((item) => {
if (item.disabled) {
return false
}
if (supportedDatasourceType) {
return indexOf(supportedDatasourceType, item.code) !== -1
}
return true
})
.map((item) => ({ label: item.code, value: item.code })) .map((item) => ({ label: item.code, value: item.code }))
loading.value = false loading.value = false
} catch (err) { } catch (err) {
@ -87,7 +100,11 @@ export function useDatasourceType(model: { [field: string]: any }): IJsonItem {
} }
const onChange = (type: string) => { const onChange = (type: string) => {
model.type = type if (field) {
model[field] = type
} else {
model.type = type
}
} }
onMounted(() => { onMounted(() => {
@ -95,7 +112,7 @@ export function useDatasourceType(model: { [field: string]: any }): IJsonItem {
}) })
return { return {
type: 'select', type: 'select',
field: 'datasourceType', field: field ? field : 'datasourceType',
span: 12, span: 12,
name: t('project.node.datasource_type'), name: t('project.node.datasource_type'),
props: { props: {
@ -105,8 +122,7 @@ export function useDatasourceType(model: { [field: string]: any }): IJsonItem {
options: options, options: options,
validate: { validate: {
trigger: ['input', 'blur'], trigger: ['input', 'blur'],
required: true, required: true
message: t('project.node.worker_group_tips')
}, },
value: model.type value: model.type
} }

7
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource.ts

@ -22,7 +22,10 @@ import type { IJsonItem } from '../types'
import { TypeReq } from '@/service/modules/data-source/types' import { TypeReq } from '@/service/modules/data-source/types'
import { find } from 'lodash' import { find } from 'lodash'
export function useDatasource(model: { [field: string]: any }): IJsonItem { export function useDatasource(
model: { [field: string]: any },
field?: string
): IJsonItem {
const { t } = useI18n() const { t } = useI18n()
const options = ref([] as { label: string; value: string }[]) const options = ref([] as { label: string; value: string }[])
@ -71,7 +74,7 @@ export function useDatasource(model: { [field: string]: any }): IJsonItem {
}) })
return { return {
type: 'select', type: 'select',
field: 'datasource', field: field ? field : 'datasource',
span: 12, span: 12,
name: t('project.node.datasource_instances'), name: t('project.node.datasource_instances'),
props: { props: {

454
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datax.ts

@ -0,0 +1,454 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ref, onMounted, watch } from 'vue'
import { useI18n } from 'vue-i18n'
import type { IJsonItem } from '../types'
import { find, indexOf } from 'lodash'
import { TypeReq } from '@/service/modules/data-source/types'
import { queryDataSourceList } from '@/service/modules/data-source'
export function useDataX(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n()
const datasourceTypes = [
{
id: 0,
code: 'MYSQL',
disabled: false
},
{
id: 1,
code: 'POSTGRESQL',
disabled: false
},
{
id: 2,
code: 'HIVE',
disabled: true
},
{
id: 3,
code: 'SPARK',
disabled: true
},
{
id: 4,
code: 'CLICKHOUSE',
disabled: false
},
{
id: 5,
code: 'ORACLE',
disabled: false
},
{
id: 6,
code: 'SQLSERVER',
disabled: false
},
{
id: 7,
code: 'DB2',
disabled: true
},
{
id: 8,
code: 'PRESTO',
disabled: true
}
]
const datasourceTypeOptions = ref([] as any)
const datasourceOptions = ref([] as any)
const destinationDatasourceOptions = ref([] as any)
const jobSpeedByteOptions: any[] = [
{
label: '不限制',
value: 0
},
{
label: '1KB',
value: 1024
},
{
label: '10KB',
value: 10240
},
{
label: '50KB',
value: 51200
},
{
label: '100KB',
value: 102400
},
{
label: '512KB',
value: 524288
}
]
const jobSpeedRecordOptions: any[] = [
{
label: '不限制',
value: 0
},
{
label: '500',
value: 500
},
{
label: '1000',
value: 1000
},
{
label: '1500',
value: 1500
},
{
label: '2000',
value: 2000
},
{
label: '2500',
value: 2500
},
{
label: '3000',
value: 3000
}
]
const memoryLimitOptions = [
{
label: '1G',
value: 1
},
{
label: '2G',
value: 2
},
{
label: '3G',
value: 3
},
{
label: '4G',
value: 4
}
]
const loading = ref(false)
const getDatasourceTypes = async () => {
if (loading.value) return
loading.value = true
try {
datasourceTypeOptions.value = datasourceTypes
.filter((item) => !item.disabled)
.map((item) => ({ label: item.code, value: item.code }))
loading.value = false
} catch (err) {
loading.value = false
}
}
const getDatasourceInstances = async () => {
const params = { type: model.dsType } as TypeReq
const res = await queryDataSourceList(params)
datasourceOptions.value = []
res.map((item: any) => {
datasourceOptions.value.push({ label: item.name, value: String(item.id) })
})
if (datasourceOptions.value && model.dataSource) {
let item = find(datasourceOptions.value, {
value: String(model.dataSource)
})
if (!item) {
model.dataSource = null
}
}
}
const getDestinationDatasourceInstances = async () => {
const params = { type: model.dtType } as TypeReq
const res = await queryDataSourceList(params)
destinationDatasourceOptions.value = []
res.map((item: any) => {
destinationDatasourceOptions.value.push({
label: item.name,
value: String(item.id)
})
})
if (destinationDatasourceOptions.value && model.dataTarget) {
let item = find(destinationDatasourceOptions.value, {
value: String(model.dataTarget)
})
if (!item) {
model.dataTarget = null
}
}
}
const sqlEditorSpan = ref(24)
const jsonEditorSpan = ref(0)
const datasourceSpan = ref(12)
const destinationDatasourceSpan = ref(8)
const otherStatementSpan = ref(22)
const jobSpeedSpan = ref(12)
const customParameterSpan = ref(0)
const initConstants = () => {
if (model.customConfigSwitch) {
model.customConfig = 1
sqlEditorSpan.value = 0
jsonEditorSpan.value = 24
datasourceSpan.value = 0
destinationDatasourceSpan.value = 0
otherStatementSpan.value = 0
jobSpeedSpan.value = 0
customParameterSpan.value = 24
} else {
model.customConfig = 0
sqlEditorSpan.value = 24
jsonEditorSpan.value = 0
datasourceSpan.value = 12
destinationDatasourceSpan.value = 8
otherStatementSpan.value = 22
jobSpeedSpan.value = 12
customParameterSpan.value = 0
}
}
onMounted(() => {
getDatasourceTypes()
getDatasourceInstances()
getDestinationDatasourceInstances()
initConstants()
})
const onSourceTypeChange = (type: string) => {
model.dsType = type
getDatasourceInstances()
}
const onDestinationTypeChange = (type: string) => {
model.dtType = type
getDestinationDatasourceInstances()
}
watch(
() => model.customConfigSwitch,
() => {
initConstants()
}
)
return [
{
type: 'switch',
field: 'customConfigSwitch',
name: t('project.node.datax_custom_template')
},
{
type: 'select',
field: 'dsType',
span: datasourceSpan,
name: t('project.node.datasource_type'),
props: {
loading: loading,
'on-update:value': onSourceTypeChange
},
options: datasourceTypeOptions,
validate: {
trigger: ['input', 'blur'],
required: true
}
},
{
type: 'select',
field: 'dataSource',
span: datasourceSpan,
name: t('project.node.datasource_instances'),
props: {
loading: loading
},
options: datasourceOptions,
validate: {
trigger: ['input', 'blur'],
required: true
}
},
{
type: 'editor',
field: 'sql',
name: t('project.node.sql_statement'),
span: sqlEditorSpan,
validate: {
trigger: ['input', 'trigger'],
required: true,
message: t('project.node.sql_empty_tips')
}
},
{
type: 'editor',
field: 'json',
name: t('project.node.datax_json_template'),
span: jsonEditorSpan,
validate: {
trigger: ['input', 'trigger'],
required: true,
message: t('project.node.sql_empty_tips')
}
},
{
type: 'select',
field: 'dtType',
name: t('project.node.datax_target_datasource_type'),
span: destinationDatasourceSpan,
props: {
loading: loading,
'on-update:value': onDestinationTypeChange
},
options: datasourceTypeOptions,
validate: {
trigger: ['input', 'blur'],
required: true
}
},
{
type: 'select',
field: 'dataTarget',
name: t('project.node.datax_target_database'),
span: destinationDatasourceSpan,
props: {
loading: loading
},
options: destinationDatasourceOptions,
validate: {
trigger: ['input', 'blur'],
required: true
}
},
{
type: 'input',
field: 'targetTable',
name: t('project.node.datax_target_table'),
span: destinationDatasourceSpan,
props: {
placeholder: t('project.node.datax_target_table_tips')
},
validate: {
trigger: ['input', 'blur'],
required: true
}
},
{
type: 'multi-input',
field: 'preStatements',
name: t('project.node.datax_target_database_pre_sql'),
span: otherStatementSpan,
props: {
placeholder: t('project.node.datax_non_query_sql_tips'),
type: 'textarea',
autosize: { minRows: 1 }
}
},
{
type: 'multi-input',
field: 'postStatements',
name: t('project.node.datax_target_database_post_sql'),
span: otherStatementSpan,
props: {
placeholder: t('project.node.datax_non_query_sql_tips'),
type: 'textarea',
autosize: { minRows: 1 }
}
},
{
type: 'select',
field: 'jobSpeedByte',
name: t('project.node.datax_job_speed_byte'),
span: jobSpeedSpan,
options: jobSpeedByteOptions,
value: 0
},
{
type: 'select',
field: 'jobSpeedRecord',
name: t('project.node.datax_job_speed_record'),
span: jobSpeedSpan,
options: jobSpeedRecordOptions,
value: 1000
},
{
type: 'custom-parameters',
field: 'localParams',
name: t('project.node.custom_parameters'),
span: customParameterSpan,
children: [
{
type: 'input',
field: 'prop',
span: 10,
props: {
placeholder: t('project.node.prop_tips'),
maxLength: 256
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.prop_tips'))
}
const sameItems = model.localParams.filter(
(item: { prop: string }) => item.prop === value
)
if (sameItems.length > 1) {
return new Error(t('project.node.prop_repeat'))
}
}
}
},
{
type: 'input',
field: 'value',
span: 10,
props: {
placeholder: t('project.node.value_tips'),
maxLength: 256
}
}
]
},
{
type: 'select',
field: 'xms',
name: t('project.node.datax_job_runtime_memory_xms'),
span: 12,
options: memoryLimitOptions,
value: 1
},
{
type: 'select',
field: 'xmx',
name: t('project.node.datax_job_runtime_memory_xmx'),
span: 12,
options: memoryLimitOptions,
value: 1
}
]
}

12
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-environment-name.ts

@ -36,11 +36,13 @@ export function useEnvironmentName(
loading.value = true loading.value = true
try { try {
const res = await queryAllEnvironmentList() const res = await queryAllEnvironmentList()
environmentList = res.map((item: { code: string; name: string; workerGroups: string[] }) => ({ environmentList = res.map(
label: item.name, (item: { code: string; name: string; workerGroups: string[] }) => ({
value: item.code, label: item.name,
workerGroups: item.workerGroups value: item.code,
})) workerGroups: item.workerGroups
})
)
options.value = environmentList.filter((option: IEnvironmentNameOption) => options.value = environmentList.filter((option: IEnvironmentNameOption) =>
filterByWorkerGroup(option) filterByWorkerGroup(option)
) )

27
dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts

@ -21,7 +21,8 @@ import type {
ITaskData, ITaskData,
ITaskParams, ITaskParams,
ISqoopTargetParams, ISqoopTargetParams,
ISqoopSourceParams ISqoopSourceParams,
ILocalParam
} from './types' } from './types'
export function formatParams(data: INodeData): { export function formatParams(data: INodeData): {
@ -204,6 +205,30 @@ export function formatParams(data: INodeData): {
} }
} }
if (data.taskType === 'DATAX') {
taskParams.customConfig = data.customConfig
if (taskParams.customConfig === 0) {
taskParams.dsType = data.dsType
taskParams.dataSource = data.dataSource
taskParams.dtType = data.dtType
taskParams.dataTarget = data.dataTarget
taskParams.sql = data.sql
taskParams.targetTable = data.targetTable
taskParams.jobSpeedByte = data.jobSpeedByte
taskParams.jobSpeedRecord = data.jobSpeedRecord
taskParams.preStatements = data.preStatements
taskParams.postStatements = data.postStatements
} else {
taskParams.json = data.json
data?.localParams?.map((param: ILocalParam) => {
param.direct = 'IN'
param.type = 'VARCHAR'
})
}
taskParams.xms = data.xms
taskParams.xmx = data.xmx
}
const params = { const params = {
processDefinitionCode: data.processName ? String(data.processName) : '', processDefinitionCode: data.processName ? String(data.processName) : '',
upstreamCodes: data?.preTasks?.join(','), upstreamCodes: data?.preTasks?.join(','),

102
dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-datax.ts

@ -0,0 +1,102 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { reactive } from 'vue'
import * as Fields from '../fields/index'
import type { IJsonItem, INodeData } from '../types'
import { ITaskData } from '../types'
export function useDataX({
projectCode,
from = 0,
readonly,
data
}: {
projectCode: number
from?: number
readonly?: boolean
data?: ITaskData
}) {
const model = reactive({
name: '',
taskType: 'DATAX',
flag: 'YES',
description: '',
timeoutFlag: false,
localParams: [],
environmentCode: null,
failRetryInterval: 1,
failRetryTimes: 0,
workerGroup: 'default',
delayTime: 0,
timeout: 30,
customConfig: data?.taskParams?.customConfig
? data?.taskParams?.customConfig
: 0,
customConfigSwitch: data?.taskParams?.customConfig !== 0,
dsType: data?.taskParams?.dsType ? data?.taskParams?.dsType : 'MYSQL',
dataSource: data?.taskParams?.dataSource,
dtType: data?.taskParams?.dtType ? data?.taskParams?.dtType : 'MYSQL',
dataTarget: data?.taskParams?.dataTarget,
sql: data?.taskParams?.sql,
targetTable: data?.taskParams?.targetTable,
preStatements: data?.taskParams?.preStatements
? data?.taskParams?.preStatements
: [],
postStatements: data?.taskParams?.postStatements
? data?.taskParams?.postStatements
: [],
jobSpeedByte: data?.taskParams?.jobSpeedByte,
jobSpeedRecord: data?.taskParams?.jobSpeedRecord,
xms: data?.taskParams?.xms,
xmx: data?.taskParams?.xmx
} as INodeData)
let extra: IJsonItem[] = []
if (from === 1) {
extra = [
Fields.useTaskType(model, readonly),
Fields.useProcessName({
model,
projectCode,
isCreate: !data?.id,
from,
processName: data?.processName,
code: data?.code
})
]
}
return {
json: [
Fields.useName(),
...extra,
Fields.useRunFlag(),
Fields.useDescription(),
Fields.useTaskPriority(),
Fields.useWorkerGroup(),
Fields.useEnvironmentName(model, !model.id),
...Fields.useTaskGroup(model, projectCode),
...Fields.useFailed(),
Fields.useDelayTime(model),
...Fields.useTimeoutAlarm(model),
...Fields.useDataX(model),
Fields.usePreTasks(model)
] as IJsonItem[],
model
}
}

11
dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts

@ -206,6 +206,17 @@ interface ITaskParams {
relation?: RelationType relation?: RelationType
dependTaskList?: IDependTask[] dependTaskList?: IDependTask[]
} }
customConfig?: number
json?: string
dsType?: string
dataSource?: number
dtType?: string
dataTarget?: number
targetTable?: string
jobSpeedByte?: number
jobSpeedRecord?: number
xms?: number
xmx?: number
} }
interface INodeData interface INodeData

10
dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts

@ -29,6 +29,7 @@ import { useSqoop } from './tasks/use-sqoop'
import { useSeaTunnel } from './tasks/use-sea-tunnel' import { useSeaTunnel } from './tasks/use-sea-tunnel'
import { useSwitch } from './tasks/use-switch' import { useSwitch } from './tasks/use-switch'
import { useConditions } from './tasks/use-conditions' import { useConditions } from './tasks/use-conditions'
import { useDataX } from './tasks/use-datax'
import { IJsonItem, INodeData, ITaskData } from './types' import { IJsonItem, INodeData, ITaskData } from './types'
export function useTask({ export function useTask({
@ -159,5 +160,14 @@ export function useTask({
}) })
} }
if (taskType === 'DATAX') {
node = useDataX({
projectCode,
from,
readonly,
data
})
}
return node return node
} }

8
dolphinscheduler-ui-next/src/views/resource/file/edit/index.tsx

@ -75,10 +75,10 @@ export default defineComponent({
class={styles['form-content']} class={styles['form-content']}
> >
<NFormItem path='content'> <NFormItem path='content'>
<div <div style={{ width: '90%' }}>
style={{width: '90%'}} <MonacoEditor
> v-model={[this.resourceViewRef.value.content, 'value']}
<MonacoEditor v-model={[this.resourceViewRef.value.content, 'value']} /> />
</div> </div>
</NFormItem> </NFormItem>
{this.routeNameRef === 'resource-file-edit' && ( {this.routeNameRef === 'resource-file-edit' && (

Loading…
Cancel
Save