Browse Source

[Feature][UI Next] Add the form of the sea tunnel task to the module of next ui. (#8507)

* add the component of seatunnel

* add the form of the sea tunnel task

* developed the form of the sea tunnel task
3.0.0/version-upgrade
calvin 3 years ago committed by GitHub
parent
commit
6fb112d1da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2725
      dolphinscheduler-ui-next/pnpm-lock.yaml
  2. 8
      dolphinscheduler-ui-next/src/locales/modules/en_US.ts
  3. 7
      dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts
  4. 1
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/index.ts
  5. 1
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource-type.ts
  6. 313
      dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sea-tunnel.ts
  7. 63
      dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts
  8. 87
      dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-sea-tunnel.ts
  9. 5
      dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts
  10. 9
      dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts
  11. 14
      dolphinscheduler-ui-next/src/views/projects/workflow/components/dag/menu.module.scss
  12. 2
      dolphinscheduler-ui-next/src/views/projects/workflow/components/dag/startup.module.scss
  13. 2
      dolphinscheduler-ui-next/src/views/security/user-manage/components/user-modal.tsx
  14. 2
      dolphinscheduler-ui-next/src/views/security/user-manage/use-table.tsx

2725
dolphinscheduler-ui-next/pnpm-lock.yaml

File diff suppressed because it is too large Load Diff

8
dolphinscheduler-ui-next/src/locales/modules/en_US.ts

@ -750,7 +750,13 @@ const project = {
only_update: 'OnlyUpdate',
allow_insert: 'AllowInsert',
concurrency: 'Concurrency',
concurrency_tips: 'Please enter Concurrency'
concurrency_tips: 'Please enter Concurrency',
sea_tunnel_deploy_mode: 'Deploy Mode',
sea_tunnel_master: 'Master',
sea_tunnel_master_url: 'Master URL',
sea_tunnel_queue: 'Queue',
sea_tunnel_master_url_tips:
'Please enter the master url, e.g., 127.0.0.1:7077'
}
}

7
dolphinscheduler-ui-next/src/locales/modules/zh_CN.ts

@ -742,7 +742,12 @@ const project = {
only_update: '只更新',
allow_insert: '无更新便插入',
concurrency: '并发度',
concurrency_tips: '请输入并发度'
concurrency_tips: '请输入并发度',
sea_tunnel_deploy_mode: '部署方式',
sea_tunnel_master: 'Master',
sea_tunnel_master_url: 'Master URL',
sea_tunnel_queue: '队列',
sea_tunnel_master_url_tips: '请直接填写地址,例如:127.0.0.1:7077'
}
}

1
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/index.ts

@ -45,3 +45,4 @@ export { useFlink } from './use-flink'
export { useHttp } from './use-http'
export { useSql } from './use-sql'
export { useSqoop } from './use-sqoop'
export { useSeaTunnel } from './use-sea-tunnel'

1
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-datasource-type.ts

@ -18,7 +18,6 @@
import { ref, onMounted } from 'vue'
import { useI18n } from 'vue-i18n'
import type { IJsonItem } from '../types'
import { number } from 'echarts'
export function useDatasourceType(model: { [field: string]: any }): IJsonItem {
const { t } = useI18n()

313
dolphinscheduler-ui-next/src/views/projects/task/components/node/fields/use-sea-tunnel.ts

@ -0,0 +1,313 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ref, onMounted, watch, computed } from 'vue'
import { useI18n } from 'vue-i18n'
import { queryResourceList } from '@/service/modules/resources'
import type { IJsonItem } from '../types'
export function useSeaTunnel(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n()
const options = ref([])
const deployModeOptions = [
{
label: 'client',
value: 'client'
},
{
label: 'cluster',
value: 'cluster'
},
{
label: 'local',
value: 'local'
}
]
const masterTypeOptions = [
{
label: 'yarn',
value: 'yarn'
},
{
label: 'local',
value: 'local'
},
{
label: 'spark://',
value: 'spark://'
},
{
label: 'mesos://',
value: 'mesos://'
}
]
const queueOptions = [
{
label: 'default',
value: 'default'
}
]
const loading = ref(false)
const getResourceList = async () => {
if (loading.value) return
loading.value = true
try {
model.resourceFiles = []
const res = await queryResourceList({ type: 'FILE' })
removeUselessChildren(res)
options.value = res || []
loading.value = false
} catch (err) {
loading.value = false
}
}
function removeUselessChildren(
list: { children?: []; fullName: string; id: number }[]
) {
if (!list.length) return
list.forEach((item) => {
if (!item.children) {
return
}
if (item.children.length === 0) {
model.resourceFiles.push({ id: item.id, fullName: item.fullName })
delete item.children
return
}
removeUselessChildren(item.children)
})
}
onMounted(() => {
getResourceList()
})
const masterSpan = computed(() => (model.deployMode === 'local' ? 0 : 12))
const queueSpan = computed(() =>
model.deployMode === 'local' || model.master != 'yarn' ? 0 : 12
)
const masterUrlSpan = computed(() =>
model.deployMode === 'local' ||
(model.master != 'spark://' && model.master != 'mesos://')
? 0
: 12
)
const baseScript = 'sh ${WATERDROP_HOME}/bin/start-waterdrop.sh'
const parseRawScript = () => {
if (model.rawScript) {
model.rawScript.split('\n').forEach((script: string) => {
let params = script.replace(baseScript, '').split('--')
params?.forEach((param: string) => {
let pair = param.split(' ')
if (pair && pair.length >= 2) {
if (pair[0] === 'master') {
let prefix = pair[1].substring(0, 8)
if (pair[1] && (prefix === 'mesos://' || prefix === 'spark://')) {
model.master = prefix
model.masterUrl = pair[1].substring(8, pair[1].length)
} else {
model.master = pair[1]
}
} else if (pair[0] === 'deploy-mode') {
model.deployMode = pair[1]
} else if (pair[0] === 'queue') {
model.queue = pair[1]
}
}
})
})
}
}
watch(
() => model.rawScript,
() => {
parseRawScript()
}
)
return [
{
type: 'radio',
field: 'deployMode',
name: t('project.node.sea_tunnel_deploy_mode'),
options: deployModeOptions,
value: model.deployMode
},
{
type: 'select',
field: 'master',
name: t('project.node.sea_tunnel_master'),
options: masterTypeOptions,
value: model.master,
span: masterSpan
},
{
type: 'input',
field: 'masterUrl',
name: t('project.node.sea_tunnel_master_url'),
value: model.masterUrl,
span: masterUrlSpan,
props: {
placeholder: t('project.node.sea_tunnel_master_url_tips')
}
},
{
type: 'select',
field: 'queue',
name: t('project.node.sea_tunnel_queue'),
options: queueOptions,
value: model.queue,
span: queueSpan
},
{
type: 'tree-select',
field: 'resourceList',
name: t('project.node.resources'),
options,
props: {
multiple: true,
checkable: true,
cascade: true,
showPath: true,
checkStrategy: 'child',
placeholder: t('project.node.resources_tips'),
keyField: 'id',
labelField: 'name',
loading,
validate: {
trigger: ['input', 'blur'],
required: true
}
}
},
{
type: 'custom-parameters',
field: 'localParams',
name: t('project.node.custom_parameters'),
children: [
{
type: 'input',
field: 'prop',
span: 6,
props: {
placeholder: t('project.node.prop_tips'),
maxLength: 256
},
validate: {
trigger: ['input', 'blur'],
required: true,
validator(validate: any, value: string) {
if (!value) {
return new Error(t('project.node.prop_tips'))
}
const sameItems = model.localParams.filter(
(item: { prop: string }) => item.prop === value
)
if (sameItems.length > 1) {
return new Error(t('project.node.prop_repeat'))
}
}
}
},
{
type: 'select',
field: 'direct',
span: 4,
options: DIRECT_LIST,
value: 'IN'
},
{
type: 'select',
field: 'type',
span: 6,
options: TYPE_LIST,
value: 'VARCHAR'
},
{
type: 'input',
field: 'value',
span: 6,
props: {
placeholder: t('project.node.value_tips'),
maxLength: 256
}
}
]
}
]
}
export const TYPE_LIST = [
{
value: 'VARCHAR',
label: 'VARCHAR'
},
{
value: 'INTEGER',
label: 'INTEGER'
},
{
value: 'LONG',
label: 'LONG'
},
{
value: 'FLOAT',
label: 'FLOAT'
},
{
value: 'DOUBLE',
label: 'DOUBLE'
},
{
value: 'DATE',
label: 'DATE'
},
{
value: 'TIME',
label: 'TIME'
},
{
value: 'TIMESTAMP',
label: 'TIMESTAMP'
},
{
value: 'BOOLEAN',
label: 'BOOLEAN'
}
]
export const DIRECT_LIST = [
{
value: 'IN',
label: 'IN'
},
{
value: 'OUT',
label: 'OUT'
}
]

63
dolphinscheduler-ui-next/src/views/projects/task/components/node/format-data.ts

@ -15,7 +15,7 @@
* limitations under the License.
*/
import { omit } from 'lodash'
import { find, omit } from 'lodash'
import type {
INodeData,
ITaskData,
@ -72,6 +72,7 @@ export function formatParams(data: INodeData): {
taskParams.connectTimeout = data.connectTimeout
taskParams.socketTimeout = data.socketTimeout
}
if (data.taskType === 'SQOOP') {
taskParams.jobType = data.isCustomTask ? 'CUSTOM' : 'TEMPLATE'
taskParams.localParams = data.localParams
@ -181,6 +182,15 @@ export function formatParams(data: INodeData): {
taskParams.method = data.method
}
if (data.taskType === 'SEATUNNEL') {
if (data.deployMode === 'local') {
data.master = 'local'
data.masterUrl = ''
data.deployMode = 'client'
}
buildRawScript(data)
}
const params = {
processDefinitionCode: data.processName ? String(data.processName) : '',
upstreamCodes: data?.preTasks?.join(','),
@ -261,6 +271,7 @@ export function formatModel(data: ITaskData) {
if (data.taskParams?.method) {
params.method = data.taskParams?.method
}
if (data.taskParams?.targetParams) {
const targetParams: ISqoopTargetParams = JSON.parse(
data.taskParams.targetParams
@ -310,5 +321,55 @@ export function formatModel(data: ITaskData) {
params.sourceHivePartitionValue = sourceParams.hivePartitionValue
}
if (data.taskParams?.rawScript) {
params.rawScript = data.taskParams?.rawScript
}
return params
}
const buildRawScript = (model: INodeData) => {
const baseScript = 'sh ${WATERDROP_HOME}/bin/start-waterdrop.sh'
if (!model.resourceList) return
let master = model.master
let masterUrl = model?.masterUrl ? model?.masterUrl : ''
let deployMode = model.deployMode
let queue = model.queue
if (model.deployMode === 'local') {
master = 'local'
masterUrl = ''
deployMode = 'client'
}
if (master === 'yarn' || master === 'local') {
masterUrl = ''
}
let localParams = ''
model?.localParams?.forEach((param: any) => {
localParams = localParams + ' --variable ' + param.prop + '=' + param.value
})
let rawScript = ''
model.resourceList?.forEach((id: number) => {
let item = find(model.resourceFiles, { id: id })
rawScript =
rawScript +
baseScript +
' --master ' +
master +
masterUrl +
' --deploy-mode ' +
deployMode +
' --queue ' +
queue
if (item && item.fullName) {
rawScript = rawScript + ' --config ' + item.fullName
}
rawScript = rawScript + localParams + ' \n'
})
model.rawScript = rawScript ? rawScript : ''
}

87
dolphinscheduler-ui-next/src/views/projects/task/components/node/tasks/use-sea-tunnel.ts

@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { reactive } from 'vue'
import * as Fields from '../fields/index'
import type { IJsonItem, INodeData } from '../types'
import { ITaskData } from '../types'
export function useSeaTunnel({
projectCode,
from = 0,
readonly,
data
}: {
projectCode: number
from?: number
readonly?: boolean
data?: ITaskData
}) {
const model = reactive({
name: '',
taskType: 'SEATUNNEL',
flag: 'YES',
description: '',
timeoutFlag: false,
localParams: [],
environmentCode: null,
failRetryInterval: 1,
failRetryTimes: 0,
workerGroup: 'default',
delayTime: 0,
timeout: 30,
deployMode: 'client',
queue: 'default',
master: 'yarn',
masterUrl: '',
resourceFiles: []
} as INodeData)
let extra: IJsonItem[] = []
if (from === 1) {
extra = [
Fields.useTaskType(model, readonly),
Fields.useProcessName({
model,
projectCode,
isCreate: !data?.id,
from,
processName: data?.processName,
code: data?.code
})
]
}
return {
json: [
Fields.useName(),
...extra,
Fields.useRunFlag(),
Fields.useDescription(),
Fields.useTaskPriority(),
Fields.useWorkerGroup(),
Fields.useEnvironmentName(model, !model.id),
...Fields.useTaskGroup(model, projectCode),
...Fields.useFailed(),
Fields.useDelayTime(model),
...Fields.useTimeoutAlarm(model),
...Fields.useSeaTunnel(model),
Fields.usePreTasks(model)
] as IJsonItem[],
model
}
}

5
dolphinscheduler-ui-next/src/views/projects/task/components/node/types.ts

@ -184,6 +184,8 @@ interface ITaskParams {
targetType?: SourceType
targetParams?: string
sourceParams?: string
queue?: string
master?: string
}
type ITaskType = TaskType
@ -220,6 +222,9 @@ interface INodeData
mainJar?: number
timeoutSetting?: boolean
isCustomTask?: boolean
method?: string
masterUrl?: string
resourceFiles?: { id: number; fullName: string }[] | null
}
interface ITaskData

9
dolphinscheduler-ui-next/src/views/projects/task/components/node/use-task.ts

@ -26,6 +26,7 @@ import { useHttp } from './tasks/use-http'
import { useSql } from './tasks/use-sql'
import { useProcedure } from './tasks/use-procedure'
import { useSqoop } from './tasks/use-sqoop'
import { useSeaTunnel } from './tasks/use-sea-tunnel'
import { IJsonItem, INodeData, ITaskData } from './types'
export function useTask({
@ -129,6 +130,14 @@ export function useTask({
data
})
}
if (taskType === 'SEATUNNEL') {
node = useSeaTunnel({
projectCode,
from,
readonly,
data
})
}
return node
}

14
dolphinscheduler-ui-next/src/views/projects/workflow/components/dag/menu.module.scss

@ -15,29 +15,29 @@
* limitations under the License.
*/
.dag-context-menu{
.dag-context-menu {
position: fixed;
left: 0;
top: 0;
width: 100px;
background-color: #ffffff;
box-shadow: 0 2px 10px rgba(0,0,0,0.12);
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.12);
.menu-item{
.menu-item {
padding: 5px 10px;
border-bottom: solid 1px #f2f3f7;
cursor: pointer;
color: rgb(89, 89, 89);
font-size: 12px;
&:hover:not(.disabled){
&:hover:not(.disabled) {
color: #262626;
background-color: #f5f5f5;
background-color: #f5f5f5;
}
&.disabled{
&.disabled {
cursor: not-allowed;
color: rgba(89, 89, 89, .4);
color: rgba(89, 89, 89, 0.4);
}
}
}

2
dolphinscheduler-ui-next/src/views/projects/workflow/components/dag/startup.module.scss

@ -28,7 +28,7 @@
font-weight: bold;
margin-right: 10px;
}
.content {
font-size: 12px;
&:hover {

2
dolphinscheduler-ui-next/src/views/security/user-manage/components/user-modal.tsx

@ -121,7 +121,7 @@ export const UserModal = defineComponent({
multiple
cascade
checkable
checkStrategy="child"
checkStrategy='child'
defaultExpandAll
options={this.resourceTree}
v-model:value={this.authorizedFiles}

2
dolphinscheduler-ui-next/src/views/security/user-manage/use-table.tsx

@ -107,7 +107,7 @@ function useColumns({ onEdit, onDelete }: UseTableProps) {
trigger: () => (
<NButton
circle
type="warning"
type='warning'
size='small'
class='authorize'
>

Loading…
Cancel
Save