sqoop front-end optimization

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/datasource.vue

@@ -131,7 +131,7 @@
     },
     created () {
       let supportType = this.supportType || []
-      this.typeList = _.cloneDeep(this.store.state.dag.dsTypeListS)
+      this.typeList = this.data.typeList || _.cloneDeep(this.store.state.dag.dsTypeListS)
       // Have a specified data source
       if (supportType.length) {
         let is = (type) => {

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue

@@ -1,29 +1,65 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
+* Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
+* contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
+* this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
+* The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
+* (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
+* the License.  You may obtain a copy of the License at
- *
+*
- *    http://www.apache.org/licenses/LICENSE-2.0
+*    http://www.apache.org/licenses/LICENSE-2.0
- *
+*
- * Unless required by applicable law or agreed to in writing, software
+* Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
+* distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+* See the License for the specific language governing permissions and
- * limitations under the License.
+* limitations under the License.
- */
+*/
 <template>
   <div class="sql-model">
 
+    <m-list-box>
+      <div slot="text">{{$t('Custom Job')}}</div>
+      <div slot="content">
+        <x-switch
+          v-model="isCustomTask"
+          @on-click="_onSwitch"
+          :disabled="isDetails"
+        >
+        </x-switch>
+      </div>
+    </m-list-box>
+    <m-list-box v-show="isCustomTask">
+      <div slot="text">{{$t('Custom Script')}}</div>
+      <div slot="content">
+        <div class="from-mirror">
+          <textarea
+            id="code-shell-mirror"
+            name="code-shell-mirror"
+            style="opacity: 0;">
+          </textarea>
+        </div>
+      </div>
+    </m-list-box>
+    <template v-if="!isCustomTask">
+      <m-list-box>
+        <div slot="text">{{$t('Sqoop Job Name')}}</div>
+        <div slot="content">
+          <x-input
+            :disabled="isDetails"
+            type="text"
+            v-model="jobName"
+            :placeholder="$t('Please enter Job Name(required)')">
+          </x-input>
+        </div>
+      </m-list-box>
       <m-list-box>
         <div slot="text">{{$t('Direct')}}</div>
         <div slot="content">
           <x-select
             style="width: 130px;"
             v-model="modelType"
-                :disabled="isDetails">
+            :disabled="isDetails"
+            @on-change="_handleModelTypeChange">
             <x-option
               v-for="city in modelTypeList"
               :key="city.code"
@@ -33,6 +69,28 @@
           </x-select>
         </div>
       </m-list-box>
+      <m-list-box>
+        <div slot="text" style="width: 110px;">{{$t('Hadoop Custom Params')}}</div>
+        <div slot="content">
+          <m-local-params
+            ref="refMapColumnHadoopParams"
+            @on-local-params="_onHadoopCustomParams"
+            :udp-list="hadoopCustomParams"
+            :hide="false">
+          </m-local-params>
+        </div>
+      </m-list-box>
+      <m-list-box>
+        <div slot="text" style="width: 100px;">{{$t('Sqoop Advanced Parameters')}}</div>
+        <div slot="content">
+          <m-local-params
+            ref="refMapColumnAdvancedParams"
+            @on-local-params="_onSqoopAdvancedParams"
+            :udp-list="sqoopAdvancedParams"
+            :hide="false">
+          </m-local-params>
+        </div>
+      </m-list-box>
 
       <template>
         <m-list-box>
@@ -65,7 +123,9 @@
               <m-datasource
                 ref="refSourceDs"
                 @on-dsData="_onSourceDsData"
-                    :data="{ type:sourceMysqlParams.srcType,datasource:sourceMysqlParams.srcDatasource }"
+                :data="{type:sourceMysqlParams.srcType,
+                      typeList: [{id: 0, code: 'MYSQL', disabled: false}],
+                      datasource:sourceMysqlParams.srcDatasource }"
               >
               </m-datasource>
             </div>
@@ -385,7 +445,9 @@
             <m-datasource
               ref="refTargetDs"
               @on-dsData="_onTargetDsData"
-              :data="{ type:targetMysqlParams.targetType,datasource:targetMysqlParams.targetDatasource }"
+              :data="{ type:targetMysqlParams.targetType,
+                      typeList: [{id: 0, code: 'MYSQL', disabled: false}],
+                      datasource:targetMysqlParams.targetDatasource }"
             >
             </m-datasource>
           </div>
@@ -474,7 +536,7 @@
           </x-input>
         </div>
       </m-list-box>
-
+    </template>
     <m-list-box>
       <div slot="text">{{$t('Custom Parameters')}}</div>
       <div slot="content">
@@ -498,16 +560,38 @@
   import codemirror from '@/conf/home/pages/resource/pages/file/pages/_source/codemirror'
 
   let editor
+  let shellEditor
 
   export default {
     name: 'sql',
     data () {
       return {
-
+        /**
+         * Is Custom Task
+         */
+        isCustomTask: false,
         /**
          * Customer Params
          */
         localParams: [],
+
+        /**
+         * Hadoop Custom Params
+         */
+        hadoopCustomParams: [],
+
+        /**
+         * Sqoop Advanced Params
+         */
+        sqoopAdvancedParams: [],
+        /**
+         * script
+         */
+        customShell: '',
+        /**
+         * task name
+         */
+        jobName: '',
         /**
          * mysql query type
          */
@@ -524,6 +608,10 @@
          * concurrency
          */
         concurrency:1,
+        /**
+         * default job type
+         */
+        jobType:'TEMPLATE',
         /**
          * direct model type
          */
@@ -531,16 +619,10 @@
 
         modelTypeList: [{ code: 'import' }, { code: 'export' }],
 
-        sourceTypeList:[
+        sourceTypeList: [
           {
-            code:"MYSQL"
+            code: "MYSQL"
           },
-          {
-            code:"HDFS"
-          },
-          {
-            code:"HIVE"
-          }
         ],
 
         targetTypeList:[
@@ -620,8 +702,32 @@
     },
     methods: {
 
+      _onSwitch(is){
+        if(is) {
+          this.jobType = 'CUSTOM'
+          this.isCustomTask = true
+          setTimeout(() => {
+            this._handlerShellEditor()
+          }, 200)
+        } else {
+          this.jobType = 'TEMPLATE'
+          this.isCustomTask = false
+          setTimeout(() => {
+            this._handlerEditor()
+          }, 200)
+        }
+      },
+
       _handleQueryType(o){
         this.sourceMysqlParams.srcQueryType = this.srcQueryType
+        this._getTargetTypeList(this.sourceType)
+        this.targetType = this.targetTypeList[0].code
+      },
+
+      _handleModelTypeChange(a){
+        this._getSourceTypeList(a.label)
+        this.sourceType = this.sourceTypeList[0].code
+        this._handleSourceTypeChange({label: this.sourceType, value: this.sourceType})
       },
 
       _handleSourceTypeChange(a){
@@ -629,17 +735,59 @@
         this.targetType = this.targetTypeList[0].code
       },
 
+      _getSourceTypeList(data){
+        switch(data){
+          case 'import':
+            this.sourceTypeList = [
+              {
+                code:"MYSQL"
+              },
+            ]
+            break;
+          case 'export':
+            this.sourceTypeList = [
+              {
+                code: "HDFS"
+              },
+              {
+                code: "HIVE"
+              }
+            ]
+            break;
+          default:
+            this.sourceTypeList = [
+              {
+                code:"MYSQL"
+              },
+              {
+                code:"HIVE"
+              },
+              {
+                code:"HDFS"
+              }
+            ]
+            break;
+        }
+      },
+
       _getTargetTypeList(data){
         switch(data){
           case 'MYSQL':
+            if (this.srcQueryType === "1") {
               this.targetTypeList = [
                 {
-                code:"HIVE"
+                  code: "HDFS"
+                }]
+            } else {
+              this.targetTypeList = [
+                {
+                  code: "HIVE"
                 },
                 {
-                code:"HDFS"
+                  code: "HDFS"
                 }
               ]
+            }
             break;
           case 'HDFS':
             this.targetTypeList = [
@@ -670,12 +818,10 @@
 
       _onMapColumnHive (a) {
         this.sourceMysqlParams.mapColumnHive = a
-        console.log(this.sourceMysqlParams.mapColumnHive)
       },
 
       _onMapColumnJava (a) {
         this.sourceMysqlParams.mapColumnJava = a
-        console.log(this.sourceMysqlParams.mapColumnJava)
       },
 
       /**
@@ -701,7 +847,8 @@
         var params = null
         switch(this.sourceType){
           case "MYSQL":
-            this.sourceMysqlParams.srcQuerySql = editor ? editor.getValue() : this.sourceMysqlParams.srcQuerySql
+            this.sourceMysqlParams.srcQuerySql = this.sourceMysqlParams.srcQueryType === "1" && editor ?
+              editor.getValue() : this.sourceMysqlParams.srcQuerySql
             params = JSON.stringify(this.sourceMysqlParams)
             break;
           case "ORACLE":
@@ -791,42 +938,63 @@
        * verification
        */
       _verification () {
+        let sqoopParams = {
+          jobType: this.jobType,
+          localParams: this.localParams
+        }
+        if (this.jobType === 'CUSTOM') {
+          if (!shellEditor.getValue()) {
+            this.$message.warning(`${i18n.$t('Please enter Custom Shell(required)')}`)
+            return false
+          }
+          sqoopParams['customShell'] = shellEditor.getValue()
+        } else {
+          if (!this.jobName) {
+            this.$message.warning(`${i18n.$t('Please enter Job Name(required)')}`)
+            return false
+          }
 
-        switch(this.sourceType){
+          switch (this.sourceType) {
             case "MYSQL":
               if (!this.$refs.refSourceDs._verifDatasource()) {
                 return false
               }
-            if(this.srcQueryType === '1'){
+              if (this.srcQueryType === '1') {
                 if (!editor.getValue()) {
                   this.$message.warning(`${i18n.$t('Please enter a SQL Statement(required)')}`)
                   return false
                 }
-            }else{
+                this.sourceMysqlParams.srcTable = ""
+                this.sourceMysqlParams.srcColumnType = "0"
+                this.sourceMysqlParams.srcColumns = ""
+              } else {
                 if (this.sourceMysqlParams.srcTable === "") {
                   this.$message.warning(`${i18n.$t('Please enter Mysql Table(required)')}`)
                   return false
                 }
-
+                this.sourceMysqlParams.srcQuerySql = ""
-              if(this.sourceMysqlParams.srcColumnType === "1" && this.sourceMysqlParams.srcColumns === ""){
+                if (this.sourceMysqlParams.srcColumnType === "1" && this.sourceMysqlParams.srcColumns === "") {
                   this.$message.warning(`${i18n.$t('Please enter Columns (Comma separated)')}`)
                   return false
                 }
+                if (this.sourceMysqlParams.srcColumnType === "0") {
+                  this.sourceMysqlParams.srcColumns = ""
+                }
               }
 
               break;
             case "HDFS":
-              if(this.sourceHdfsParams.exportDir === ""){
+              if (this.sourceHdfsParams.exportDir === "") {
                 this.$message.warning(`${i18n.$t('Please enter Export Dir(required)')}`)
                 return false
               }
               break;
             case "HIVE":
-              if(this.sourceHiveParams.hiveDatabase === ""){
+              if (this.sourceHiveParams.hiveDatabase === "") {
                 this.$message.warning(`${i18n.$t('Please enter Hive Database(required)')}`)
                 return false
               }
-              if(this.sourceHiveParams.hiveTable === ""){
+              if (this.sourceHiveParams.hiveTable === "") {
                 this.$message.warning(`${i18n.$t('Please enter Hive Table(required)')}`)
                 return false
               }
@@ -835,19 +1003,19 @@
               break;
           }
 
-        switch(this.targetType){
+          switch (this.targetType) {
             case "HIVE":
-            if(this.targetHiveParams.hiveDatabase === ""){
+              if (this.targetHiveParams.hiveDatabase === "") {
                 this.$message.warning(`${i18n.$t('Please enter Hive Database(required)')}`)
                 return false
               }
-              if(this.targetHiveParams.hiveTable === ""){
+              if (this.targetHiveParams.hiveTable === "") {
                 this.$message.warning(`${i18n.$t('Please enter Hive Table(required)')}`)
                 return false
               }
               break;
             case "HDFS":
-             if(this.targetHdfsParams.targetPath === ""){
+              if (this.targetHdfsParams.targetPath === "") {
                 this.$message.warning(`${i18n.$t('Please enter Target Dir(required)')}`)
                 return false
               }
@@ -857,7 +1025,7 @@
                 return false
               }
 
-              if(this.targetMysqlParams.targetTable === ""){
+              if (this.targetMysqlParams.targetTable === "") {
                 this.$message.warning(`${i18n.$t('Please enter Mysql Table(required)')}`)
                 return false
               }
@@ -865,17 +1033,19 @@
             default:
               break;
           }
+          sqoopParams['jobName'] = this.jobName
+          sqoopParams['hadoopCustomParams'] = this.hadoopCustomParams
+          sqoopParams['sqoopAdvancedParams'] = this.sqoopAdvancedParams
+          sqoopParams['concurrency'] = this.concurrency
+          sqoopParams['modelType'] = this.modelType
+          sqoopParams['sourceType'] = this.sourceType
+          sqoopParams['targetType'] = this.targetType
+          sqoopParams['targetParams'] = this._handleTargetParams()
+          sqoopParams['sourceParams'] = this._handleSourceParams()
+        }
 
         // storage
-        this.$emit('on-params', {
+        this.$emit('on-params', sqoopParams)
-          concurrency:this.concurrency,
-          modelType:this.modelType,
-          sourceType:this.sourceType,
-          targetType:this.targetType,
-          sourceParams:this._handleSourceParams(),
-          targetParams:this._handleTargetParams(),
-          localParams:this.localParams
-        })
         return true
       },
 
@@ -912,6 +1082,33 @@
         return editor
       },
 
+      /**
+       * Processing code highlighting
+       */
+      _handlerShellEditor () {
+        this._destroyShellEditor()
+
+        // shellEditor
+        shellEditor = codemirror('code-shell-mirror', {
+          mode: 'shell',
+          readOnly: this.isDetails
+        })
+
+        this.keypress = () => {
+          if (!shellEditor.getOption('readOnly')) {
+            shellEditor.showHint({
+              completeSingle: false
+            })
+          }
+        }
+
+        // Monitor keyboard
+        shellEditor.on('keypress', this.keypress)
+        shellEditor.setValue(this.customShell)
+
+        return shellEditor
+      },
+
       /**
        * return localParams
        */
@@ -919,6 +1116,20 @@
         this.localParams = a
       },
 
+      /**
+       * return hadoopParams
+       */
+      _onHadoopCustomParams (a) {
+        this.hadoopCustomParams = a
+      },
+
+      /**
+       * return sqoopAdvancedParams
+       */
+      _onSqoopAdvancedParams (a) {
+        this.sqoopAdvancedParams = a
+      },
+
       _cacheParams () {
         this.$emit('on-cache-params', {
           concurrency:this.concurrency,
@@ -939,6 +1150,13 @@
           editor = null
         }
       },
+      _destroyShellEditor () {
+        if (shellEditor) {
+          shellEditor.toTextArea() // Uninstall
+          shellEditor.off($('.code-shell-mirror'), 'keypress', this.keypress)
+          shellEditor.off($('.code-shell-mirror'), 'changes', this.changes)
+        }
+      }
     },
     watch: {
       // Listening to sqlType
@@ -970,15 +1188,25 @@
 
       // Non-null objects represent backfill
       if (!_.isEmpty(o)) {
-        this.concurrency = o.params.concurrency || 1,
+        this.jobType = o.params.jobType
-        this.modelType = o.params.modelType,
+        this.isCustomTask = false
-        this.sourceType = o.params.sourceType,
+        if (this.jobType === 'CUSTOM') {
+          this.customShell = o.params.customShell
+          this.isCustomTask = true
+        } else {
+          this.jobName = o.params.jobName
+          this.hadoopCustomParams = o.params.hadoopCustomParams
+          this.sqoopAdvancedParams = o.params.sqoopAdvancedParams
+          this.concurrency = o.params.concurrency || 1
+          this.modelType = o.params.modelType
+          this.sourceType = o.params.sourceType
           this._getTargetTypeList(this.sourceType)
-        this.targetType = o.params.targetType,
+          this.targetType = o.params.targetType
-        this._getSourceParams(o.params.sourceParams),
+          this._getSourceParams(o.params.sourceParams)
-        this._getTargetParams(o.params.targetParams),
+          this._getTargetParams(o.params.targetParams)
           this.localParams = o.params.localParams
         }
+      }
     },
 
     mounted () {
@@ -986,6 +1214,10 @@
         this._handlerEditor()
       }, 200)
 
+      setTimeout(() => {
+        this._handlerShellEditor()
+      }, 200)
+
       setTimeout(() => {
         this.srcQueryType = this.sourceMysqlParams.srcQueryType
       }, 500)

dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js

@@ -540,6 +540,9 @@ export default {
   'Whether directory': 'Whether directory',
   Yes: 'Yes',
   No: 'No',
+  'Hadoop Custom Params': 'Hadoop Params',
+  'Sqoop Advanced Parameters': 'Sqoop Params',
+  'Sqoop Job Name': 'Job Name',
   'Please enter Mysql Database(required)': 'Please enter Mysql Database(required)',
   'Please enter Mysql Table(required)': 'Please enter Mysql Table(required)',
   'Please enter Columns (Comma separated)': 'Please enter Columns (Comma separated)',
@@ -554,6 +557,8 @@ export default {
   'Please enter Lines Terminated': 'Please enter Lines Terminated',
   'Please enter Concurrency': 'Please enter Concurrency',
   'Please enter Update Key': 'Please enter Update Key',
+  'Please enter Job Name(required)': 'Please enter Job Name(required)',
+  'Please enter Custom Shell(required)': 'Please enter Custom Shell(required)',
   Direct: 'Direct',
   Type: 'Type',
   ModelType: 'ModelType',
@@ -587,6 +592,8 @@ export default {
   'All Columns': 'All Columns',
   'Some Columns': 'Some Columns',
   'Branch flow': 'Branch flow',
+  'Custom Job': 'Custom Job',
+  'Custom Script': 'Custom Script',
   'Cannot select the same node for successful branch flow and failed branch flow': 'Cannot select the same node for successful branch flow and failed branch flow',
   'Successful branch flow and failed branch flow are required': 'Successful branch flow and failed branch flow are required',
   'Unauthorized or deleted resources': 'Unauthorized or deleted resources',

dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@@ -540,6 +540,9 @@ export default {
   'Whether directory': '是否文件夹',
   Yes: '是',
   No: '否',
+  'Hadoop Custom Params': 'Hadoop参数',
+  'Sqoop Advanced Parameters': 'Sqoop参数',
+  'Sqoop Job Name': '任务名称',
   'Please enter Mysql Database(required)': '请输入Mysql数据库(必填)',
   'Please enter Mysql Table(required)': '请输入Mysql表名(必填)',
   'Please enter Columns (Comma separated)': '请输入列名，用 , 隔开',
@@ -554,6 +557,8 @@ export default {
   'Please enter Lines Terminated': '请输入行分隔符',
   'Please enter Concurrency': '请输入并发度',
   'Please enter Update Key': '请输入更新列',
+  'Please enter Job Name(required)': '请输入任务名称(必填)',
+  'Please enter Custom Shell(required)': '请输入自定义脚本',
   Direct: '流向',
   Type: '类型',
   ModelType: '模式',
@@ -587,6 +592,8 @@ export default {
   'All Columns': '全表导入',
   'Some Columns': '选择列',
   'Branch flow': '分支流转',
+  'Custom Job': '自定义任务',
+  'Custom Script': '自定义脚本',
   'Cannot select the same node for successful branch flow and failed branch flow': '成功分支流转和失败分支流转不能选择同一个节点',
   'Successful branch flow and failed branch flow are required': '成功分支流转和失败分支流转必填',
   'Unauthorized or deleted resources': '未授权或已删除资源',