sqoop front-end optimization

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/datasource.vue

@@ -131,7 +131,7 @@
     },
     created () {
       let supportType = this.supportType || []
-      this.typeList = _.cloneDeep(this.store.state.dag.dsTypeListS)
+      this.typeList = this.data.typeList || _.cloneDeep(this.store.state.dag.dsTypeListS)
       // Have a specified data source
       if (supportType.length) {
         let is = (type) => {

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue

@@ -17,13 +17,49 @@
 <template>
   <div class="sql-model">
 
+    <m-list-box>
+      <div slot="text">{{$t('Custom Job')}}</div>
+      <div slot="content">
+        <x-switch
+          v-model="isCustomTask"
+          @on-click="_onSwitch"
+          :disabled="isDetails"
+        >
+        </x-switch>
+      </div>
+    </m-list-box>
+    <m-list-box v-show="isCustomTask">
+      <div slot="text">{{$t('Custom Script')}}</div>
+      <div slot="content">
+        <div class="from-mirror">
+          <textarea
+            id="code-shell-mirror"
+            name="code-shell-mirror"
+            style="opacity: 0;">
+          </textarea>
+        </div>
+      </div>
+    </m-list-box>
+    <template v-if="!isCustomTask">
+      <m-list-box>
+        <div slot="text">{{$t('Sqoop Job Name')}}</div>
+        <div slot="content">
+          <x-input
+            :disabled="isDetails"
+            type="text"
+            v-model="jobName"
+            :placeholder="$t('Please enter Job Name(required)')">
+          </x-input>
+        </div>
+      </m-list-box>
       <m-list-box>
         <div slot="text">{{$t('Direct')}}</div>
         <div slot="content">
           <x-select
             style="width: 130px;"
             v-model="modelType"
-                :disabled="isDetails">
+            :disabled="isDetails"
+            @on-change="_handleModelTypeChange">
             <x-option
               v-for="city in modelTypeList"
               :key="city.code"
@@ -33,6 +69,28 @@
           </x-select>
         </div>
       </m-list-box>
+      <m-list-box>
+        <div slot="text" style="width: 110px;">{{$t('Hadoop Custom Params')}}</div>
+        <div slot="content">
+          <m-local-params
+            ref="refMapColumnHadoopParams"
+            @on-local-params="_onHadoopCustomParams"
+            :udp-list="hadoopCustomParams"
+            :hide="false">
+          </m-local-params>
+        </div>
+      </m-list-box>
+      <m-list-box>
+        <div slot="text" style="width: 100px;">{{$t('Sqoop Advanced Parameters')}}</div>
+        <div slot="content">
+          <m-local-params
+            ref="refMapColumnAdvancedParams"
+            @on-local-params="_onSqoopAdvancedParams"
+            :udp-list="sqoopAdvancedParams"
+            :hide="false">
+          </m-local-params>
+        </div>
+      </m-list-box>
 
       <template>
         <m-list-box>
@@ -65,7 +123,9 @@
               <m-datasource
                 ref="refSourceDs"
                 @on-dsData="_onSourceDsData"
-                    :data="{ type:sourceMysqlParams.srcType,datasource:sourceMysqlParams.srcDatasource }"
+                :data="{type:sourceMysqlParams.srcType,
+                      typeList: [{id: 0, code: 'MYSQL', disabled: false}],
+                      datasource:sourceMysqlParams.srcDatasource }"
               >
               </m-datasource>
             </div>
@@ -385,7 +445,9 @@
             <m-datasource
               ref="refTargetDs"
               @on-dsData="_onTargetDsData"
-              :data="{ type:targetMysqlParams.targetType,datasource:targetMysqlParams.targetDatasource }"
+              :data="{ type:targetMysqlParams.targetType,
+                      typeList: [{id: 0, code: 'MYSQL', disabled: false}],
+                      datasource:targetMysqlParams.targetDatasource }"
             >
             </m-datasource>
           </div>
@@ -474,7 +536,7 @@
           </x-input>
         </div>
       </m-list-box>
-
+    </template>
     <m-list-box>
       <div slot="text">{{$t('Custom Parameters')}}</div>
       <div slot="content">
@@ -498,16 +560,38 @@
   import codemirror from '@/conf/home/pages/resource/pages/file/pages/_source/codemirror'
 
   let editor
+  let shellEditor
 
   export default {
     name: 'sql',
     data () {
       return {
-
+        /**
+         * Is Custom Task
+         */
+        isCustomTask: false,
         /**
          * Customer Params
          */
         localParams: [],
+
+        /**
+         * Hadoop Custom Params
+         */
+        hadoopCustomParams: [],
+
+        /**
+         * Sqoop Advanced Params
+         */
+        sqoopAdvancedParams: [],
+        /**
+         * script
+         */
+        customShell: '',
+        /**
+         * task name
+         */
+        jobName: '',
         /**
          * mysql query type
          */
@@ -524,6 +608,10 @@
          * concurrency
          */
         concurrency:1,
+        /**
+         * default job type
+         */
+        jobType:'TEMPLATE',
         /**
          * direct model type
          */
@@ -535,12 +623,6 @@
           {
             code: "MYSQL"
           },
-          {
-            code:"HDFS"
-          },
-          {
-            code:"HIVE"
-          }
         ],
 
         targetTypeList:[
@@ -620,8 +702,32 @@
     },
     methods: {
 
+      _onSwitch(is){
+        if(is) {
+          this.jobType = 'CUSTOM'
+          this.isCustomTask = true
+          setTimeout(() => {
+            this._handlerShellEditor()
+          }, 200)
+        } else {
+          this.jobType = 'TEMPLATE'
+          this.isCustomTask = false
+          setTimeout(() => {
+            this._handlerEditor()
+          }, 200)
+        }
+      },
+
       _handleQueryType(o){
         this.sourceMysqlParams.srcQueryType = this.srcQueryType
+        this._getTargetTypeList(this.sourceType)
+        this.targetType = this.targetTypeList[0].code
+      },
+
+      _handleModelTypeChange(a){
+        this._getSourceTypeList(a.label)
+        this.sourceType = this.sourceTypeList[0].code
+        this._handleSourceTypeChange({label: this.sourceType, value: this.sourceType})
       },
 
       _handleSourceTypeChange(a){
@@ -629,9 +735,50 @@
         this.targetType = this.targetTypeList[0].code
       },
 
+      _getSourceTypeList(data){
+        switch(data){
+          case 'import':
+            this.sourceTypeList = [
+              {
+                code:"MYSQL"
+              },
+            ]
+            break;
+          case 'export':
+            this.sourceTypeList = [
+              {
+                code: "HDFS"
+              },
+              {
+                code: "HIVE"
+              }
+            ]
+            break;
+          default:
+            this.sourceTypeList = [
+              {
+                code:"MYSQL"
+              },
+              {
+                code:"HIVE"
+              },
+              {
+                code:"HDFS"
+              }
+            ]
+            break;
+        }
+      },
+
       _getTargetTypeList(data){
         switch(data){
           case 'MYSQL':
+            if (this.srcQueryType === "1") {
+              this.targetTypeList = [
+                {
+                  code: "HDFS"
+                }]
+            } else {
               this.targetTypeList = [
                 {
                   code: "HIVE"
@@ -640,6 +787,7 @@
                   code: "HDFS"
                 }
               ]
+            }
             break;
           case 'HDFS':
             this.targetTypeList = [
@@ -670,12 +818,10 @@
 
       _onMapColumnHive (a) {
         this.sourceMysqlParams.mapColumnHive = a
-        console.log(this.sourceMysqlParams.mapColumnHive)
       },
 
       _onMapColumnJava (a) {
         this.sourceMysqlParams.mapColumnJava = a
-        console.log(this.sourceMysqlParams.mapColumnJava)
       },
 
       /**
@@ -701,7 +847,8 @@
         var params = null
         switch(this.sourceType){
           case "MYSQL":
-            this.sourceMysqlParams.srcQuerySql = editor ? editor.getValue() : this.sourceMysqlParams.srcQuerySql
+            this.sourceMysqlParams.srcQuerySql = this.sourceMysqlParams.srcQueryType === "1" && editor ?
+              editor.getValue() : this.sourceMysqlParams.srcQuerySql
             params = JSON.stringify(this.sourceMysqlParams)
             break;
           case "ORACLE":
@@ -791,6 +938,21 @@
        * verification
        */
       _verification () {
+        let sqoopParams = {
+          jobType: this.jobType,
+          localParams: this.localParams
+        }
+        if (this.jobType === 'CUSTOM') {
+          if (!shellEditor.getValue()) {
+            this.$message.warning(`${i18n.$t('Please enter Custom Shell(required)')}`)
+            return false
+          }
+          sqoopParams['customShell'] = shellEditor.getValue()
+        } else {
+          if (!this.jobName) {
+            this.$message.warning(`${i18n.$t('Please enter Job Name(required)')}`)
+            return false
+          }
 
           switch (this.sourceType) {
             case "MYSQL":
@@ -802,16 +964,22 @@
                   this.$message.warning(`${i18n.$t('Please enter a SQL Statement(required)')}`)
                   return false
                 }
+                this.sourceMysqlParams.srcTable = ""
+                this.sourceMysqlParams.srcColumnType = "0"
+                this.sourceMysqlParams.srcColumns = ""
               } else {
                 if (this.sourceMysqlParams.srcTable === "") {
                   this.$message.warning(`${i18n.$t('Please enter Mysql Table(required)')}`)
                   return false
                 }
-
+                this.sourceMysqlParams.srcQuerySql = ""
                 if (this.sourceMysqlParams.srcColumnType === "1" && this.sourceMysqlParams.srcColumns === "") {
                   this.$message.warning(`${i18n.$t('Please enter Columns (Comma separated)')}`)
                   return false
                 }
+                if (this.sourceMysqlParams.srcColumnType === "0") {
+                  this.sourceMysqlParams.srcColumns = ""
+                }
               }
 
               break;
@@ -865,17 +1033,19 @@
             default:
               break;
           }
+          sqoopParams['jobName'] = this.jobName
+          sqoopParams['hadoopCustomParams'] = this.hadoopCustomParams
+          sqoopParams['sqoopAdvancedParams'] = this.sqoopAdvancedParams
+          sqoopParams['concurrency'] = this.concurrency
+          sqoopParams['modelType'] = this.modelType
+          sqoopParams['sourceType'] = this.sourceType
+          sqoopParams['targetType'] = this.targetType
+          sqoopParams['targetParams'] = this._handleTargetParams()
+          sqoopParams['sourceParams'] = this._handleSourceParams()
+        }
 
         // storage
-        this.$emit('on-params', {
+        this.$emit('on-params', sqoopParams)
-          concurrency:this.concurrency,
-          modelType:this.modelType,
-          sourceType:this.sourceType,
-          targetType:this.targetType,
-          sourceParams:this._handleSourceParams(),
-          targetParams:this._handleTargetParams(),
-          localParams:this.localParams
-        })
         return true
       },
 
@@ -912,6 +1082,33 @@
         return editor
       },
 
+      /**
+       * Processing code highlighting
+       */
+      _handlerShellEditor () {
+        this._destroyShellEditor()
+
+        // shellEditor
+        shellEditor = codemirror('code-shell-mirror', {
+          mode: 'shell',
+          readOnly: this.isDetails
+        })
+
+        this.keypress = () => {
+          if (!shellEditor.getOption('readOnly')) {
+            shellEditor.showHint({
+              completeSingle: false
+            })
+          }
+        }
+
+        // Monitor keyboard
+        shellEditor.on('keypress', this.keypress)
+        shellEditor.setValue(this.customShell)
+
+        return shellEditor
+      },
+
       /**
        * return localParams
        */
@@ -919,6 +1116,20 @@
         this.localParams = a
       },
 
+      /**
+       * return hadoopParams
+       */
+      _onHadoopCustomParams (a) {
+        this.hadoopCustomParams = a
+      },
+
+      /**
+       * return sqoopAdvancedParams
+       */
+      _onSqoopAdvancedParams (a) {
+        this.sqoopAdvancedParams = a
+      },
+
       _cacheParams () {
         this.$emit('on-cache-params', {
           concurrency:this.concurrency,
@@ -939,6 +1150,13 @@
           editor = null
         }
       },
+      _destroyShellEditor () {
+        if (shellEditor) {
+          shellEditor.toTextArea() // Uninstall
+          shellEditor.off($('.code-shell-mirror'), 'keypress', this.keypress)
+          shellEditor.off($('.code-shell-mirror'), 'changes', this.changes)
+        }
+      }
     },
     watch: {
       // Listening to sqlType
@@ -970,15 +1188,25 @@
 
       // Non-null objects represent backfill
       if (!_.isEmpty(o)) {
-        this.concurrency = o.params.concurrency || 1,
+        this.jobType = o.params.jobType
-        this.modelType = o.params.modelType,
+        this.isCustomTask = false
-        this.sourceType = o.params.sourceType,
+        if (this.jobType === 'CUSTOM') {
+          this.customShell = o.params.customShell
+          this.isCustomTask = true
+        } else {
+          this.jobName = o.params.jobName
+          this.hadoopCustomParams = o.params.hadoopCustomParams
+          this.sqoopAdvancedParams = o.params.sqoopAdvancedParams
+          this.concurrency = o.params.concurrency || 1
+          this.modelType = o.params.modelType
+          this.sourceType = o.params.sourceType
           this._getTargetTypeList(this.sourceType)
-        this.targetType = o.params.targetType,
+          this.targetType = o.params.targetType
-        this._getSourceParams(o.params.sourceParams),
+          this._getSourceParams(o.params.sourceParams)
-        this._getTargetParams(o.params.targetParams),
+          this._getTargetParams(o.params.targetParams)
           this.localParams = o.params.localParams
         }
+      }
     },
 
     mounted () {
@@ -986,6 +1214,10 @@
         this._handlerEditor()
       }, 200)
 
+      setTimeout(() => {
+        this._handlerShellEditor()
+      }, 200)
+
       setTimeout(() => {
         this.srcQueryType = this.sourceMysqlParams.srcQueryType
       }, 500)

dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js

@@ -540,6 +540,9 @@ export default {
   'Whether directory': 'Whether directory',
   Yes: 'Yes',
   No: 'No',
+  'Hadoop Custom Params': 'Hadoop Params',
+  'Sqoop Advanced Parameters': 'Sqoop Params',
+  'Sqoop Job Name': 'Job Name',
   'Please enter Mysql Database(required)': 'Please enter Mysql Database(required)',
   'Please enter Mysql Table(required)': 'Please enter Mysql Table(required)',
   'Please enter Columns (Comma separated)': 'Please enter Columns (Comma separated)',
@@ -554,6 +557,8 @@ export default {
   'Please enter Lines Terminated': 'Please enter Lines Terminated',
   'Please enter Concurrency': 'Please enter Concurrency',
   'Please enter Update Key': 'Please enter Update Key',
+  'Please enter Job Name(required)': 'Please enter Job Name(required)',
+  'Please enter Custom Shell(required)': 'Please enter Custom Shell(required)',
   Direct: 'Direct',
   Type: 'Type',
   ModelType: 'ModelType',
@@ -587,6 +592,8 @@ export default {
   'All Columns': 'All Columns',
   'Some Columns': 'Some Columns',
   'Branch flow': 'Branch flow',
+  'Custom Job': 'Custom Job',
+  'Custom Script': 'Custom Script',
   'Cannot select the same node for successful branch flow and failed branch flow': 'Cannot select the same node for successful branch flow and failed branch flow',
   'Successful branch flow and failed branch flow are required': 'Successful branch flow and failed branch flow are required',
   'Unauthorized or deleted resources': 'Unauthorized or deleted resources',

dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@@ -540,6 +540,9 @@ export default {
   'Whether directory': '是否文件夹',
   Yes: '是',
   No: '否',
+  'Hadoop Custom Params': 'Hadoop参数',
+  'Sqoop Advanced Parameters': 'Sqoop参数',
+  'Sqoop Job Name': '任务名称',
   'Please enter Mysql Database(required)': '请输入Mysql数据库(必填)',
   'Please enter Mysql Table(required)': '请输入Mysql表名(必填)',
   'Please enter Columns (Comma separated)': '请输入列名，用 , 隔开',
@@ -554,6 +557,8 @@ export default {
   'Please enter Lines Terminated': '请输入行分隔符',
   'Please enter Concurrency': '请输入并发度',
   'Please enter Update Key': '请输入更新列',
+  'Please enter Job Name(required)': '请输入任务名称(必填)',
+  'Please enter Custom Shell(required)': '请输入自定义脚本',
   Direct: '流向',
   Type: '类型',
   ModelType: '模式',
@@ -587,6 +592,8 @@ export default {
   'All Columns': '全表导入',
   'Some Columns': '选择列',
   'Branch flow': '分支流转',
+  'Custom Job': '自定义任务',
+  'Custom Script': '自定义脚本',
   'Cannot select the same node for successful branch flow and failed branch flow': '成功分支流转和失败分支流转不能选择同一个节点',
   'Successful branch flow and failed branch flow are required': '成功分支流转和失败分支流转必填',
   'Unauthorized or deleted resources': '未授权或已删除资源',