feat: add validation rules for csv import(in progress)

Signed-off-by: Pranav C <pranavxc@gmail.com>

package.json

@@ -10,7 +10,7 @@
     "lerna": "^3.20.1"
   },
   "scripts": {
-    "start:api": "cd ./packages/nocodb; npm install; cross-env NC_DISABLE_TELE=true npm run watch:run",
+    "start:api": "cd ./packages/nocodb; npm install; npm run watch:run",
     "start:web": "cd ./packages/nc-gui; npm install; npm run dev",
     "cypress:run": "cypress run --config-file ./scripts/cypress/cypress.json",
     "cypress:open": "cypress open --config-file ./scripts/cypress/cypress.json",

packages/nc-gui/components/import/dropOrSelectFileModal.vue

@@ -63,10 +63,11 @@ export default {
     }
   },
   methods: {
-    _change(file) {
+    _change(event) {
-      const files = file.target.files
+      const files = event.target.files
       if (files && files[0]) {
         this.$emit('file', files[0])
+        event.target.value = ''
       }
     },
     dropHandler(ev) {

packages/nc-gui/components/project/spreadsheet/components/columnMappingModal.vue

@@ -6,79 +6,98 @@
           Table : {{ meta._tn }}
         </v-card-title>
         <v-spacer />
-        <v-btn color="primary" large @click="$emit('import',mappings)">
+        <v-btn
+          :disabled="!valid || requiredColumnValidationError"
+          color="primary"
+          large
+          @click="$emit('import',mappings)"
+        >
           <v-icon small class="mr-1">
             mdi-database-import-outline
-          </v-icon>   Import
+          </v-icon>
+          Import
         </v-btn>
       </v-card-actions>
+      <div v-if="requiredColumnValidationError" class="error--text caption pa-2 text-center">
+        {{ requiredColumnValidationError }}
+      </div>
       <v-divider />
       <v-container fluid>
-        <v-simple-table dense style="position:relative;">
+        <v-form ref="form" v-model="valid">
-          <thead>
+          <v-simple-table dense style="position:relative;">
-            <tr>
+            <thead>
-              <th />
+              <tr>
-              <th style="width:45%" class="grey--text">
+                <th />
-                Source column
+                <th style="width:45%" class="grey--text">
-              </th>
+                  Source column
-              <th style="width:45%" class="grey--text">
+                </th>
-                Destination column
+                <th style="width:45%" class="grey--text">
-              </th>
+                  Destination column
-            </tr>
+                </th>
-          </thead>
+              </tr>
-          <tbody>
+            </thead>
-            <tr v-for="(r,i) in mappings" :key="i">
+            <tbody>
-              <td>
+              <tr v-for="(r,i) in mappings" :key="i">
-                <v-checkbox v-model="r.enabled" class="mt-0" dense hide-details />
+                <td>
-              </td><td class="caption" style="width:45%">
+                  <v-checkbox v-model="r.enabled" class="mt-0" dense hide-details />
-                <div :title="r.sourceCn" style="">
+                </td>
-                  {{ r.sourceCn }}
+                <td class="caption" style="width:45%">
-                </div>
+                  <div :title="r.sourceCn" style="">
-              </td><td style="width:45%">
+                    {{ r.sourceCn }}
-                <v-select
+                  </div>
-                  v-model="r.destCn"
+                </td>
-                  class="caption"
+                <td style="width:45%">
-                  dense
+                  <v-select
-                  hide-details
+                    v-model="r.destCn"
-                  :items="meta.columns"
+                    class="caption"
-                  item-text="_cn"
+                    dense
-                  item-value="_cn"
+                    hide-details="auto"
-                >
+                    :items="meta.columns"
-                  <template #selection="{item}">
+                    item-text="_cn"
-                    <v-icon small class="mr-1">
+                    :item-value="v => v"
-                      {{ getIcon(item.uidt) }}
+                    :rules="[
-                    </v-icon>
+                      v => validateField(v,r)
-                    {{ item._cn }}
+                    ]"
-                  </template>
+                    @change="$refs.form.validate()"
-                  <template #item="{item}">
+                  >
-                    <v-icon small class="mr-1">
+                    <template #selection="{item}">
-                      {{ getIcon(item.uidt) }}
+                      <v-icon small class="mr-1">
-                    </v-icon>
+                        {{ getIcon(item.uidt) }}
-                    <span class="caption"> {{ item._cn }}</span>
+                      </v-icon>
-                  </template>
+                      {{ item._cn }}
-                </v-select>
+                    </template>
-              </td>
+                    <template #item="{item}">
-            </tr>
+                      <v-icon small class="mr-1">
-          </tbody>
+                        {{ getIcon(item.uidt) }}
-        </v-simple-table>
+                      </v-icon>
+                      <span class="caption"> {{ item._cn }}</span>
+                    </template>
+                  </v-select>
+                </td>
+              </tr>
+            </tbody>
+          </v-simple-table>
+        </v-form>
       </v-container>
     </v-card>
   </v-dialog>
 </template>
 
 <script>
-import { getUIDTIcon } from '~/components/project/spreadsheet/helpers/uiTypes'
+import { getUIDTIcon, UITypes } from '~/components/project/spreadsheet/helpers/uiTypes'
 
 export default {
   name: 'ColumnMappingModal',
   props: {
     meta: Object,
     importDataColumns: Array,
-    value: Boolean
+    value: Boolean,
+    parsedCsv: Object
   },
   data() {
     return {
-      mappings: []
+      mappings: [],
+      valid: false
     }
   },
   computed: {
@@ -89,12 +108,36 @@ export default {
       set(v) {
         this.$emit('input', v)
       }
+    },
+    requiredColumnValidationError() {
+      const missingRequiredColumns = this.meta.columns.filter(c => (
+        (c.pk && (!c.ai || !c.cdf)) || c.rqd) &&
+        !this.mappings.some(r => r.destCn === c._cn))
+
+      if (missingRequiredColumns.length) {
+        return `Following columns are required : ${missingRequiredColumns.map(c => c._cn).join(', ')}`
+      }
+      return false
     }
   },
   mounted() {
     this.mapDefaultColumns()
   },
+
   methods: {
+    validateField(v, row) {
+      if (!v) { return true }
+      switch (v.uidt) {
+        case UITypes.Number:
+          if (this.parsedCsv && this.parsedCsv.data && this.parsedCsv.data.slice(0, 500)
+            .some(r => r[row.sourceCn] !== null && r[row.sourceCn] !== undefined && isNaN(+r[row.sourceCn]))) {
+            return 'Source data contains some invalid numbers'
+          }
+          break
+      }
+
+      return true
+    },
     mapDefaultColumns() {
       this.mappings = []
       for (const col of this.importDataColumns) {

packages/nc-gui/components/project/spreadsheet/components/csvExportImport.vue

@@ -64,6 +64,7 @@
       v-model="columnMappingModal"
       :meta="meta"
       :import-data-columns="parsedCsv.columns"
+      :parsed-csv="parsedCsv"
       @import="importData"
     />
   </div>
@@ -239,11 +240,14 @@ export default {
         const api = this.$ncApis.get({
           table: this.meta.tn
         })
+
         const data = this.parsedCsv.data
         for (let i = 0, progress = 0; i < data.length; i += 500) {
           const batchData = data.slice(i, i + 500).map(row => columnMappings.reduce((res, col) => {
-            if (col.enabled) {
+            // todo: parse data
-              res[col.destCn] = row[col.sourceCn]
+
+            if (col.enabled && col.destCn && col.destCn._cn) {
+              res[col.destCn._cn] = row[col.sourceCn]
             }
             return res
           }, {}))

packages/nc-gui/plugins/ncApis/gqlApi.js

@@ -252,6 +252,20 @@ export default class GqlApi {
     })
     return { list: list.data.data.m2mNotChildren, count: count.data.data.m2mNotChildrenCount.count }
   }
+
+  async insertBulk(data, {
+    params = {}
+  } = {}) {
+    const data1 = await this.post(`/nc/${this.$ctx.projectId}/v1/graphql`, {
+      query: `mutation bulkInsert($data:[${this.tableCamelized}Input]){
+         ${this.gqlMutationCreateName}Bulk(data: $data)
+      }`,
+      variables: {
+        data
+      }
+    }, params)
+    return data1.data.data[`${this.gqlMutationCreateName}Bulk`]
+  }
 }
 /**
  * @copyright Copyright (c) 2021, Xgene Cloud Ltd