feat: improve excel import

- Handle excel files without column name
- Allow user to modify table name & column name while importing
- Duplicate column name handling
- Insert exact value in case Currency type

Signed-off-by: Pranav C <pranavxc@gmail.com>

.github/workflows/publish-book.yml

@@ -1,39 +0,0 @@
-name: "Publish : Book"
-
-on:
-  push:
-    branches: [ master ]
-    paths:
-      - "packages/noco-book/**"
-
-jobs:
-  copy-file:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-node@v2
-        with:
-          node-version: 14
-      - name: Build book
-        run: |
-          cd packages/noco-book
-          npm install
-          npm run generate
-
-
-      - name: Pushes generated output
-        uses: dmnemec/copy_file_to_another_repo_action@1b29cbd9a323185f20b175dc6d5f8f31be5c0658
-        env:
-          API_TOKEN_GITHUB: ${{ secrets.GH_TOKEN }}
-        with:
-          clear_folder: 'docs/dist'
-          source_file: 'packages/noco-book/dist/'
-          destination_repo: 'nocodb/noco-book'
-          destination_folder: 'docs'
-          user_email: 'oof1lab@gmail.com'
-          user_name: 'o1lab'
-          commit_message: 'Autorelease from github.com/nocodb/nocodb/packages/noco-book'

packages/nc-gui/components/import/templateParsers/ExcelTemplateAdapter.js

@@ -42,25 +42,26 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
         tableNamePrefixRef[tn] = 0
       }
 
-      const table = { tn, columns: [] }
+      const table = { tn, refTn: tn, columns: [] }
       this.data[sheet] = []
       const ws = this.wb.Sheets[sheet]
       const range = XLSX.utils.decode_range(ws['!ref'])
       const rows = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false, cellDates: true, defval: null })
+      const columnNameRowExist = +rows[0].every(v => v === null || typeof v === 'string')
 
       // const colLen = Math.max()
       for (let col = 0; col < rows[0].length; col++) {
-        let cn = ((rows[0] && rows[0][col] && rows[0][col].toString().trim()) ||
-          `field${col + 1}`).replace(/\W/g, '_').trim()
+        let cn = ((columnNameRowExist && rows[0] && rows[0][col] && rows[0][col].toString().trim()) ||
+          `field_${col + 1}`).replace(/\W/g, '_').trim()
 
-        // todo: look for duplicate
-        if (cn in columnNamePrefixRef) {
+        while (cn in columnNamePrefixRef) {
           cn = `${cn}${++columnNamePrefixRef[cn]}`
-        } else {
-          columnNamePrefixRef[cn] = 0
         }
+        columnNamePrefixRef[cn] = 0
+
         const column = {
-          cn
+          cn,
+          refCn: cn
         }
 
         table.columns.push(column)
@@ -68,7 +69,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
         // const cellId = `${col.toString(26).split('').map(s => (parseInt(s, 26) + 10).toString(36).toUpperCase())}2`;
         const cellId = XLSX.utils.encode_cell({
           c: range.s.c + col,
-          r: 1
+          r: columnNameRowExist
         })
         const cellProps = ws[cellId] || {}
         column.uidt = excelTypeToUidt[cellProps.t] || UITypes.SingleLineText
@@ -82,7 +83,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
           ) {
             column.uidt = UITypes.LongText
           } else {
-            let vals = rows.slice(1).map(r => r[col])
+            let vals = rows.slice(columnNameRowExist ? 1 : 0).map(r => r[col])
 
             const checkboxType = isCheckboxType(vals)
             if (checkboxType.length === 1) {
@@ -119,7 +120,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
           if (rows.slice(1, this.config.maxRowsToParse).every((v, i) => {
             const cellId = XLSX.utils.encode_cell({
               c: range.s.c + col,
-              r: i + 2
+              r: i + (columnNameRowExist)
             })
 
             const cellObj = ws[cellId]
@@ -132,7 +133,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
           if (rows.slice(1, this.config.maxRowsToParse).every((v, i) => {
             const cellId = XLSX.utils.encode_cell({
               c: range.s.c + col,
-              r: i + 1
+              r: i + columnNameRowExist
             })
 
             const cellObj = ws[cellId]
@@ -154,7 +155,7 @@ export default class ExcelTemplateAdapter extends TemplateGenerator {
 
             const cellId = XLSX.utils.encode_cell({
               c: range.s.c + i,
-              r: rowIndex + 1
+              r: rowIndex + columnNameRowExist
             })
 
             const cellObj = ws[cellId]

packages/nc-gui/components/templates/createProjectFromTemplateBtn.vue

@@ -173,7 +173,7 @@ export default {
       let total = 0
       let progress = 0
 
-      await Promise.all(Object.entries(this.importData).map(v => (async([table, data]) => {
+      /*      await Promise.all(Object.entries(this.importData).map(v => (async([table, data]) => {
         await this.$store.dispatch('meta/ActLoadMeta', {
           tn: `${prefix}${table}`, project_id: projectId
         })
@@ -192,7 +192,38 @@ export default {
           progress += batchData.length
         }
         this.$store.commit('loader/MutClear')
+      })(v))) */
+
+      await Promise.all(this.templateData.tables.map(v => (async(tableMeta) => {
+        const table = tableMeta.tn
+        const data = this.importData[tableMeta.refTn]
+
+        await this.$store.dispatch('meta/ActLoadMeta', {
+          tn: `${prefix}${table}`, project_id: projectId
+        })
+
+        // todo: get table name properly
+        const api = this.$ncApis.get({
+          table: `${prefix}${table}`,
+          type: projectType
+        })
+        total += data.length
+        for (let i = 0; i < data.length; i += 500) {
+          this.$store.commit('loader/MutMessage', `Importing data : ${progress}/${total}`)
+          this.$store.commit('loader/MutProgress', Math.round(progress && 100 * progress / total))
+          const batchData = this.remapColNames(data.slice(i, i + 500), tableMeta.columns)
+          await api.insertBulk(batchData)
+          progress += batchData.length
+        }
+        this.$store.commit('loader/MutClear')
       })(v)))
+    },
+    remapColNames(batchData, columns) {
+      return batchData.map(data => (columns || []).reduce((aggObj, col) => ({
+        ...aggObj,
+        [col.cn]: data[col.refCn]
+      }), {})
+      )
     }
   }
 

packages/nc-gui/components/templates/editor.vue

@@ -659,7 +659,7 @@ export default {
         const template = {
           ...this.project,
           tables: (this.project.tables || []).map((t) => {
-            const table = { tn: t.tn, columns: [], hasMany: [], manyToMany: [], belongsTo: [], v: [] }
+            const table = { ...t, columns: [], hasMany: [], manyToMany: [], belongsTo: [], v: [] }
 
             for (const column of (t.columns || [])) {
               if (this.isRelation(column)) {
@@ -1300,6 +1300,7 @@ export default {
     },
     onUidtChange(oldVal, newVal, col, table) {
       this.$set(col, 'uidt', newVal)
+      this.$set(col, 'dtxp', undefined)
 
       // delete relation column from other table
       // if previous type is relation

packages/noco-book/.gitignore

@@ -1,11 +0,0 @@
-node_modules
-*.iml
-.idea
-*.log*
-.nuxt
-.vscode
-.DS_Store
-coverage
-dist
-sw.*
-.env

packages/noco-book/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2021 nocodb
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

packages/noco-book/README.md

@@ -1 +0,0 @@
-# `noco-book`

packages/noco-book/assets/main.css

@@ -1,7 +0,0 @@
-main.container{
-    max-width:100% !important;
-}
-
-footer a[href="https://nuxtjs.org"]{
-    display: none;
-}

packages/noco-book/components/global/youtube.vue

@@ -1,33 +0,0 @@
-<template>
-  <div>
-    <iframe type="text/html" width="100%" style="height:100%"
-            :src="`https://www.youtube.com/embed/${id}`"
-            frameborder="0" allowfullscreen></iframe>
-  </div>
-</template>
-
-<script>
-export default {
-  name: "youtube",
-  props: {
-    id: String
-  }
-}
-</script>
-
-<style scoped>
-div {
-  background-color: red;
-  width: 100%;
-  padding-top: min(500px,56%);
-  position: relative;
-}
-
-iframe {
-  position: absolute;
-  top: 0;
-  left: 0;
-  bottom: 0;
-  right: 0;
-}
-</style>

packages/noco-book/content/en/index.md

@@ -1,84 +0,0 @@
----
-title: 'Setup and Usage'
-description: 'Simple installation - takes about three minutes!'
-position: 0
-category: 'Getting started'
-fullscreen: true
-menuTitle: 'Install'
-link: https://codesandbox.io/embed/vigorous-firefly-80kq5?hidenavigation=1&theme=dark
----
-
-
-## Simple installation - takes about three minutes!
-
-### Prerequisites
-
-- __Must haves__
-    * [node.js >= 12](https://nodejs.org/en/download) / [Docker](https://www.docker.com/get-started)
-    * [MySql](https://dev.mysql.com/downloads/mysql/) / [Postgres](https://www.postgresql.org/download/) / [SQLserver](https://www.microsoft.com/en-gb/sql-server/sql-server-downloads) / SQLite Database
-- Nice to haves
-    - Existing schemas can help to create APIs quickly.
-    - An example database schema can be found :<a class="grey--text"
-                                                                                         href="https://github.com/lerocha/chinook-database/tree/master/ChinookDatabase/DataSources">
-                        <u>here</u>
-                    </a>
-    
-
-
-## Quick try
-### 1-Click Deploy
-<a href="https://heroku.com/deploy?template=https://github.com/nocodb/nocodb-seed-heroku">
-    <img 
-    src="https://www.herokucdn.com/deploy/button.svg" 
-    width="300px"
-    alt="Deploy NocoDB to Heroku with 1-Click" 
-    />
-</a>
-
-### Node app or docker 
-
-
-<code-group>
-  <code-block label="NPX" active> 
-
-  ```bash
-  npx create-nocodb-app
-  ```
-
-  </code-block>
-  <code-block label="Docker" >
-
-  ```bash
-  docker run -d --name nocodb -p 8080:8080 nocodb/nocodb:latest
-  ```
-
-  </code-block>
-  <code-block label="Using Git" >
-
-  ```bash
-git clone https://github.com/nocodb/nocodb-seed
-cd nocodb-seed
-npm install
-npm start
-  ```
-
-  </code-block>
-</code-group>               
-                  
-          
-### Sample app        
-<code-sandbox :src="link"></code-sandbox>
-
-
-# Sample Demos
-### Docker deploying with one command
-
-<youtube id="K-UEecQyiOk"></youtube>
-
-### Using Npx
-
-<youtube id="v6Nn75P1p7I"></youtube>
-
-### Heroku Deployment
-<youtube id="v6Nn75P1p7I"></youtube>
-

packages/noco-book/content/settings.json

@@ -1,10 +0,0 @@
-{
-  "title": "NocoDB",
-  "url": "https://handbook.nocodb.com",
-  "logo": {
-    "light": "/favicon-128.png",
-    "dark": "/favicon-128.png"
-  },
-  "github": "nocodb/nocodb/packages/noco-book",
-  "twitter": "@nocodb"
-}

packages/noco-book/nuxt.config.js

@@ -1,10 +0,0 @@
-import theme from '@nuxt/content-theme-docs'
-
-export default theme({
-  docs: {
-    primaryColor: '#3282ff'
-  },
-  css: [
-    "./assets/main.css"
-  ]
-})

packages/noco-book/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "noco-book",
-  "version": "1.0.0",
-  "license": "MIT",
-  "scripts": {
-    "dev": "nuxt",
-    "build": "nuxt build",
-    "start": "nuxt start",
-    "generate": "nuxt generate"
-  },
-  "dependencies": {
-    "@nuxt/content-theme-docs": "0.7.2",
-    "nuxt": "^2.15.2"
-  }
-}