feat: JSON import

- Email and URL support
- Nested Data normalization
- Handle both object and array
- Optional data import option

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nc-gui/components/import/JSONImport.vue

@@ -2,7 +2,7 @@
   <div :class="{'pt-10':!hideLabel}">
     <v-dialog v-model="dropOrUpload" max-width="600">
       <v-card max-width="600">
-        <v-tabs height="30" :value="2">
+        <v-tabs height="30">
           <v-tab>
             <v-icon small class="mr-1">
               mdi-file-upload-outline
@@ -61,18 +61,18 @@
               <div class="pa-4 d-100 h-100">
                 <v-form ref="form" v-model="valid">
                   <div class="d-flex">
-                    <!--label="Enter excel file url"-->
+                    <!-- todo:  i18n label-->
                     <v-text-field
                       v-model="url"
                       hide-details="auto"
                       type="url"
-                      :label="quickImportType == 'excel' ? $t('msg.info.excelURL') : $t('msg.info.csvURL') "
+                      label="Enter JSON file url"
                       class="caption"
                       outlined
                       dense
                       :rules="[v => !!v || $t('general.required') ]"
                     />
-                    <v-btn v-t="['c:project:create:excel:load-url']" class="ml-3" color="primary" @click="loadUrl">
+                    <v-btn v-t="['c:project:create:json:load-url']" class="ml-3" color="primary" @click="loadUrl">
                       <!--Load-->
                       {{ $t('general.load') }}
                     </v-btn>
@@ -85,9 +85,14 @@
             <div class="nc-json-import-tab-item align-center">
               <div class="pa-4 d-100 h-100">
                 <v-form ref="form" v-model="valid">
-                  <div class="">
+                  <div class="nc-json-editor-wrapper">
+                    <v-btn small class="nc-json-format-btn" @click="formatJson">
+                      Format
+                    </v-btn>
+
                     <!--label="Enter excel file url"-->
                     <monaco-json-editor
+                      ref="editor"
                       v-model="jsonString"
                       style="height:320px"
                     />
@@ -137,6 +142,17 @@
                 <span class="caption">Normalize nested</span>
               </template>
             </v-checkbox>
+            <v-checkbox
+              v-model="parserConfig.importData"
+              style="width: 250px"
+              class="mx-auto mb-2"
+              dense
+              hide-details
+            >
+              <template #label>
+                <span class="caption">Import data</span>
+              </template>
+            </v-checkbox>
           </div>
         </div>
       </v-card>
@@ -220,9 +236,9 @@
 
 import TemplateEditor from '~/components/templates/Editor'
 import CreateProjectFromTemplateBtn from '~/components/templates/CreateProjectFromTemplateBtn'
-import ExcelUrlTemplateAdapter from '~/components/import/templateParsers/ExcelUrlTemplateAdapter'
 import MonacoJsonEditor from '~/components/monaco/MonacoJsonEditor'
 import JSONTemplateAdapter from '~/components/import/templateParsers/JSONTemplateAdapter'
+import JSONUrlTemplateAdapter from '~/components/import/templateParsers/JSONUrlTemplateAdapter'
 
 export default {
   name: 'JsonImport',
@@ -244,7 +260,8 @@ export default {
       showMore: false,
       parserConfig: {
         maxRowsToParse: 500,
-        normalizeNested: false
+        normalizeNested: false,
+        importData: true
       },
       filename: '',
       jsonString: ''
@@ -258,6 +275,9 @@ export default {
       get() {
         return this.value
       }
+    },
+    tables() {
+      return this.$store.state.project.tables || []
     }
   },
   mounted() {
@@ -267,6 +287,10 @@ export default {
     }
   },
   methods: {
+    formatJson() {
+      console.log(this.$refs.editor)
+      this.$refs.editor.format()
+    },
 
     selectFile() {
       this.$refs.file.files = null
@@ -319,19 +343,21 @@ export default {
         this.importData = null
         switch (type) {
           case 'file':
-            templateGenerator = new JSONTemplateAdapter(name, val, this.parserConfig)
+            templateGenerator = new JSONTemplateAdapter('JSON', val, this.parserConfig)
             break
           case 'url':
-            templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store, this.parserConfig, this.$api)
+            templateGenerator = new JSONUrlTemplateAdapter(val, this.$store, this.parserConfig, this.$api)
-            templateGenerator = new ExcelUrlTemplateAdapter(val, this.$store, this.parserConfig, this.$api)
             break
           case 'string':
-            templateGenerator = new JSONTemplateAdapter('test', val, this.parserConfig)
+            templateGenerator = new JSONTemplateAdapter('JSON', val, this.parserConfig)
             break
         }
         await templateGenerator.init()
         templateGenerator.parse()
         this.templateData = templateGenerator.getTemplate()
+
+        this.templateData.tables[0].table_name = this.populateUniqueTableName()
+
         this.importData = templateGenerator.getData()
         this.templateEditorModal = true
       } catch (e) {
@@ -364,7 +390,11 @@ export default {
       // Prevent default behavior (Prevent file from being opened)
       ev.preventDefault()
     },
-
+    populateUniqueTableName() {
+      let c = 1
+      while (this.tables.some(t => t.title === `Sheet${c}`)) { c++ }
+      return `Sheet${c}`
+    },
     async loadUrl() {
       if ((this.$refs.form && !this.$refs.form.validate()) || !this.url) {
         return
@@ -413,4 +443,15 @@ export default {
   transition: .4s max-height;
   overflow: hidden;
 }
+
+.nc-json-editor-wrapper{
+  position: relative;
+}
+
+.nc-json-format-btn{
+  position:absolute;
+  right:4px;
+  top:4px;
+  z-index:9;
+}
 </style>

packages/nc-gui/components/import/templateParsers/JSONTemplateAdapter.js

@@ -1,15 +1,21 @@
-import { TemplateGenerator } from 'nocodb-sdk'
+import { TemplateGenerator, UITypes } from 'nocodb-sdk'
-import { UITypes } from '~/components/project/spreadsheet/helpers/uiTypes'
+import {
-import { getCheckboxValue, isCheckboxType } from '~/components/import/templateParsers/parserHelpers'
+  extractMultiOrSingleSelectProps,
+  getCheckboxValue,
+  isCheckboxType, isDecimalType, isEmailType,
+  isMultiLineTextType, isUrlType
+} from '~/components/import/templateParsers/parserHelpers'
 
 const jsonTypeToUidt = {
   number: UITypes.Number,
   string: UITypes.SingleLineText,
   date: UITypes.DateTime,
   boolean: UITypes.Checkbox,
-  object: UITypes.LongText
+  object: UITypes.JSON
 }
 
+const extractNestedData = (obj, path) => path.reduce((val, key) => val && val[key], obj)
+
 export default class JSONTemplateAdapter extends TemplateGenerator {
   constructor(name = 'test', data, parserConfig = {}) {
     super()
@@ -18,7 +24,7 @@ export default class JSONTemplateAdapter extends TemplateGenerator {
       ...parserConfig
     }
     this.name = name
-    this.jsonData = typeof data === 'string' ? JSON.parse(data) : data
+    this._jsonData = typeof data === 'string' ? JSON.parse(data) : data
     this.project = {
       title: this.name,
       tables: []
@@ -42,117 +48,103 @@ export default class JSONTemplateAdapter extends TemplateGenerator {
     return this.data
   }
 
-  parse() {
+  get jsonData() {
-    const jsonData = Array.isArray(this.this.jsonData) ? this.this.jsonData : [this.jsonData]
+    return Array.isArray(this._jsonData) ? this._jsonData : [this._jsonData]
-
+  }
-    // for (let i = 0; i < this.wb.SheetNames.length; i++) {
-    // const columnNamePrefixRef = { id: 0 }
 
+  parse() {
+    const jsonData = this.jsonData
     const tn = 'table'
-
     const table = { table_name: tn, ref_table_name: tn, columns: [] }
 
     this.data[tn] = []
 
-    // const ws = this.wb.Sheets[sheet]
+    for (const col of Object.keys(jsonData[0])) {
-    // const range = XLSX.utils.decode_range(ws['!ref'])
+      const columns = this._parseColumn([col], jsonData)
-    // const rows = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false, cellDates: true, defval: null })
+      table.columns.push(...columns)
+    }
+
+    if (this.config.importData) { this._parseTableData(table) }
+
+    this.project.tables.push(table)
+  }
 
-    const objKeys = Object.keys(jsonData[0])
+  getTemplate() {
+    return this.project
+  }
 
-    for (let col = 0; col < objKeys.length; col++) {
+  _parseColumn(path = [], jsonData = this.jsonData, firstRowVal = path.reduce((val, k) => val && val[k], this.jsonData[0])) {
-      const key = objKeys[col]
+    const columns = []
-      const cn = objKeys[col].replace(/\W/g, '_').trim()
+    // parse nested
+    if (firstRowVal && typeof firstRowVal === 'object' && !Array.isArray(firstRowVal) && this.config.normalizeNested) {
+      for (const key of Object.keys(firstRowVal)) {
+        const normalizedNestedColumns = this._parseColumn([...path, key], this.jsonData, firstRowVal[key])
+        columns.push(...normalizedNestedColumns)
+      }
+    } else {
+      const cn = path.join('_').replace(/\W/g, '_').trim()
 
       const column = {
         column_name: cn,
-        ref_column_name: cn
+        ref_column_name: cn,
+        path
       }
 
-      table.columns.push(column)
+      column.uidt = jsonTypeToUidt[typeof firstRowVal] || UITypes.SingleLineText
+
+      const colData = jsonData.map(r => extractNestedData(r, path))
+      Object.assign(column, this._getColumnUIDTAndMetas(colData, column.uidt))
+      columns.push(column)
+    }
 
-      column.uidt = jsonTypeToUidt[typeof jsonData[0][key]] || UITypes.SingleLineText
+    return columns
+  }
 
-      // todo: optimize
+  _getColumnUIDTAndMetas(colData, defaultType) {
-      if (column.uidt === UITypes.SingleLineText) {
+    const colProps = { uidt: defaultType }
-        // check for long text
+    // todo: optimize
-        if (jsonData.some(r =>
+    if (colProps.uidt === UITypes.SingleLineText) {
-          (r[key] || '').toString().match(/[\r\n]/) ||
+      // check for long text
-          (r[key] || '').toString().length > 255)
+      if (isMultiLineTextType(colData)) {
-        ) {
+        colProps.uidt = UITypes.LongText
-          column.uidt = UITypes.LongText
+      } if (isEmailType(colData)) {
+        colProps.uidt = UITypes.Email
+      } if (isUrlType(colData)) {
+        colProps.uidt = UITypes.URL
+      } else {
+        const checkboxType = isCheckboxType(colData)
+        if (checkboxType.length === 1) {
+          colProps.uidt = UITypes.Checkbox
         } else {
-          const vals = jsonData
+          Object.assign(colProps, extractMultiOrSingleSelectProps(colData))
-            .map(r => r[key])
-            .filter(v => v !== null && v !== undefined && v.toString().trim() !== '')
-
-          const checkboxType = isCheckboxType(vals)
-          if (checkboxType.length === 1) {
-            column.uidt = UITypes.Checkbox
-          } else {
-            // todo: optimize
-            // check column is multi or single select by comparing unique values
-            // todo:
-            // eslint-disable-next-line no-lonely-if
-            if (vals.some(v => v && v.toString().includes(','))) {
-              let flattenedVals = vals.flatMap(v => v ? v.toString().trim().split(/\s*,\s*/) : [])
-              const uniqueVals = flattenedVals = flattenedVals
-                .filter((v, i, arr) => i === arr.findIndex(v1 => v.toLowerCase() === v1.toLowerCase()))
-              if (flattenedVals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(flattenedVals.length / 2)) {
-                column.uidt = UITypes.MultiSelect
-                column.dtxp = `'${uniqueVals.join("','")}'`
-              }
-            } else {
-              const uniqueVals = vals.map(v => v.toString().trim()).filter((v, i, arr) => i === arr.findIndex(v1 => v.toLowerCase() === v1.toLowerCase()))
-              if (vals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(vals.length / 2)) {
-                column.uidt = UITypes.SingleSelect
-                column.dtxp = `'${uniqueVals.join("','")}'`
-              }
-            }
-          }
-        }
-      } else if (column.uidt === UITypes.Number) {
-        if (jsonData.slice(1, this.config.maxRowsToParse).some((v) => {
-          return v && v[key] && parseInt(+v[key]) !== +v[key]
-        })) {
-          column.uidt = UITypes.Decimal
-        }
-        if (jsonData.every((v, i) => {
-          return v[key] && v[key].toString().startsWith('$')
-        })) {
-          column.uidt = UITypes.Currency
-        }
-      } else if (column.uidt === UITypes.DateTime) {
-        if (jsonData.every((v, i) => {
-          return v[key] && v[key].toString().split(' ').length === 1
-        })) {
-          column.uidt = UITypes.Date
         }
       }
+    } else if (colProps.uidt === UITypes.Number) {
+      if (isDecimalType(colData)) {
+        colProps.uidt = UITypes.Decimal
+      }
     }
+    return colProps
+  }
 
-    // let rowIndex = 0
+  _parseTableData(tableMeta) {
-    for (const row of jsonData) {
+    for (const row of this.jsonData) {
       const rowData = {}
-      for (let i = 0; i < table.columns.length; i++) {
+      for (let i = 0; i < tableMeta.columns.length; i++) {
-        if (table.columns[i].uidt === UITypes.Checkbox) {
+        const value = extractNestedData(row, tableMeta.columns[i].path || [])
-          rowData[table.columns[i].column_name] = getCheckboxValue(row[i])
+        if (tableMeta.columns[i].uidt === UITypes.Checkbox) {
-        } else if (table.columns[i].uidt === UITypes.Currency) {
+          rowData[tableMeta.columns[i].ref_column_name] = getCheckboxValue(value)
-          rowData[table.columns[i].column_name] = (row[table.columns[i].ref_column_name].replace(/[^\d.]+/g, '')) || row[i]
+        } else if (tableMeta.columns[i].uidt === UITypes.SingleSelect || tableMeta.columns[i].uidt === UITypes.MultiSelect) {
-        } else if (table.columns[i].uidt === UITypes.SingleSelect || table.columns[i].uidt === UITypes.MultiSelect) {
+          rowData[tableMeta.columns[i].ref_column_name] = (value || '').toString().trim() || null
-          rowData[table.columns[i].column_name] = (row[table.columns[i].ref_column_name] || '').toString().trim() || null
+        } else if (tableMeta.columns[i].uidt === UITypes.JSON) {
+          rowData[tableMeta.columns[i].ref_column_name] = JSON.stringify(value)
         } else {
           // toto: do parsing if necessary based on type
-          rowData[table.columns[i].column_name] = row[table.columns[i].ref_column_name]
+          rowData[tableMeta.columns[i].column_name] = value
         }
       }
-      this.data[tn].push(rowData)
+      this.data[tableMeta.ref_table_name].push(rowData)
       // rowIndex++
     }
-    this.project.tables.push(table)
-  }
-
-  getTemplate() {
-    return this.project
   }
 }

packages/nc-gui/components/import/templateParsers/JSONUrlTemplateAdapter.js

@@ -1,16 +1,6 @@
-import { TemplateGenerator } from 'nocodb-sdk'
+import JSONTemplateAdapter from '~/components/import/templateParsers/JSONTemplateAdapter'
-import { UITypes } from '~/components/project/spreadsheet/helpers/uiTypes'
-import { getCheckboxValue, isCheckboxType } from '~/components/import/templateParsers/parserHelpers'
 
-const jsonTypeToUidt = {
+export default class JSONUrlTemplateAdapter extends JSONTemplateAdapter {
-  number: UITypes.Number,
-  string: UITypes.SingleLineText,
-  date: UITypes.DateTime,
-  boolean: UITypes.Checkbox,
-  object: UITypes.LongText
-}
-
-export default class JSONTemplateAdapter extends JSONTemplateAdapter {
   constructor(url, $store, parserConfig, $api) {
     const name = url.split('/').pop()
     super(name, null, parserConfig)
@@ -22,11 +12,10 @@ export default class JSONTemplateAdapter extends JSONTemplateAdapter {
   async init() {
     const data = await this.$api.utils.axiosRequestMake({
       apiMeta: {
-        url: this.url,
+        url: this.url
-        responseType: 'arraybuffer'
       }
     })
-    this.jsonData = data.data
+    this._jsonData = data
     await super.init()
   }
 }

packages/nc-gui/components/import/templateParsers/parserHelpers.js

@@ -1,3 +1,6 @@
+import { UITypes } from 'nocodb-sdk'
+import { isEmail, isValidURL } from '~/helpers'
+
 const booleanOptions = [
   { checked: true, unchecked: false },
   { x: true, '': false },
@@ -11,14 +14,24 @@ const booleanOptions = [
   { '✔': true, '': false },
   { enabled: true, disabled: false },
   { on: true, off: false },
-  { done: true, '': false }
+  { done: true, '': false },
+  { true: true, false: false }
 ]
 const aggBooleanOptions = booleanOptions.reduce((obj, o) => ({ ...obj, ...o }), {})
-export const isCheckboxType = (values, col = '') => {
+
+const getColVal = (row, col = null) => {
+  return row && col ? row[col] : row
+}
+
+export const isCheckboxType = (values, col = null) => {
   let options = booleanOptions
   for (let i = 0; i < values.length; i++) {
-    let val = col ? values[i][col] : values[i]
+    const val = getColVal(values[i], col)
-    val = val === null || val === undefined ? '' : val
+
+    if (val === null || val === undefined || val.toString().trim() === '') {
+      continue
+    }
+
     options = options.filter(v => val in v)
     if (!options.length) {
       return false
@@ -29,3 +42,40 @@ export const isCheckboxType = (values, col = '') => {
 export const getCheckboxValue = (value) => {
   return value && aggBooleanOptions[value]
 }
+
+export const isMultiLineTextType = (values, col = null) => {
+  return values.some(r =>
+    (getColVal(r, col) || '').toString().match(/[\r\n]/) ||
+    (getColVal(r, col) || '').toString().length > 255)
+}
+
+export const extractMultiOrSingleSelectProps = (colData) => {
+  const colProps = {}
+  if (colData.some(v => v && v.toString().includes(','))) {
+    let flattenedVals = colData.flatMap(v => v ? v.toString().trim().split(/\s*,\s*/) : [])
+    const uniqueVals = flattenedVals = flattenedVals
+      .filter((v, i, arr) => i === arr.findIndex(v1 => v.toLowerCase() === v1.toLowerCase()))
+    if (flattenedVals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(flattenedVals.length / 2)) {
+      colProps.uidt = UITypes.MultiSelect
+      colProps.dtxp = `'${uniqueVals.join("','")}'`
+    }
+  } else {
+    const uniqueVals = colData.map(v => v.toString().trim()).filter((v, i, arr) => i === arr.findIndex(v1 => v.toLowerCase() === v1.toLowerCase()))
+    if (colData.length > uniqueVals.length && uniqueVals.length <= Math.ceil(colData.length / 2)) {
+      colProps.uidt = UITypes.SingleSelect
+      colProps.dtxp = `'${uniqueVals.join("','")}'`
+    }
+  }
+  return colProps
+}
+
+export const isDecimalType = colData => colData.some((v) => {
+  return v && parseInt(+v) !== +v
+})
+
+export const isEmailType = colData => !colData.some((v) => {
+  return v && !isEmail(v)
+})
+export const isUrlType = colData => !colData.some((v) => {
+  return v && !isValidURL(v)
+})

packages/nc-gui/components/monaco/MonacoJsonEditor.js

@@ -83,6 +83,9 @@ export default {
   },
 
   methods: {
+    format() {
+      this.editor.getAction('editor.action.formatDocument').run()
+    },
     resizeLayout() {
       this.editor.layout();
     },