From f8dff44cc7e1f3060f9f6471536afeac81a29f75 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 15 Jun 2022 23:32:52 +0530 Subject: [PATCH] wip: JSON import adapter implementation Signed-off-by: Pranav C --- packages/nc-gui/components/ProjectTabs.vue | 27 +- .../nc-gui/components/import/JSONImport.vue | 436 ++++++++++++++++++ .../templateParsers/CSVTemplateAdapter.js | 1 - .../templateParsers/JSONTemplateAdapter.js | 156 +++++++ packages/nocodb-sdk/src/index.ts | 3 +- .../nocodb-sdk/src/lib/TemplateGenerator.ts | 30 ++ 6 files changed, 650 insertions(+), 3 deletions(-) create mode 100644 packages/nc-gui/components/import/JSONImport.vue create mode 100644 packages/nc-gui/components/import/templateParsers/JSONTemplateAdapter.js create mode 100644 packages/nocodb-sdk/src/lib/TemplateGenerator.ts diff --git a/packages/nc-gui/components/ProjectTabs.vue b/packages/nc-gui/components/ProjectTabs.vue index a3fd6392ec..0a69cd3360 100644 --- a/packages/nc-gui/components/ProjectTabs.vue +++ b/packages/nc-gui/components/ProjectTabs.vue @@ -315,6 +315,21 @@ + + + + mdi-code-json + + + + JSON file + + + + + + @@ -404,9 +426,11 @@ import GlobalAcl from '~/components/GlobalAcl' import AuditTab from '~/components/project/AuditTab' import QuickImport from '~/components/import/QuickImport' import ImportFromAirtable from '~/components/import/ImportFromAirtable' +import JsonImport from '~/components/import/JSONImport' export default { components: { + JsonImport, ImportFromAirtable, SwaggerClient, // Screensaver, @@ -447,7 +471,8 @@ export default { showScreensaver: false, quickImportModal: false, quickImportType: '', - airtableImportModal: false + airtableImportModal: false, + jsonImportModal: false } }, methods: { diff --git a/packages/nc-gui/components/import/JSONImport.vue b/packages/nc-gui/components/import/JSONImport.vue new file mode 100644 index 0000000000..fc55b865b3 --- /dev/null +++ b/packages/nc-gui/components/import/JSONImport.vue @@ -0,0 +1,436 @@ + + + + + diff --git a/packages/nc-gui/components/import/templateParsers/CSVTemplateAdapter.js b/packages/nc-gui/components/import/templateParsers/CSVTemplateAdapter.js index 9a3be0d4de..c608a79f85 100644 --- a/packages/nc-gui/components/import/templateParsers/CSVTemplateAdapter.js +++ b/packages/nc-gui/components/import/templateParsers/CSVTemplateAdapter.js @@ -1,6 +1,5 @@ import Papaparse from 'papaparse' import TemplateGenerator from '~/components/import/templateParsers/TemplateGenerator' - export default class CSVTemplateAdapter extends TemplateGenerator { constructor(name, data) { super() diff --git a/packages/nc-gui/components/import/templateParsers/JSONTemplateAdapter.js b/packages/nc-gui/components/import/templateParsers/JSONTemplateAdapter.js new file mode 100644 index 0000000000..14977ebdd7 --- /dev/null +++ b/packages/nc-gui/components/import/templateParsers/JSONTemplateAdapter.js @@ -0,0 +1,156 @@ +import { TemplateGenerator } from 'nocodb-sdk' +import { UITypes } from '~/components/project/spreadsheet/helpers/uiTypes' +import { getCheckboxValue, isCheckboxType } from '~/components/import/templateParsers/parserHelpers' + +const jsonTypeToUidt = { + number: UITypes.Number, + string: UITypes.SingleLineText, + date: UITypes.DateTime, + boolean: UITypes.Checkbox, + object: UITypes.LongText +} + +export default class JSONTemplateAdapter extends TemplateGenerator { + constructor(name = 'test', data, parserConfig = {}) { + super() + this.config = { + maxRowsToParse: 500, + ...parserConfig + } + this.name = name + this.jsonData = typeof data === 'string' ? JSON.parse(data) : data + this.project = { + title: this.name, + tables: [] + } + this.data = {} + } + + async init() { + } + + parseData() { + this.columns = this.csv.meta.fields + this.data = this.csv.data + } + + getColumns() { + return this.columns + } + + getData() { + return this.data + } + + parse() { + // for (let i = 0; i < this.wb.SheetNames.length; i++) { + // const columnNamePrefixRef = { id: 0 } + + const tn = 'table' + + const table = { table_name: tn, ref_table_name: tn, columns: [] } + + this.data[tn] = [] + + // const ws = this.wb.Sheets[sheet] + // const range = XLSX.utils.decode_range(ws['!ref']) + // const rows = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false, cellDates: true, defval: null }) + + const objKeys = Object.keys(this.jsonData[0]) + + for (let col = 0; col < objKeys.length; col++) { + const key = objKeys[col] + const cn = objKeys[col].replace(/\W/g, '_').trim() + + const column = { + column_name: cn, + ref_column_name: cn + } + + table.columns.push(column) + + column.uidt = jsonTypeToUidt[typeof this.jsonData[0][key]] || UITypes.SingleLineText + + // todo: optimize + if (column.uidt === UITypes.SingleLineText) { + // check for long text + if (this.jsonData.some(r => + (r[key] || '').toString().match(/[\r\n]/) || + (r[key] || '').toString().length > 255) + ) { + column.uidt = UITypes.LongText + } else { + const vals = this.jsonData + .map(r => r[key]) + .filter(v => v !== null && v !== undefined && v.toString().trim() !== '') + + const checkboxType = isCheckboxType(vals) + if (checkboxType.length === 1) { + column.uidt = UITypes.Checkbox + } else { + // todo: optimize + // check column is multi or single select by comparing unique values + // todo: + // eslint-disable-next-line no-lonely-if + if (vals.some(v => v && v.toString().includes(','))) { + let flattenedVals = vals.flatMap(v => v ? v.toString().trim().split(/\s*,\s*/) : []) + const uniqueVals = flattenedVals = flattenedVals + .filter((v, i, arr) => i === arr.findIndex(v1 => v.toLowerCase() === v1.toLowerCase())) + if (flattenedVals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(flattenedVals.length / 2)) { + column.uidt = UITypes.MultiSelect + column.dtxp = `'${uniqueVals.join("','")}'` + } + } else { + const uniqueVals = vals.map(v => v.toString().trim()).filter((v, i, arr) => i === arr.findIndex(v1 => v.toLowerCase() === v1.toLowerCase())) + if (vals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(vals.length / 2)) { + column.uidt = UITypes.SingleSelect + column.dtxp = `'${uniqueVals.join("','")}'` + } + } + } + } + } else if (column.uidt === UITypes.Number) { + if (this.jsonData.slice(1, this.config.maxRowsToParse).some((v) => { + return v && v[key] && parseInt(+v[key]) !== +v[key] + })) { + column.uidt = UITypes.Decimal + } + if (this.jsonData.every((v, i) => { + return v[key] && v[key].toString().startsWith('$') + })) { + column.uidt = UITypes.Currency + } + } else if (column.uidt === UITypes.DateTime) { + if (this.jsonData.every((v, i) => { + return v[key] && v[key].toString().split(' ').length === 1 + })) { + column.uidt = UITypes.Date + } + } + } + + // let rowIndex = 0 + for (const row of this.jsonData) { + const rowData = {} + for (let i = 0; i < table.columns.length; i++) { + if (table.columns[i].uidt === UITypes.Checkbox) { + rowData[table.columns[i].column_name] = getCheckboxValue(row[i]) + } else if (table.columns[i].uidt === UITypes.Currency) { + rowData[table.columns[i].column_name] = (row[table.columns[i].ref_column_name].replace(/[^\d.]+/g, '')) || row[i] + } else if (table.columns[i].uidt === UITypes.SingleSelect || table.columns[i].uidt === UITypes.MultiSelect) { + rowData[table.columns[i].column_name] = (row[table.columns[i].ref_column_name] || '').toString().trim() || null + } else { + // toto: do parsing if necessary based on type + rowData[table.columns[i].column_name] = row[table.columns[i].ref_column_name] + } + } + this.data[tn].push(rowData) + // rowIndex++ + } + this.project.tables.push(table) + } + + getTemplate() { + return this.project + } +} diff --git a/packages/nocodb-sdk/src/index.ts b/packages/nocodb-sdk/src/index.ts index 805152d812..561b841c36 100644 --- a/packages/nocodb-sdk/src/index.ts +++ b/packages/nocodb-sdk/src/index.ts @@ -5,6 +5,7 @@ export * from './lib/sqlUi'; export * from './lib/globals'; export * from './lib/helperFunctions'; export * from './lib/formulaHelpers'; -export * from './lib/passwordHelpers'; export { default as UITypes, isVirtualCol } from './lib/UITypes'; export { default as CustomAPI } from './lib/CustomAPI'; +export { default as TemplateGenerator } from './lib/TemplateGenerator'; +export * from './lib/passwordHelpers'; diff --git a/packages/nocodb-sdk/src/lib/TemplateGenerator.ts b/packages/nocodb-sdk/src/lib/TemplateGenerator.ts new file mode 100644 index 0000000000..ad74741b65 --- /dev/null +++ b/packages/nocodb-sdk/src/lib/TemplateGenerator.ts @@ -0,0 +1,30 @@ +import UITypes from './UITypes'; + +export interface Column { + column_name: string; + ref_column_name: string; + uidt?: UITypes; + dtxp?: any; + dt?: any; +} +export interface Table { + table_name: string; + ref_table_name: string; + columns: Array; +} +export interface Template { + title: string; + tables: Array; +} + +export default abstract class TemplateGenerator { + abstract parse(): Promise; + abstract parseTemplate(): Promise