diff --git a/packages/nc-gui/components/ProjectTabs.vue b/packages/nc-gui/components/ProjectTabs.vue index a3fd6392ec..0a69cd3360 100644 --- a/packages/nc-gui/components/ProjectTabs.vue +++ b/packages/nc-gui/components/ProjectTabs.vue @@ -315,6 +315,21 @@ + + + + mdi-code-json + + + + JSON file + + + + + + @@ -404,9 +426,11 @@ import GlobalAcl from '~/components/GlobalAcl' import AuditTab from '~/components/project/AuditTab' import QuickImport from '~/components/import/QuickImport' import ImportFromAirtable from '~/components/import/ImportFromAirtable' +import JsonImport from '~/components/import/JSONImport' export default { components: { + JsonImport, ImportFromAirtable, SwaggerClient, // Screensaver, @@ -447,7 +471,8 @@ export default { showScreensaver: false, quickImportModal: false, quickImportType: '', - airtableImportModal: false + airtableImportModal: false, + jsonImportModal: false } }, methods: { diff --git a/packages/nc-gui/components/import/JSONImport.vue b/packages/nc-gui/components/import/JSONImport.vue new file mode 100644 index 0000000000..73c29de7dd --- /dev/null +++ b/packages/nc-gui/components/import/JSONImport.vue @@ -0,0 +1,484 @@ + + + + + diff --git a/packages/nc-gui/components/import/templateParsers/CSVTemplateAdapter.js b/packages/nc-gui/components/import/templateParsers/CSVTemplateAdapter.js index 9a3be0d4de..c608a79f85 100644 --- a/packages/nc-gui/components/import/templateParsers/CSVTemplateAdapter.js +++ b/packages/nc-gui/components/import/templateParsers/CSVTemplateAdapter.js @@ -1,6 +1,5 @@ import Papaparse from 'papaparse' import TemplateGenerator from '~/components/import/templateParsers/TemplateGenerator' - export default class CSVTemplateAdapter extends TemplateGenerator { constructor(name, data) { super() diff --git a/packages/nc-gui/components/import/templateParsers/JSONTemplateAdapter.js b/packages/nc-gui/components/import/templateParsers/JSONTemplateAdapter.js new file mode 100644 index 0000000000..b53230b789 --- /dev/null +++ b/packages/nc-gui/components/import/templateParsers/JSONTemplateAdapter.js @@ -0,0 +1,150 @@ +import { TemplateGenerator, UITypes } from 'nocodb-sdk' +import { + extractMultiOrSingleSelectProps, + getCheckboxValue, + isCheckboxType, isDecimalType, isEmailType, + isMultiLineTextType, isUrlType +} from '~/components/import/templateParsers/parserHelpers' + +const jsonTypeToUidt = { + number: UITypes.Number, + string: UITypes.SingleLineText, + date: UITypes.DateTime, + boolean: UITypes.Checkbox, + object: UITypes.JSON +} + +const extractNestedData = (obj, path) => path.reduce((val, key) => val && val[key], obj) + +export default class JSONTemplateAdapter extends TemplateGenerator { + constructor(name = 'test', data, parserConfig = {}) { + super() + this.config = { + maxRowsToParse: 500, + ...parserConfig + } + this.name = name + this._jsonData = typeof data === 'string' ? JSON.parse(data) : data + this.project = { + title: this.name, + tables: [] + } + this.data = {} + } + + async init() { + } + + parseData() { + this.columns = this.csv.meta.fields + this.data = this.csv.data + } + + getColumns() { + return this.columns + } + + getData() { + return this.data + } + + get jsonData() { + return Array.isArray(this._jsonData) ? this._jsonData : [this._jsonData] + } + + parse() { + const jsonData = this.jsonData + const tn = 'table' + const table = { table_name: tn, ref_table_name: tn, columns: [] } + + this.data[tn] = [] + + for (const col of Object.keys(jsonData[0])) { + const columns = this._parseColumn([col], jsonData) + table.columns.push(...columns) + } + + if (this.config.importData) { this._parseTableData(table) } + + this.project.tables.push(table) + } + + getTemplate() { + return this.project + } + + _parseColumn(path = [], jsonData = this.jsonData, firstRowVal = path.reduce((val, k) => val && val[k], this.jsonData[0])) { + const columns = [] + // parse nested + if (firstRowVal && typeof firstRowVal === 'object' && !Array.isArray(firstRowVal) && this.config.normalizeNested) { + for (const key of Object.keys(firstRowVal)) { + const normalizedNestedColumns = this._parseColumn([...path, key], this.jsonData, firstRowVal[key]) + columns.push(...normalizedNestedColumns) + } + } else { + const cn = path.join('_').replace(/\W/g, '_').trim() + + const column = { + column_name: cn, + ref_column_name: cn, + path + } + + column.uidt = jsonTypeToUidt[typeof firstRowVal] || UITypes.SingleLineText + + const colData = jsonData.map(r => extractNestedData(r, path)) + Object.assign(column, this._getColumnUIDTAndMetas(colData, column.uidt)) + columns.push(column) + } + + return columns + } + + _getColumnUIDTAndMetas(colData, defaultType) { + const colProps = { uidt: defaultType } + // todo: optimize + if (colProps.uidt === UITypes.SingleLineText) { + // check for long text + if (isMultiLineTextType(colData)) { + colProps.uidt = UITypes.LongText + } if (isEmailType(colData)) { + colProps.uidt = UITypes.Email + } if (isUrlType(colData)) { + colProps.uidt = UITypes.URL + } else { + const checkboxType = isCheckboxType(colData) + if (checkboxType.length === 1) { + colProps.uidt = UITypes.Checkbox + } else { + Object.assign(colProps, extractMultiOrSingleSelectProps(colData)) + } + } + } else if (colProps.uidt === UITypes.Number) { + if (isDecimalType(colData)) { + colProps.uidt = UITypes.Decimal + } + } + return colProps + } + + _parseTableData(tableMeta) { + for (const row of this.jsonData) { + const rowData = {} + for (let i = 0; i < tableMeta.columns.length; i++) { + const value = extractNestedData(row, tableMeta.columns[i].path || []) + if (tableMeta.columns[i].uidt === UITypes.Checkbox) { + rowData[tableMeta.columns[i].ref_column_name] = getCheckboxValue(value) + } else if (tableMeta.columns[i].uidt === UITypes.SingleSelect || tableMeta.columns[i].uidt === UITypes.MultiSelect) { + rowData[tableMeta.columns[i].ref_column_name] = (value || '').toString().trim() || null + } else if (tableMeta.columns[i].uidt === UITypes.JSON) { + rowData[tableMeta.columns[i].ref_column_name] = JSON.stringify(value) + } else { + // toto: do parsing if necessary based on type + rowData[tableMeta.columns[i].column_name] = value + } + } + this.data[tableMeta.ref_table_name].push(rowData) + // rowIndex++ + } + } +} diff --git a/packages/nc-gui/components/import/templateParsers/JSONUrlTemplateAdapter.js b/packages/nc-gui/components/import/templateParsers/JSONUrlTemplateAdapter.js new file mode 100644 index 0000000000..34494e00fc --- /dev/null +++ b/packages/nc-gui/components/import/templateParsers/JSONUrlTemplateAdapter.js @@ -0,0 +1,21 @@ +import JSONTemplateAdapter from '~/components/import/templateParsers/JSONTemplateAdapter' + +export default class JSONUrlTemplateAdapter extends JSONTemplateAdapter { + constructor(url, $store, parserConfig, $api) { + const name = url.split('/').pop() + super(name, null, parserConfig) + this.url = url + this.$api = $api + this.$store = $store + } + + async init() { + const data = await this.$api.utils.axiosRequestMake({ + apiMeta: { + url: this.url + } + }) + this._jsonData = data + await super.init() + } +} diff --git a/packages/nc-gui/components/import/templateParsers/parserHelpers.js b/packages/nc-gui/components/import/templateParsers/parserHelpers.js index 75fd80ead7..43156d3bea 100644 --- a/packages/nc-gui/components/import/templateParsers/parserHelpers.js +++ b/packages/nc-gui/components/import/templateParsers/parserHelpers.js @@ -1,3 +1,6 @@ +import { UITypes } from 'nocodb-sdk' +import { isEmail, isValidURL } from '~/helpers' + const booleanOptions = [ { checked: true, unchecked: false }, { x: true, '': false }, @@ -11,14 +14,24 @@ const booleanOptions = [ { '✔': true, '': false }, { enabled: true, disabled: false }, { on: true, off: false }, - { done: true, '': false } + { done: true, '': false }, + { true: true, false: false } ] const aggBooleanOptions = booleanOptions.reduce((obj, o) => ({ ...obj, ...o }), {}) -export const isCheckboxType = (values, col = '') => { + +const getColVal = (row, col = null) => { + return row && col ? row[col] : row +} + +export const isCheckboxType = (values, col = null) => { let options = booleanOptions for (let i = 0; i < values.length; i++) { - let val = col ? values[i][col] : values[i] - val = val === null || val === undefined ? '' : val + const val = getColVal(values[i], col) + + if (val === null || val === undefined || val.toString().trim() === '') { + continue + } + options = options.filter(v => val in v) if (!options.length) { return false @@ -29,3 +42,40 @@ export const isCheckboxType = (values, col = '') => { export const getCheckboxValue = (value) => { return value && aggBooleanOptions[value] } + +export const isMultiLineTextType = (values, col = null) => { + return values.some(r => + (getColVal(r, col) || '').toString().match(/[\r\n]/) || + (getColVal(r, col) || '').toString().length > 255) +} + +export const extractMultiOrSingleSelectProps = (colData) => { + const colProps = {} + if (colData.some(v => v && (v || '').toString().includes(','))) { + let flattenedVals = colData.flatMap(v => v ? v.toString().trim().split(/\s*,\s*/) : []) + const uniqueVals = flattenedVals = flattenedVals + .filter((v, i, arr) => i === arr.findIndex(v1 => v.toLowerCase() === v1.toLowerCase())) + if (flattenedVals.length > uniqueVals.length && uniqueVals.length <= Math.ceil(flattenedVals.length / 2)) { + colProps.uidt = UITypes.MultiSelect + colProps.dtxp = `'${uniqueVals.join("','")}'` + } + } else { + const uniqueVals = colData.map(v => (v || '').toString().trim()).filter((v, i, arr) => i === arr.findIndex(v1 => v.toLowerCase() === v1.toLowerCase())) + if (colData.length > uniqueVals.length && uniqueVals.length <= Math.ceil(colData.length / 2)) { + colProps.uidt = UITypes.SingleSelect + colProps.dtxp = `'${uniqueVals.join("','")}'` + } + } + return colProps +} + +export const isDecimalType = colData => colData.some((v) => { + return v && parseInt(+v) !== +v +}) + +export const isEmailType = colData => !colData.some((v) => { + return v && !isEmail(v) +}) +export const isUrlType = colData => !colData.some((v) => { + return v && !isValidURL(v) +}) diff --git a/packages/nc-gui/components/monaco/MonacoJsonEditor.js b/packages/nc-gui/components/monaco/MonacoJsonEditor.js index 1362ac4f1c..aa05a99738 100644 --- a/packages/nc-gui/components/monaco/MonacoJsonEditor.js +++ b/packages/nc-gui/components/monaco/MonacoJsonEditor.js @@ -83,6 +83,9 @@ export default { }, methods: { + format() { + this.editor.getAction('editor.action.formatDocument').run() + }, resizeLayout() { this.editor.layout(); }, diff --git a/packages/nocodb-sdk/src/index.ts b/packages/nocodb-sdk/src/index.ts index 805152d812..561b841c36 100644 --- a/packages/nocodb-sdk/src/index.ts +++ b/packages/nocodb-sdk/src/index.ts @@ -5,6 +5,7 @@ export * from './lib/sqlUi'; export * from './lib/globals'; export * from './lib/helperFunctions'; export * from './lib/formulaHelpers'; -export * from './lib/passwordHelpers'; export { default as UITypes, isVirtualCol } from './lib/UITypes'; export { default as CustomAPI } from './lib/CustomAPI'; +export { default as TemplateGenerator } from './lib/TemplateGenerator'; +export * from './lib/passwordHelpers'; diff --git a/packages/nocodb-sdk/src/lib/TemplateGenerator.ts b/packages/nocodb-sdk/src/lib/TemplateGenerator.ts new file mode 100644 index 0000000000..ad74741b65 --- /dev/null +++ b/packages/nocodb-sdk/src/lib/TemplateGenerator.ts @@ -0,0 +1,30 @@ +import UITypes from './UITypes'; + +export interface Column { + column_name: string; + ref_column_name: string; + uidt?: UITypes; + dtxp?: any; + dt?: any; +} +export interface Table { + table_name: string; + ref_table_name: string; + columns: Array; +} +export interface Template { + title: string; + tables: Array; +} + +export default abstract class TemplateGenerator { + abstract parse(): Promise; + abstract parseTemplate(): Promise