@ -1,15 +1,21 @@
import { TemplateGenerator } from 'nocodb-sdk'
import { UITypes } from '~/components/project/spreadsheet/helpers/uiTypes'
import { getCheckboxValue , isCheckboxType } from '~/components/import/templateParsers/parserHelpers'
import { TemplateGenerator , UITypes } from 'nocodb-sdk'
import {
extractMultiOrSingleSelectProps ,
getCheckboxValue ,
isCheckboxType , isDecimalType , isEmailType ,
isMultiLineTextType , isUrlType
} from '~/components/import/templateParsers/parserHelpers'
const jsonTypeToUidt = {
number : UITypes . Number ,
string : UITypes . SingleLineText ,
date : UITypes . DateTime ,
boolean : UITypes . Checkbox ,
object : UITypes . LongText
object : UITypes . JSON
}
const extractNestedData = ( obj , path ) => path . reduce ( ( val , key ) => val && val [ key ] , obj )
export default class JSONTemplateAdapter extends TemplateGenerator {
constructor ( name = 'test' , data , parserConfig = { } ) {
super ( )
@ -18,7 +24,7 @@ export default class JSONTemplateAdapter extends TemplateGenerator {
... parserConfig
}
this . name = name
this . jsonData = typeof data === 'string' ? JSON . parse ( data ) : data
this . _ jsonData= typeof data === 'string' ? JSON . parse ( data ) : data
this . project = {
title : this . name ,
tables : [ ]
@ -42,117 +48,103 @@ export default class JSONTemplateAdapter extends TemplateGenerator {
return this . data
}
parse ( ) {
const jsonData = Array . isArray ( this . this . jsonData ) ? this . this . jsonData : [ this . jsonData ]
// for (let i = 0; i < this.wb.SheetNames.length; i++) {
// const columnNamePrefixRef = { id: 0 }
get jsonData ( ) {
return Array . isArray ( this . _jsonData ) ? this . _jsonData : [ this . _jsonData ]
}
parse ( ) {
const jsonData = this . jsonData
const tn = 'table'
const table = { table _name : tn , ref _table _name : tn , columns : [ ] }
this . data [ tn ] = [ ]
// const ws = this.wb.Sheets[sheet]
// const range = XLSX.utils.decode_range(ws['!ref'])
// const rows = XLSX.utils.sheet_to_json(ws, { header: 1, blankrows: false, cellDates: true, defval: null })
for ( const col of Object . keys ( jsonData [ 0 ] ) ) {
const columns = this . _parseColumn ( [ col ] , jsonData )
table . columns . push ( ... columns )
}
if ( this . config . importData ) { this . _parseTableData ( table ) }
this . project . tables . push ( table )
}
const objKeys = Object . keys ( jsonData [ 0 ] )
getTemplate ( ) {
return this . project
}
for ( let col = 0 ; col < objKeys . length ; col ++ ) {
const key = objKeys [ col ]
const cn = objKeys [ col ] . replace ( /\W/g , '_' ) . trim ( )
_parseColumn ( path = [ ] , jsonData = this . jsonData , firstRowVal = path . reduce ( ( val , k ) => val && val [ k ] , this . jsonData [ 0 ] ) ) {
const columns = [ ]
// parse nested
if ( firstRowVal && typeof firstRowVal === 'object' && ! Array . isArray ( firstRowVal ) && this . config . normalizeNested ) {
for ( const key of Object . keys ( firstRowVal ) ) {
const normalizedNestedColumns = this . _parseColumn ( [ ... path , key ] , this . jsonData , firstRowVal [ key ] )
columns . push ( ... normalizedNestedColumns )
}
} else {
const cn = path . join ( '_' ) . replace ( /\W/g , '_' ) . trim ( )
const column = {
column _name : cn ,
ref _column _name : cn
ref _column _name : cn ,
path
}
table . columns . push ( column )
column . uidt = jsonTypeToUidt [ typeof firstRowVal ] || UITypes . SingleLineText
const colData = jsonData . map ( r => extractNestedData ( r , path ) )
Object . assign ( column , this . _getColumnUIDTAndMetas ( colData , column . uidt ) )
columns . push ( column )
}
column . uidt = jsonTypeToUidt [ typeof jsonData [ 0 ] [ key ] ] || UITypes . SingleLineText
return columns
}
// todo: optimize
if ( column . uidt === UITypes . SingleLineText ) {
// check for long text
if ( jsonData . some ( r =>
( r [ key ] || '' ) . toString ( ) . match ( /[\r\n]/ ) ||
( r [ key ] || '' ) . toString ( ) . length > 255 )
) {
column . uidt = UITypes . LongText
_getColumnUIDTAndMetas ( colData , defaultType ) {
const colProps = { uidt : defaultType }
// todo: optimize
if ( colProps . uidt === UITypes . SingleLineText ) {
// check for long text
if ( isMultiLineTextType ( colData ) ) {
colProps . uidt = UITypes . LongText
} if ( isEmailType ( colData ) ) {
colProps . uidt = UITypes . Email
} if ( isUrlType ( colData ) ) {
colProps . uidt = UITypes . URL
} else {
const checkboxType = isCheckboxType ( colData )
if ( checkboxType . length === 1 ) {
colProps . uidt = UITypes . Checkbox
} else {
const vals = jsonData
. map ( r => r [ key ] )
. filter ( v => v !== null && v !== undefined && v . toString ( ) . trim ( ) !== '' )
const checkboxType = isCheckboxType ( vals )
if ( checkboxType . length === 1 ) {
column . uidt = UITypes . Checkbox
} else {
// todo: optimize
// check column is multi or single select by comparing unique values
// todo:
// eslint-disable-next-line no-lonely-if
if ( vals . some ( v => v && v . toString ( ) . includes ( ',' ) ) ) {
let flattenedVals = vals . flatMap ( v => v ? v . toString ( ) . trim ( ) . split ( /\s*,\s*/ ) : [ ] )
const uniqueVals = flattenedVals = flattenedVals
. filter ( ( v , i , arr ) => i === arr . findIndex ( v1 => v . toLowerCase ( ) === v1 . toLowerCase ( ) ) )
if ( flattenedVals . length > uniqueVals . length && uniqueVals . length <= Math . ceil ( flattenedVals . length / 2 ) ) {
column . uidt = UITypes . MultiSelect
column . dtxp = ` ' ${ uniqueVals . join ( "','" ) } ' `
}
} else {
const uniqueVals = vals . map ( v => v . toString ( ) . trim ( ) ) . filter ( ( v , i , arr ) => i === arr . findIndex ( v1 => v . toLowerCase ( ) === v1 . toLowerCase ( ) ) )
if ( vals . length > uniqueVals . length && uniqueVals . length <= Math . ceil ( vals . length / 2 ) ) {
column . uidt = UITypes . SingleSelect
column . dtxp = ` ' ${ uniqueVals . join ( "','" ) } ' `
}
}
}
}
} else if ( column . uidt === UITypes . Number ) {
if ( jsonData . slice ( 1 , this . config . maxRowsToParse ) . some ( ( v ) => {
return v && v [ key ] && parseInt ( + v [ key ] ) !== + v [ key ]
} ) ) {
column . uidt = UITypes . Decimal
}
if ( jsonData . every ( ( v , i ) => {
return v [ key ] && v [ key ] . toString ( ) . startsWith ( '$' )
} ) ) {
column . uidt = UITypes . Currency
}
} else if ( column . uidt === UITypes . DateTime ) {
if ( jsonData . every ( ( v , i ) => {
return v [ key ] && v [ key ] . toString ( ) . split ( ' ' ) . length === 1
} ) ) {
column . uidt = UITypes . Date
Object . assign ( colProps , extractMultiOrSingleSelectProps ( colData ) )
}
}
} else if ( colProps . uidt === UITypes . Number ) {
if ( isDecimalType ( colData ) ) {
colProps . uidt = UITypes . Decimal
}
}
return colProps
}
// let rowIndex = 0
for ( const row of jsonData ) {
_parseTableData ( tableMeta ) {
for ( const row of this . jsonData ) {
const rowData = { }
for ( let i = 0 ; i < table . columns . length ; i ++ ) {
if ( table . columns [ i ] . uidt === UITypes . Checkbox ) {
rowData [ table . columns [ i ] . column _name ] = getCheckboxValue ( row [ i ] )
} else if ( table . columns [ i ] . uidt === UITypes . Currency ) {
rowData [ table . columns [ i ] . column _name ] = ( row [ table . columns [ i ] . ref _column _name ] . replace ( /[^\d.]+/g , '' ) ) || row [ i ]
} else if ( table . columns [ i ] . uidt === UITypes . SingleSelect || table . columns [ i ] . uidt === UITypes . MultiSelect ) {
rowData [ table . columns [ i ] . column _name ] = ( row [ table . columns [ i ] . ref _column _name ] || '' ) . toString ( ) . trim ( ) || null
for ( let i = 0 ; i < tableMeta . columns . length ; i ++ ) {
const value = extractNestedData ( row , tableMeta . columns [ i ] . path || [ ] )
if ( tableMeta . columns [ i ] . uidt === UITypes . Checkbox ) {
rowData [ tableMeta . columns [ i ] . ref _column _name ] = getCheckboxValue ( value )
} else if ( tableMeta . columns [ i ] . uidt === UITypes . SingleSelect || tableMeta . columns [ i ] . uidt === UITypes . MultiSelect ) {
rowData [ tableMeta . columns [ i ] . ref _column _name ] = ( value || '' ) . toString ( ) . trim ( ) || null
} else if ( tableMeta . columns [ i ] . uidt === UITypes . JSON ) {
rowData [ tableMeta . columns [ i ] . ref _column _name ] = JSON . stringify ( value )
} else {
// toto: do parsing if necessary based on type
rowData [ table . columns [ i ] . column _name ] = row [ table . columns [ i ] . ref _column _name ]
rowData [ tableMeta . columns [ i ] . column _name ] = value
}
}
this . data [ tn ] . push ( rowData )
this . data [ tableMeta . ref _table _name ] . push ( rowData )
// rowIndex++
}
this . project . tables . push ( table )
}
getTemplate ( ) {
return this . project
}
}
Pranav C
2 years ago